drivers/mmc/host/sdhci.c

   1 /*
   2  *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
   3  *
   4  *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or (at
   9  * your option) any later version.
  10  *
  11  * Thanks to the following companies for their support:
  12  *
  13  *     - JMicron (hardware and technical support)
  14  */
  15
  16 #include <linux/delay.h>
  17 #include <linux/ktime.h>
  18 #include <linux/highmem.h>
  19 #include <linux/io.h>
  20 #include <linux/module.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/slab.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/sizes.h>
  25 #include <linux/swiotlb.h>
  26 #include <linux/regulator/consumer.h>
  27 #include <linux/pm_runtime.h>
  28 #include <linux/of.h>
  29
  30 #include <linux/leds.h>
  31
  32 #include <linux/mmc/mmc.h>
  33 #include <linux/mmc/host.h>
  34 #include <linux/mmc/card.h>
  35 #include <linux/mmc/sdio.h>
  36 #include <linux/mmc/slot-gpio.h>
  37
  38 #include "sdhci.h"
  39
  40 #define DRIVER_NAME "sdhci"
  41
  42 #define DBG(f, x...) \
  43         pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  44
  45 #define SDHCI_DUMP(f, x...) \
  46         pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
  47
  48 #define MAX_TUNING_LOOP 40
  49
  50 static unsigned int debug_quirks = 0;
  51 static unsigned int debug_quirks2;
  52
  53 static void sdhci_finish_data(struct sdhci_host *);
  54
  55 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable);
  56
  57 void sdhci_dumpregs(struct sdhci_host *host)
  58 {
  59         SDHCI_DUMP("============ SDHCI REGISTER DUMP ===========\n");
  60
  61         SDHCI_DUMP("Sys addr:  0x%08x | Version:  0x%08x\n",
  62                    sdhci_readl(host, SDHCI_DMA_ADDRESS),
  63                    sdhci_readw(host, SDHCI_HOST_VERSION));
  64         SDHCI_DUMP("Blk size:  0x%08x | Blk cnt:  0x%08x\n",
  65                    sdhci_readw(host, SDHCI_BLOCK_SIZE),
  66                    sdhci_readw(host, SDHCI_BLOCK_COUNT));
  67         SDHCI_DUMP("Argument:  0x%08x | Trn mode: 0x%08x\n",
  68                    sdhci_readl(host, SDHCI_ARGUMENT),
  69                    sdhci_readw(host, SDHCI_TRANSFER_MODE));
  70         SDHCI_DUMP("Present:   0x%08x | Host ctl: 0x%08x\n",
  71                    sdhci_readl(host, SDHCI_PRESENT_STATE),
  72                    sdhci_readb(host, SDHCI_HOST_CONTROL));
  73         SDHCI_DUMP("Power:     0x%08x | Blk gap:  0x%08x\n",
  74                    sdhci_readb(host, SDHCI_POWER_CONTROL),
  75                    sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL));
  76         SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",
  77                    sdhci_readb(host, SDHCI_WAKE_UP_CONTROL),
  78                    sdhci_readw(host, SDHCI_CLOCK_CONTROL));
  79         SDHCI_DUMP("Timeout:   0x%08x | Int stat: 0x%08x\n",
  80                    sdhci_readb(host, SDHCI_TIMEOUT_CONTROL),
  81                    sdhci_readl(host, SDHCI_INT_STATUS));
  82         SDHCI_DUMP("Int enab:  0x%08x | Sig enab: 0x%08x\n",
  83                    sdhci_readl(host, SDHCI_INT_ENABLE),
  84                    sdhci_readl(host, SDHCI_SIGNAL_ENABLE));
  85         SDHCI_DUMP("ACmd stat: 0x%08x | Slot int: 0x%08x\n",
  86                    sdhci_readw(host, SDHCI_AUTO_CMD_STATUS),
  87                    sdhci_readw(host, SDHCI_SLOT_INT_STATUS));
  88         SDHCI_DUMP("Caps:      0x%08x | Caps_1:   0x%08x\n",
  89                    sdhci_readl(host, SDHCI_CAPABILITIES),
  90                    sdhci_readl(host, SDHCI_CAPABILITIES_1));
  91         SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",
  92                    sdhci_readw(host, SDHCI_COMMAND),
  93                    sdhci_readl(host, SDHCI_MAX_CURRENT));
  94         SDHCI_DUMP("Resp[0]:   0x%08x | Resp[1]:  0x%08x\n",
  95                    sdhci_readl(host, SDHCI_RESPONSE),
  96                    sdhci_readl(host, SDHCI_RESPONSE + 4));
  97         SDHCI_DUMP("Resp[2]:   0x%08x | Resp[3]:  0x%08x\n",
  98                    sdhci_readl(host, SDHCI_RESPONSE + 8),
  99                    sdhci_readl(host, SDHCI_RESPONSE + 12));
 100         SDHCI_DUMP("Host ctl2: 0x%08x\n",
 101                    sdhci_readw(host, SDHCI_HOST_CONTROL2));
 102
 103         if (host->flags & SDHCI_USE_ADMA) {
 104                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
 105                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x%08x\n",
 106                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 107                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS_HI),
 108                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 109                 } else {
 110                         SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x\n",
 111                                    sdhci_readl(host, SDHCI_ADMA_ERROR),
 112                                    sdhci_readl(host, SDHCI_ADMA_ADDRESS));
 113                 }
 114         }
 115
 116         SDHCI_DUMP("============================================\n");
 117 }
 118 EXPORT_SYMBOL_GPL(sdhci_dumpregs);
 119
 120 /*****************************************************************************\
 121  *                                                                           *
 122  * Low level functions                                                       *
 123  *                                                                           *
 124 \*****************************************************************************/
 125
 126 static void sdhci_do_enable_v4_mode(struct sdhci_host *host)
 127 {
 128         u16 ctrl2;
 129
 130         ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
 131         if (ctrl2 & SDHCI_CTRL_V4_MODE)
 132                 return;
 133
 134         ctrl2 |= SDHCI_CTRL_V4_MODE;
 135         sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2);
 136 }
 137
 138 /*
 139  * This can be called before sdhci_add_host() by Vendor's host controller
 140  * driver to enable v4 mode if supported.
 141  */
 142 void sdhci_enable_v4_mode(struct sdhci_host *host)
 143 {
 144         host->v4_mode = true;
 145         sdhci_do_enable_v4_mode(host);
 146 }
 147 EXPORT_SYMBOL_GPL(sdhci_enable_v4_mode);
 148
 149 static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)
 150 {
 151         return cmd->data || cmd->flags & MMC_RSP_BUSY;
 152 }
 153
 154 static void sdhci_set_card_detection(struct sdhci_host *host, bool enable)
 155 {
 156         u32 present;
 157
 158         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||
 159             !mmc_card_is_removable(host->mmc))
 160                 return;
 161
 162         if (enable) {
 163                 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
 164                                       SDHCI_CARD_PRESENT;
 165
 166                 host->ier |= present ? SDHCI_INT_CARD_REMOVE :
 167                                        SDHCI_INT_CARD_INSERT;
 168         } else {
 169                 host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
 170         }
 171
 172         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 173         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 174 }
 175
 176 static void sdhci_enable_card_detection(struct sdhci_host *host)
 177 {
 178         sdhci_set_card_detection(host, true);
 179 }
 180
 181 static void sdhci_disable_card_detection(struct sdhci_host *host)
 182 {
 183         sdhci_set_card_detection(host, false);
 184 }
 185
 186 static void sdhci_runtime_pm_bus_on(struct sdhci_host *host)
 187 {
 188         if (host->bus_on)
 189                 return;
 190         host->bus_on = true;
 191         pm_runtime_get_noresume(host->mmc->parent);
 192 }
 193
 194 static void sdhci_runtime_pm_bus_off(struct sdhci_host *host)
 195 {
 196         if (!host->bus_on)
 197                 return;
 198         host->bus_on = false;
 199         pm_runtime_put_noidle(host->mmc->parent);
 200 }
 201
 202 void sdhci_reset(struct sdhci_host *host, u8 mask)
 203 {
 204         ktime_t timeout;
 205
 206         sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET);
 207
 208         if (mask & SDHCI_RESET_ALL) {
 209                 host->clock = 0;
 210                 /* Reset-all turns off SD Bus Power */
 211                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
 212                         sdhci_runtime_pm_bus_off(host);
 213         }
 214
 215         /* Wait max 100 ms */
 216         timeout = ktime_add_ms(ktime_get(), 100);
 217
 218         /* hw clears the bit when it's done */
 219         while (1) {
 220                 bool timedout = ktime_after(ktime_get(), timeout);
 221
 222                 if (!(sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask))
 223                         break;
 224                 if (timedout) {
 225                         pr_err("%s: Reset 0x%x never completed.\n",
 226                                 mmc_hostname(host->mmc), (int)mask);
 227                         sdhci_dumpregs(host);
 228                         return;
 229                 }
 230                 udelay(10);
 231         }
 232 }
 233 EXPORT_SYMBOL_GPL(sdhci_reset);
 234
 235 static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
 236 {
 237         if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
 238                 struct mmc_host *mmc = host->mmc;
 239
 240                 if (!mmc->ops->get_cd(mmc))
 241                         return;
 242         }
 243
 244         host->ops->reset(host, mask);
 245
 246         if (mask & SDHCI_RESET_ALL) {
 247                 if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 248                         if (host->ops->enable_dma)
 249                                 host->ops->enable_dma(host);
 250                 }
 251
 252                 /* Resetting the controller clears many */
 253                 host->preset_enabled = false;
 254         }
 255 }
 256
 257 static void sdhci_set_default_irqs(struct sdhci_host *host)
 258 {
 259         host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
 260                     SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT |
 261                     SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC |
 262                     SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END |
 263                     SDHCI_INT_RESPONSE;
 264
 265         if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||
 266             host->tuning_mode == SDHCI_TUNING_MODE_3)
 267                 host->ier |= SDHCI_INT_RETUNE;
 268
 269         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 270         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 271 }
 272
 273 static void sdhci_config_dma(struct sdhci_host *host)
 274 {
 275         u8 ctrl;
 276         u16 ctrl2;
 277
 278         if (host->version < SDHCI_SPEC_200)
 279                 return;
 280
 281         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 282
 283         /*
 284          * Always adjust the DMA selection as some controllers
 285          * (e.g. JMicron) can't do PIO properly when the selection
 286          * is ADMA.
 287          */
 288         ctrl &= ~SDHCI_CTRL_DMA_MASK;
 289         if (!(host->flags & SDHCI_REQ_USE_DMA))
 290                 goto out;
 291
 292         /* Note if DMA Select is zero then SDMA is selected */
 293         if (host->flags & SDHCI_USE_ADMA)
 294                 ctrl |= SDHCI_CTRL_ADMA32;
 295
 296         if (host->flags & SDHCI_USE_64_BIT_DMA) {
 297                 /*
 298                  * If v4 mode, all supported DMA can be 64-bit addressing if
 299                  * controller supports 64-bit system address, otherwise only
 300                  * ADMA can support 64-bit addressing.
 301                  */
 302                 if (host->v4_mode) {
 303                         ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
 304                         ctrl2 |= SDHCI_CTRL_64BIT_ADDR;
 305                         sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2);
 306                 } else if (host->flags & SDHCI_USE_ADMA) {
 307                         /*
 308                          * Don't need to undo SDHCI_CTRL_ADMA32 in order to
 309                          * set SDHCI_CTRL_ADMA64.
 310                          */
 311                         ctrl |= SDHCI_CTRL_ADMA64;
 312                 }
 313         }
 314
 315 out:
 316         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 317 }
 318
 319 static void sdhci_init(struct sdhci_host *host, int soft)
 320 {
 321         struct mmc_host *mmc = host->mmc;
 322
 323         if (soft)
 324                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 325         else
 326                 sdhci_do_reset(host, SDHCI_RESET_ALL);
 327
 328         if (host->v4_mode)
 329                 sdhci_do_enable_v4_mode(host);
 330
 331         sdhci_set_default_irqs(host);
 332
 333         host->cqe_on = false;
 334
 335         if (soft) {
 336                 /* force clock reconfiguration */
 337                 host->clock = 0;
 338                 mmc->ops->set_ios(mmc, &mmc->ios);
 339         }
 340 }
 341
 342 static void sdhci_reinit(struct sdhci_host *host)
 343 {
 344         sdhci_init(host, 0);
 345         sdhci_enable_card_detection(host);
 346 }
 347
 348 static void __sdhci_led_activate(struct sdhci_host *host)
 349 {
 350         u8 ctrl;
 351
 352         if (host->quirks & SDHCI_QUIRK_NO_LED)
 353                 return;
 354
 355         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 356         ctrl |= SDHCI_CTRL_LED;
 357         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 358 }
 359
 360 static void __sdhci_led_deactivate(struct sdhci_host *host)
 361 {
 362         u8 ctrl;
 363
 364         if (host->quirks & SDHCI_QUIRK_NO_LED)
 365                 return;
 366
 367         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 368         ctrl &= ~SDHCI_CTRL_LED;
 369         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 370 }
 371
 372 #if IS_REACHABLE(CONFIG_LEDS_CLASS)
 373 static void sdhci_led_control(struct led_classdev *led,
 374                               enum led_brightness brightness)
 375 {
 376         struct sdhci_host *host = container_of(led, struct sdhci_host, led);
 377         unsigned long flags;
 378
 379         spin_lock_irqsave(&host->lock, flags);
 380
 381         if (host->runtime_suspended)
 382                 goto out;
 383
 384         if (brightness == LED_OFF)
 385                 __sdhci_led_deactivate(host);
 386         else
 387                 __sdhci_led_activate(host);
 388 out:
 389         spin_unlock_irqrestore(&host->lock, flags);
 390 }
 391
 392 static int sdhci_led_register(struct sdhci_host *host)
 393 {
 394         struct mmc_host *mmc = host->mmc;
 395
 396         if (host->quirks & SDHCI_QUIRK_NO_LED)
 397                 return 0;
 398
 399         snprintf(host->led_name, sizeof(host->led_name),
 400                  "%s::", mmc_hostname(mmc));
 401
 402         host->led.name = host->led_name;
 403         host->led.brightness = LED_OFF;
 404         host->led.default_trigger = mmc_hostname(mmc);
 405         host->led.brightness_set = sdhci_led_control;
 406
 407         return led_classdev_register(mmc_dev(mmc), &host->led);
 408 }
 409
 410 static void sdhci_led_unregister(struct sdhci_host *host)
 411 {
 412         if (host->quirks & SDHCI_QUIRK_NO_LED)
 413                 return;
 414
 415         led_classdev_unregister(&host->led);
 416 }
 417
 418 static inline void sdhci_led_activate(struct sdhci_host *host)
 419 {
 420 }
 421
 422 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 423 {
 424 }
 425
 426 #else
 427
 428 static inline int sdhci_led_register(struct sdhci_host *host)
 429 {
 430         return 0;
 431 }
 432
 433 static inline void sdhci_led_unregister(struct sdhci_host *host)
 434 {
 435 }
 436
 437 static inline void sdhci_led_activate(struct sdhci_host *host)
 438 {
 439         __sdhci_led_activate(host);
 440 }
 441
 442 static inline void sdhci_led_deactivate(struct sdhci_host *host)
 443 {
 444         __sdhci_led_deactivate(host);
 445 }
 446
 447 #endif
 448
 449 /*****************************************************************************\
 450  *                                                                           *
 451  * Core functions                                                            *
 452  *                                                                           *
 453 \*****************************************************************************/
 454
 455 static void sdhci_read_block_pio(struct sdhci_host *host)
 456 {
 457         unsigned long flags;
 458         size_t blksize, len, chunk;
 459         u32 uninitialized_var(scratch);
 460         u8 *buf;
 461
 462         DBG("PIO reading\n");
 463
 464         blksize = host->data->blksz;
 465         chunk = 0;
 466
 467         local_irq_save(flags);
 468
 469         while (blksize) {
 470                 BUG_ON(!sg_miter_next(&host->sg_miter));
 471
 472                 len = min(host->sg_miter.length, blksize);
 473
 474                 blksize -= len;
 475                 host->sg_miter.consumed = len;
 476
 477                 buf = host->sg_miter.addr;
 478
 479                 while (len) {
 480                         if (chunk == 0) {
 481                                 scratch = sdhci_readl(host, SDHCI_BUFFER);
 482                                 chunk = 4;
 483                         }
 484
 485                         *buf = scratch & 0xFF;
 486
 487                         buf++;
 488                         scratch >>= 8;
 489                         chunk--;
 490                         len--;
 491                 }
 492         }
 493
 494         sg_miter_stop(&host->sg_miter);
 495
 496         local_irq_restore(flags);
 497 }
 498
 499 static void sdhci_write_block_pio(struct sdhci_host *host)
 500 {
 501         unsigned long flags;
 502         size_t blksize, len, chunk;
 503         u32 scratch;
 504         u8 *buf;
 505
 506         DBG("PIO writing\n");
 507
 508         blksize = host->data->blksz;
 509         chunk = 0;
 510         scratch = 0;
 511
 512         local_irq_save(flags);
 513
 514         while (blksize) {
 515                 BUG_ON(!sg_miter_next(&host->sg_miter));
 516
 517                 len = min(host->sg_miter.length, blksize);
 518
 519                 blksize -= len;
 520                 host->sg_miter.consumed = len;
 521
 522                 buf = host->sg_miter.addr;
 523
 524                 while (len) {
 525                         scratch |= (u32)*buf << (chunk * 8);
 526
 527                         buf++;
 528                         chunk++;
 529                         len--;
 530
 531                         if ((chunk == 4) || ((len == 0) && (blksize == 0))) {
 532                                 sdhci_writel(host, scratch, SDHCI_BUFFER);
 533                                 chunk = 0;
 534                                 scratch = 0;
 535                         }
 536                 }
 537         }
 538
 539         sg_miter_stop(&host->sg_miter);
 540
 541         local_irq_restore(flags);
 542 }
 543
 544 static void sdhci_transfer_pio(struct sdhci_host *host)
 545 {
 546         u32 mask;
 547
 548         if (host->blocks == 0)
 549                 return;
 550
 551         if (host->data->flags & MMC_DATA_READ)
 552                 mask = SDHCI_DATA_AVAILABLE;
 553         else
 554                 mask = SDHCI_SPACE_AVAILABLE;
 555
 556         /*
 557          * Some controllers (JMicron JMB38x) mess up the buffer bits
 558          * for transfers < 4 bytes. As long as it is just one block,
 559          * we can ignore the bits.
 560          */
 561         if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&
 562                 (host->data->blocks == 1))
 563                 mask = ~0;
 564
 565         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
 566                 if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)
 567                         udelay(100);
 568
 569                 if (host->data->flags & MMC_DATA_READ)
 570                         sdhci_read_block_pio(host);
 571                 else
 572                         sdhci_write_block_pio(host);
 573
 574                 host->blocks--;
 575                 if (host->blocks == 0)
 576                         break;
 577         }
 578
 579         DBG("PIO transfer complete.\n");
 580 }
 581
 582 static int sdhci_pre_dma_transfer(struct sdhci_host *host,
 583                                   struct mmc_data *data, int cookie)
 584 {
 585         int sg_count;
 586
 587         /*
 588          * If the data buffers are already mapped, return the previous
 589          * dma_map_sg() result.
 590          */
 591         if (data->host_cookie == COOKIE_PRE_MAPPED)
 592                 return data->sg_count;
 593
 594         /* Bounce write requests to the bounce buffer */
 595         if (host->bounce_buffer) {
 596                 unsigned int length = data->blksz * data->blocks;
 597
 598                 if (length > host->bounce_buffer_size) {
 599                         pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
 600                                mmc_hostname(host->mmc), length,
 601                                host->bounce_buffer_size);
 602                         return -EIO;
 603                 }
 604                 if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
 605                         /* Copy the data to the bounce buffer */
 606                         sg_copy_to_buffer(data->sg, data->sg_len,
 607                                           host->bounce_buffer,
 608                                           length);
 609                 }
 610                 /* Switch ownership to the DMA */
 611                 dma_sync_single_for_device(host->mmc->parent,
 612                                            host->bounce_addr,
 613                                            host->bounce_buffer_size,
 614                                            mmc_get_dma_dir(data));
 615                 /* Just a dummy value */
 616                 sg_count = 1;
 617         } else {
 618                 /* Just access the data directly from memory */
 619                 sg_count = dma_map_sg(mmc_dev(host->mmc),
 620                                       data->sg, data->sg_len,
 621                                       mmc_get_dma_dir(data));
 622         }
 623
 624         if (sg_count == 0)
 625                 return -ENOSPC;
 626
 627         data->sg_count = sg_count;
 628         data->host_cookie = cookie;
 629
 630         return sg_count;
 631 }
 632
 633 static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags)
 634 {
 635         local_irq_save(*flags);
 636         return kmap_atomic(sg_page(sg)) + sg->offset;
 637 }
 638
 639 static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags)
 640 {
 641         kunmap_atomic(buffer);
 642         local_irq_restore(*flags);
 643 }
 644
 645 void sdhci_adma_write_desc(struct sdhci_host *host, void **desc,
 646                            dma_addr_t addr, int len, unsigned int cmd)
 647 {
 648         struct sdhci_adma2_64_desc *dma_desc = *desc;
 649
 650         /* 32-bit and 64-bit descriptors have these members in same position */
 651         dma_desc->cmd = cpu_to_le16(cmd);
 652         dma_desc->len = cpu_to_le16(len);
 653         dma_desc->addr_lo = cpu_to_le32((u32)addr);
 654
 655         if (host->flags & SDHCI_USE_64_BIT_DMA)
 656                 dma_desc->addr_hi = cpu_to_le32((u64)addr >> 32);
 657
 658         *desc += host->desc_sz;
 659 }
 660 EXPORT_SYMBOL_GPL(sdhci_adma_write_desc);
 661
 662 static inline void __sdhci_adma_write_desc(struct sdhci_host *host,
 663                                            void **desc, dma_addr_t addr,
 664                                            int len, unsigned int cmd)
 665 {
 666         if (host->ops->adma_write_desc)
 667                 host->ops->adma_write_desc(host, desc, addr, len, cmd);
 668         else
 669                 sdhci_adma_write_desc(host, desc, addr, len, cmd);
 670 }
 671
 672 static void sdhci_adma_mark_end(void *desc)
 673 {
 674         struct sdhci_adma2_64_desc *dma_desc = desc;
 675
 676         /* 32-bit and 64-bit descriptors have 'cmd' in same position */
 677         dma_desc->cmd |= cpu_to_le16(ADMA2_END);
 678 }
 679
 680 static void sdhci_adma_table_pre(struct sdhci_host *host,
 681         struct mmc_data *data, int sg_count)
 682 {
 683         struct scatterlist *sg;
 684         unsigned long flags;
 685         dma_addr_t addr, align_addr;
 686         void *desc, *align;
 687         char *buffer;
 688         int len, offset, i;
 689
 690         /*
 691          * The spec does not specify endianness of descriptor table.
 692          * We currently guess that it is LE.
 693          */
 694
 695         host->sg_count = sg_count;
 696
 697         desc = host->adma_table;
 698         align = host->align_buffer;
 699
 700         align_addr = host->align_addr;
 701
 702         for_each_sg(data->sg, sg, host->sg_count, i) {
 703                 addr = sg_dma_address(sg);
 704                 len = sg_dma_len(sg);
 705
 706                 /*
 707                  * The SDHCI specification states that ADMA addresses must
 708                  * be 32-bit aligned. If they aren't, then we use a bounce
 709                  * buffer for the (up to three) bytes that screw up the
 710                  * alignment.
 711                  */
 712                 offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &
 713                          SDHCI_ADMA2_MASK;
 714                 if (offset) {
 715                         if (data->flags & MMC_DATA_WRITE) {
 716                                 buffer = sdhci_kmap_atomic(sg, &flags);
 717                                 memcpy(align, buffer, offset);
 718                                 sdhci_kunmap_atomic(buffer, &flags);
 719                         }
 720
 721                         /* tran, valid */
 722                         __sdhci_adma_write_desc(host, &desc, align_addr,
 723                                                 offset, ADMA2_TRAN_VALID);
 724
 725                         BUG_ON(offset > 65536);
 726
 727                         align += SDHCI_ADMA2_ALIGN;
 728                         align_addr += SDHCI_ADMA2_ALIGN;
 729
 730                         addr += offset;
 731                         len -= offset;
 732                 }
 733
 734                 BUG_ON(len > 65536);
 735
 736                 /* tran, valid */
 737                 if (len)
 738                         __sdhci_adma_write_desc(host, &desc, addr, len,
 739                                                 ADMA2_TRAN_VALID);
 740
 741                 /*
 742                  * If this triggers then we have a calculation bug
 743                  * somewhere. :/
 744                  */
 745                 WARN_ON((desc - host->adma_table) >= host->adma_table_sz);
 746         }
 747
 748         if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {
 749                 /* Mark the last descriptor as the terminating descriptor */
 750                 if (desc != host->adma_table) {
 751                         desc -= host->desc_sz;
 752                         sdhci_adma_mark_end(desc);
 753                 }
 754         } else {
 755                 /* Add a terminating entry - nop, end, valid */
 756                 __sdhci_adma_write_desc(host, &desc, 0, 0, ADMA2_NOP_END_VALID);
 757         }
 758 }
 759
 760 static void sdhci_adma_table_post(struct sdhci_host *host,
 761         struct mmc_data *data)
 762 {
 763         struct scatterlist *sg;
 764         int i, size;
 765         void *align;
 766         char *buffer;
 767         unsigned long flags;
 768
 769         if (data->flags & MMC_DATA_READ) {
 770                 bool has_unaligned = false;
 771
 772                 /* Do a quick scan of the SG list for any unaligned mappings */
 773                 for_each_sg(data->sg, sg, host->sg_count, i)
 774                         if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 775                                 has_unaligned = true;
 776                                 break;
 777                         }
 778
 779                 if (has_unaligned) {
 780                         dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,
 781                                             data->sg_len, DMA_FROM_DEVICE);
 782
 783                         align = host->align_buffer;
 784
 785                         for_each_sg(data->sg, sg, host->sg_count, i) {
 786                                 if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
 787                                         size = SDHCI_ADMA2_ALIGN -
 788                                                (sg_dma_address(sg) & SDHCI_ADMA2_MASK);
 789
 790                                         buffer = sdhci_kmap_atomic(sg, &flags);
 791                                         memcpy(buffer, align, size);
 792                                         sdhci_kunmap_atomic(buffer, &flags);
 793
 794                                         align += SDHCI_ADMA2_ALIGN;
 795                                 }
 796                         }
 797                 }
 798         }
 799 }
 800
 801 static dma_addr_t sdhci_sdma_address(struct sdhci_host *host)
 802 {
 803         if (host->bounce_buffer)
 804                 return host->bounce_addr;
 805         else
 806                 return sg_dma_address(host->data->sg);
 807 }
 808
 809 static void sdhci_set_sdma_addr(struct sdhci_host *host, dma_addr_t addr)
 810 {
 811         if (host->v4_mode) {
 812                 sdhci_writel(host, addr, SDHCI_ADMA_ADDRESS);
 813                 if (host->flags & SDHCI_USE_64_BIT_DMA)
 814                         sdhci_writel(host, (u64)addr >> 32, SDHCI_ADMA_ADDRESS_HI);
 815         } else {
 816                 sdhci_writel(host, addr, SDHCI_DMA_ADDRESS);
 817         }
 818 }
 819
 820 static unsigned int sdhci_target_timeout(struct sdhci_host *host,
 821                                          struct mmc_command *cmd,
 822                                          struct mmc_data *data)
 823 {
 824         unsigned int target_timeout;
 825
 826         /* timeout in us */
 827         if (!data) {
 828                 target_timeout = cmd->busy_timeout * 1000;
 829         } else {
 830                 target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
 831                 if (host->clock && data->timeout_clks) {
 832                         unsigned long long val;
 833
 834                         /*
 835                          * data->timeout_clks is in units of clock cycles.
 836                          * host->clock is in Hz.  target_timeout is in us.
 837                          * Hence, us = 1000000 * cycles / Hz.  Round up.
 838                          */
 839                         val = 1000000ULL * data->timeout_clks;
 840                         if (do_div(val, host->clock))
 841                                 target_timeout++;
 842                         target_timeout += val;
 843                 }
 844         }
 845
 846         return target_timeout;
 847 }
 848
 849 static void sdhci_calc_sw_timeout(struct sdhci_host *host,
 850                                   struct mmc_command *cmd)
 851 {
 852         struct mmc_data *data = cmd->data;
 853         struct mmc_host *mmc = host->mmc;
 854         struct mmc_ios *ios = &mmc->ios;
 855         unsigned char bus_width = 1 << ios->bus_width;
 856         unsigned int blksz;
 857         unsigned int freq;
 858         u64 target_timeout;
 859         u64 transfer_time;
 860
 861         target_timeout = sdhci_target_timeout(host, cmd, data);
 862         target_timeout *= NSEC_PER_USEC;
 863
 864         if (data) {
 865                 blksz = data->blksz;
 866                 freq = host->mmc->actual_clock ? : host->clock;
 867                 transfer_time = (u64)blksz * NSEC_PER_SEC * (8 / bus_width);
 868                 do_div(transfer_time, freq);
 869                 /* multiply by '2' to account for any unknowns */
 870                 transfer_time = transfer_time * 2;
 871                 /* calculate timeout for the entire data */
 872                 host->data_timeout = data->blocks * target_timeout +
 873                                      transfer_time;
 874         } else {
 875                 host->data_timeout = target_timeout;
 876         }
 877
 878         if (host->data_timeout)
 879                 host->data_timeout += MMC_CMD_TRANSFER_TIME;
 880 }
 881
 882 static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd,
 883                              bool *too_big)
 884 {
 885         u8 count;
 886         struct mmc_data *data;
 887         unsigned target_timeout, current_timeout;
 888
 889         *too_big = true;
 890
 891         /*
 892          * If the host controller provides us with an incorrect timeout
 893          * value, just skip the check and use 0xE.  The hardware may take
 894          * longer to time out, but that's much better than having a too-short
 895          * timeout value.
 896          */
 897         if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
 898                 return 0xE;
 899
 900         /* Unspecified command, asume max */
 901         if (cmd == NULL)
 902                 return 0xE;
 903
 904         data = cmd->data;
 905         /* Unspecified timeout, assume max */
 906         if (!data && !cmd->busy_timeout)
 907                 return 0xE;
 908
 909         /* timeout in us */
 910         target_timeout = sdhci_target_timeout(host, cmd, data);
 911
 912         /*
 913          * Figure out needed cycles.
 914          * We do this in steps in order to fit inside a 32 bit int.
 915          * The first step is the minimum timeout, which will have a
 916          * minimum resolution of 6 bits:
 917          * (1) 2^13*1000 > 2^22,
 918          * (2) host->timeout_clk < 2^16
 919          *     =>
 920          *     (1) / (2) > 2^6
 921          */
 922         count = 0;
 923         current_timeout = (1 << 13) * 1000 / host->timeout_clk;
 924         while (current_timeout < target_timeout) {
 925                 count++;
 926                 current_timeout <<= 1;
 927                 if (count >= 0xF)
 928                         break;
 929         }
 930
 931         if (count >= 0xF) {
 932                 if (!(host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT))
 933                         DBG("Too large timeout 0x%x requested for CMD%d!\n",
 934                             count, cmd->opcode);
 935                 count = 0xE;
 936         } else {
 937                 *too_big = false;
 938         }
 939
 940         return count;
 941 }
 942
 943 static void sdhci_set_transfer_irqs(struct sdhci_host *host)
 944 {
 945         u32 pio_irqs = SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL;
 946         u32 dma_irqs = SDHCI_INT_DMA_END | SDHCI_INT_ADMA_ERROR;
 947
 948         if (host->flags & SDHCI_REQ_USE_DMA)
 949                 host->ier = (host->ier & ~pio_irqs) | dma_irqs;
 950         else
 951                 host->ier = (host->ier & ~dma_irqs) | pio_irqs;
 952
 953         if (host->flags & (SDHCI_AUTO_CMD23 | SDHCI_AUTO_CMD12))
 954                 host->ier |= SDHCI_INT_AUTO_CMD_ERR;
 955         else
 956                 host->ier &= ~SDHCI_INT_AUTO_CMD_ERR;
 957
 958         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 959         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 960 }
 961
 962 static void sdhci_set_data_timeout_irq(struct sdhci_host *host, bool enable)
 963 {
 964         if (enable)
 965                 host->ier |= SDHCI_INT_DATA_TIMEOUT;
 966         else
 967                 host->ier &= ~SDHCI_INT_DATA_TIMEOUT;
 968         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 969         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
 970 }
 971
 972 static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 973 {
 974         u8 count;
 975
 976         if (host->ops->set_timeout) {
 977                 host->ops->set_timeout(host, cmd);
 978         } else {
 979                 bool too_big = false;
 980
 981                 count = sdhci_calc_timeout(host, cmd, &too_big);
 982
 983                 if (too_big &&
 984                     host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT) {
 985                         sdhci_calc_sw_timeout(host, cmd);
 986                         sdhci_set_data_timeout_irq(host, false);
 987                 } else if (!(host->ier & SDHCI_INT_DATA_TIMEOUT)) {
 988                         sdhci_set_data_timeout_irq(host, true);
 989                 }
 990
 991                 sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
 992         }
 993 }
 994
 995 static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
 996 {
 997         struct mmc_data *data = cmd->data;
 998
 999         host->data_timeout = 0;
1000
1001         if (sdhci_data_line_cmd(cmd))
1002                 sdhci_set_timeout(host, cmd);
1003
1004         if (!data)
1005                 return;
1006
1007         WARN_ON(host->data);
1008
1009         /* Sanity checks */
1010         BUG_ON(data->blksz * data->blocks > 524288);
1011         BUG_ON(data->blksz > host->mmc->max_blk_size);
1012         BUG_ON(data->blocks > 65535);
1013
1014         host->data = data;
1015         host->data_early = 0;
1016         host->data->bytes_xfered = 0;
1017
1018         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
1019                 struct scatterlist *sg;
1020                 unsigned int length_mask, offset_mask;
1021                 int i;
1022
1023                 host->flags |= SDHCI_REQ_USE_DMA;
1024
1025                 /*
1026                  * FIXME: This doesn't account for merging when mapping the
1027                  * scatterlist.
1028                  *
1029                  * The assumption here being that alignment and lengths are
1030                  * the same after DMA mapping to device address space.
1031                  */
1032                 length_mask = 0;
1033                 offset_mask = 0;
1034                 if (host->flags & SDHCI_USE_ADMA) {
1035                         if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {
1036                                 length_mask = 3;
1037                                 /*
1038                                  * As we use up to 3 byte chunks to work
1039                                  * around alignment problems, we need to
1040                                  * check the offset as well.
1041                                  */
1042                                 offset_mask = 3;
1043                         }
1044                 } else {
1045                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)
1046                                 length_mask = 3;
1047                         if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)
1048                                 offset_mask = 3;
1049                 }
1050
1051                 if (unlikely(length_mask | offset_mask)) {
1052                         for_each_sg(data->sg, sg, data->sg_len, i) {
1053                                 if (sg->length & length_mask) {
1054                                         DBG("Reverting to PIO because of transfer size (%d)\n",
1055                                             sg->length);
1056                                         host->flags &= ~SDHCI_REQ_USE_DMA;
1057                                         break;
1058                                 }
1059                                 if (sg->offset & offset_mask) {
1060                                         DBG("Reverting to PIO because of bad alignment\n");
1061                                         host->flags &= ~SDHCI_REQ_USE_DMA;
1062                                         break;
1063                                 }
1064                         }
1065                 }
1066         }
1067
1068         if (host->flags & SDHCI_REQ_USE_DMA) {
1069                 int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
1070
1071                 if (sg_cnt <= 0) {
1072                         /*
1073                          * This only happens when someone fed
1074                          * us an invalid request.
1075                          */
1076                         WARN_ON(1);
1077                         host->flags &= ~SDHCI_REQ_USE_DMA;
1078                 } else if (host->flags & SDHCI_USE_ADMA) {
1079                         sdhci_adma_table_pre(host, data, sg_cnt);
1080
1081                         sdhci_writel(host, host->adma_addr, SDHCI_ADMA_ADDRESS);
1082                         if (host->flags & SDHCI_USE_64_BIT_DMA)
1083                                 sdhci_writel(host,
1084                                              (u64)host->adma_addr >> 32,
1085                                              SDHCI_ADMA_ADDRESS_HI);
1086                 } else {
1087                         WARN_ON(sg_cnt != 1);
1088                         sdhci_set_sdma_addr(host, sdhci_sdma_address(host));
1089                 }
1090         }
1091
1092         sdhci_config_dma(host);
1093
1094         if (!(host->flags & SDHCI_REQ_USE_DMA)) {
1095                 int flags;
1096
1097                 flags = SG_MITER_ATOMIC;
1098                 if (host->data->flags & MMC_DATA_READ)
1099                         flags |= SG_MITER_TO_SG;
1100                 else
1101                         flags |= SG_MITER_FROM_SG;
1102                 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
1103                 host->blocks = data->blocks;
1104         }
1105
1106         sdhci_set_transfer_irqs(host);
1107
1108         /* Set the DMA boundary value and block size */
1109         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
1110                      SDHCI_BLOCK_SIZE);
1111
1112         /*
1113          * For Version 4.10 onwards, if v4 mode is enabled, 32-bit Block Count
1114          * can be supported, in that case 16-bit block count register must be 0.
1115          */
1116         if (host->version >= SDHCI_SPEC_410 && host->v4_mode &&
1117             (host->quirks2 & SDHCI_QUIRK2_USE_32BIT_BLK_CNT)) {
1118                 if (sdhci_readw(host, SDHCI_BLOCK_COUNT))
1119                         sdhci_writew(host, 0, SDHCI_BLOCK_COUNT);
1120                 sdhci_writew(host, data->blocks, SDHCI_32BIT_BLK_CNT);
1121         } else {
1122                 sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
1123         }
1124 }
1125
1126 static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
1127                                     struct mmc_request *mrq)
1128 {
1129         return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
1130                !mrq->cap_cmd_during_tfr;
1131 }
1132
1133 static inline void sdhci_auto_cmd_select(struct sdhci_host *host,
1134                                          struct mmc_command *cmd,
1135                                          u16 *mode)
1136 {
1137         bool use_cmd12 = sdhci_auto_cmd12(host, cmd->mrq) &&
1138                          (cmd->opcode != SD_IO_RW_EXTENDED);
1139         bool use_cmd23 = cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23);
1140         u16 ctrl2;
1141
1142         /*
1143          * In case of Version 4.10 or later, use of 'Auto CMD Auto
1144          * Select' is recommended rather than use of 'Auto CMD12
1145          * Enable' or 'Auto CMD23 Enable'.
1146          */
1147         if (host->version >= SDHCI_SPEC_410 && (use_cmd12 || use_cmd23)) {
1148                 *mode |= SDHCI_TRNS_AUTO_SEL;
1149
1150                 ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1151                 if (use_cmd23)
1152                         ctrl2 |= SDHCI_CMD23_ENABLE;
1153                 else
1154                         ctrl2 &= ~SDHCI_CMD23_ENABLE;
1155                 sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2);
1156
1157                 return;
1158         }
1159
1160         /*
1161          * If we are sending CMD23, CMD12 never gets sent
1162          * on successful completion (so no Auto-CMD12).
1163          */
1164         if (use_cmd12)
1165                 *mode |= SDHCI_TRNS_AUTO_CMD12;
1166         else if (use_cmd23)
1167                 *mode |= SDHCI_TRNS_AUTO_CMD23;
1168 }
1169
1170 static void sdhci_set_transfer_mode(struct sdhci_host *host,
1171         struct mmc_command *cmd)
1172 {
1173         u16 mode = 0;
1174         struct mmc_data *data = cmd->data;
1175
1176         if (data == NULL) {
1177                 if (host->quirks2 &
1178                         SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) {
1179                         /* must not clear SDHCI_TRANSFER_MODE when tuning */
1180                         if (cmd->opcode != MMC_SEND_TUNING_BLOCK_HS200)
1181                                 sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
1182                 } else {
1183                 /* clear Auto CMD settings for no data CMDs */
1184                         mode = sdhci_readw(host, SDHCI_TRANSFER_MODE);
1185                         sdhci_writew(host, mode & ~(SDHCI_TRNS_AUTO_CMD12 |
1186                                 SDHCI_TRNS_AUTO_CMD23), SDHCI_TRANSFER_MODE);
1187                 }
1188                 return;
1189         }
1190
1191         WARN_ON(!host->data);
1192
1193         if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))
1194                 mode = SDHCI_TRNS_BLK_CNT_EN;
1195
1196         if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {
1197                 mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI;
1198                 sdhci_auto_cmd_select(host, cmd, &mode);
1199                 if (cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23))
1200                         sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);
1201         }
1202
1203         if (data->flags & MMC_DATA_READ)
1204                 mode |= SDHCI_TRNS_READ;
1205         if (host->flags & SDHCI_REQ_USE_DMA)
1206                 mode |= SDHCI_TRNS_DMA;
1207
1208         sdhci_writew(host, mode, SDHCI_TRANSFER_MODE);
1209 }
1210
1211 static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq)
1212 {
1213         return (!(host->flags & SDHCI_DEVICE_DEAD) &&
1214                 ((mrq->cmd && mrq->cmd->error) ||
1215                  (mrq->sbc && mrq->sbc->error) ||
1216                  (mrq->data && mrq->data->stop && mrq->data->stop->error) ||
1217                  (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));
1218 }
1219
1220 static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1221 {
1222         int i;
1223
1224         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1225                 if (host->mrqs_done[i] == mrq) {
1226                         WARN_ON(1);
1227                         return;
1228                 }
1229         }
1230
1231         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1232                 if (!host->mrqs_done[i]) {
1233                         host->mrqs_done[i] = mrq;
1234                         break;
1235                 }
1236         }
1237
1238         WARN_ON(i >= SDHCI_MAX_MRQS);
1239
1240         tasklet_schedule(&host->finish_tasklet);
1241 }
1242
1243 static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1244 {
1245         if (host->cmd && host->cmd->mrq == mrq)
1246                 host->cmd = NULL;
1247
1248         if (host->data_cmd && host->data_cmd->mrq == mrq)
1249                 host->data_cmd = NULL;
1250
1251         if (host->data && host->data->mrq == mrq)
1252                 host->data = NULL;
1253
1254         if (sdhci_needs_reset(host, mrq))
1255                 host->pending_reset = true;
1256
1257         __sdhci_finish_mrq(host, mrq);
1258 }
1259
1260 static void sdhci_finish_data(struct sdhci_host *host)
1261 {
1262         struct mmc_command *data_cmd = host->data_cmd;
1263         struct mmc_data *data = host->data;
1264
1265         host->data = NULL;
1266         host->data_cmd = NULL;
1267
1268         /*
1269          * The controller needs a reset of internal state machines upon error
1270          * conditions.
1271          */
1272         if (data->error) {
1273                 if (!host->cmd || host->cmd == data_cmd)
1274                         sdhci_do_reset(host, SDHCI_RESET_CMD);
1275                 sdhci_do_reset(host, SDHCI_RESET_DATA);
1276         }
1277
1278         if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
1279             (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA))
1280                 sdhci_adma_table_post(host, data);
1281
1282         /*
1283          * The specification states that the block count register must
1284          * be updated, but it does not specify at what point in the
1285          * data flow. That makes the register entirely useless to read
1286          * back so we have to assume that nothing made it to the card
1287          * in the event of an error.
1288          */
1289         if (data->error)
1290                 data->bytes_xfered = 0;
1291         else
1292                 data->bytes_xfered = data->blksz * data->blocks;
1293
1294         /*
1295          * Need to send CMD12 if -
1296          * a) open-ended multiblock transfer (no CMD23)
1297          * b) error in multiblock transfer
1298          */
1299         if (data->stop &&
1300             (data->error ||
1301              !data->mrq->sbc)) {
1302                 /*
1303                  * 'cap_cmd_during_tfr' request must not use the command line
1304                  * after mmc_command_done() has been called. It is upper layer's
1305                  * responsibility to send the stop command if required.
1306                  */
1307                 if (data->mrq->cap_cmd_during_tfr) {
1308                         sdhci_finish_mrq(host, data->mrq);
1309                 } else {
1310                         /* Avoid triggering warning in sdhci_send_command() */
1311                         host->cmd = NULL;
1312                         sdhci_send_command(host, data->stop);
1313                 }
1314         } else {
1315                 sdhci_finish_mrq(host, data->mrq);
1316         }
1317 }
1318
1319 static void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq,
1320                             unsigned long timeout)
1321 {
1322         if (sdhci_data_line_cmd(mrq->cmd))
1323                 mod_timer(&host->data_timer, timeout);
1324         else
1325                 mod_timer(&host->timer, timeout);
1326 }
1327
1328 static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq)
1329 {
1330         if (sdhci_data_line_cmd(mrq->cmd))
1331                 del_timer(&host->data_timer);
1332         else
1333                 del_timer(&host->timer);
1334 }
1335
1336 void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
1337 {
1338         int flags;
1339         u32 mask;
1340         unsigned long timeout;
1341
1342         WARN_ON(host->cmd);
1343
1344         /* Initially, a command has no error */
1345         cmd->error = 0;
1346
1347         if ((host->quirks2 & SDHCI_QUIRK2_STOP_WITH_TC) &&
1348             cmd->opcode == MMC_STOP_TRANSMISSION)
1349                 cmd->flags |= MMC_RSP_BUSY;
1350
1351         /* Wait max 10 ms */
1352         timeout = 10;
1353
1354         mask = SDHCI_CMD_INHIBIT;
1355         if (sdhci_data_line_cmd(cmd))
1356                 mask |= SDHCI_DATA_INHIBIT;
1357
1358         /* We shouldn't wait for data inihibit for stop commands, even
1359            though they might use busy signaling */
1360         if (cmd->mrq->data && (cmd == cmd->mrq->data->stop))
1361                 mask &= ~SDHCI_DATA_INHIBIT;
1362
1363         while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
1364                 if (timeout == 0) {
1365                         pr_err("%s: Controller never released inhibit bit(s).\n",
1366                                mmc_hostname(host->mmc));
1367                         sdhci_dumpregs(host);
1368                         cmd->error = -EIO;
1369                         sdhci_finish_mrq(host, cmd->mrq);
1370                         return;
1371                 }
1372                 timeout--;
1373                 mdelay(1);
1374         }
1375
1376         host->cmd = cmd;
1377         if (sdhci_data_line_cmd(cmd)) {
1378                 WARN_ON(host->data_cmd);
1379                 host->data_cmd = cmd;
1380         }
1381
1382         sdhci_prepare_data(host, cmd);
1383
1384         sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT);
1385
1386         sdhci_set_transfer_mode(host, cmd);
1387
1388         if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1389                 pr_err("%s: Unsupported response type!\n",
1390                         mmc_hostname(host->mmc));
1391                 cmd->error = -EINVAL;
1392                 sdhci_finish_mrq(host, cmd->mrq);
1393                 return;
1394         }
1395
1396         if (!(cmd->flags & MMC_RSP_PRESENT))
1397                 flags = SDHCI_CMD_RESP_NONE;
1398         else if (cmd->flags & MMC_RSP_136)
1399                 flags = SDHCI_CMD_RESP_LONG;
1400         else if (cmd->flags & MMC_RSP_BUSY)
1401                 flags = SDHCI_CMD_RESP_SHORT_BUSY;
1402         else
1403                 flags = SDHCI_CMD_RESP_SHORT;
1404
1405         if (cmd->flags & MMC_RSP_CRC)
1406                 flags |= SDHCI_CMD_CRC;
1407         if (cmd->flags & MMC_RSP_OPCODE)
1408                 flags |= SDHCI_CMD_INDEX;
1409
1410         /* CMD19 is special in that the Data Present Select should be set */
1411         if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK ||
1412             cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)
1413                 flags |= SDHCI_CMD_DATA;
1414
1415         timeout = jiffies;
1416         if (host->data_timeout)
1417                 timeout += nsecs_to_jiffies(host->data_timeout);
1418         else if (!cmd->data && cmd->busy_timeout > 9000)
1419                 timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
1420         else
1421                 timeout += 10 * HZ;
1422         sdhci_mod_timer(host, cmd->mrq, timeout);
1423
1424         sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);
1425 }
1426 EXPORT_SYMBOL_GPL(sdhci_send_command);
1427
1428 static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)
1429 {
1430         int i, reg;
1431
1432         for (i = 0; i < 4; i++) {
1433                 reg = SDHCI_RESPONSE + (3 - i) * 4;
1434                 cmd->resp[i] = sdhci_readl(host, reg);
1435         }
1436
1437         if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)
1438                 return;
1439
1440         /* CRC is stripped so we need to do some shifting */
1441         for (i = 0; i < 4; i++) {
1442                 cmd->resp[i] <<= 8;
1443                 if (i != 3)
1444                         cmd->resp[i] |= cmd->resp[i + 1] >> 24;
1445         }
1446 }
1447
1448 static void sdhci_finish_command(struct sdhci_host *host)
1449 {
1450         struct mmc_command *cmd = host->cmd;
1451
1452         host->cmd = NULL;
1453
1454         if (cmd->flags & MMC_RSP_PRESENT) {
1455                 if (cmd->flags & MMC_RSP_136) {
1456                         sdhci_read_rsp_136(host, cmd);
1457                 } else {
1458                         cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);
1459                 }
1460         }
1461
1462         if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)
1463                 mmc_command_done(host->mmc, cmd->mrq);
1464
1465         /*
1466          * The host can send and interrupt when the busy state has
1467          * ended, allowing us to wait without wasting CPU cycles.
1468          * The busy signal uses DAT0 so this is similar to waiting
1469          * for data to complete.
1470          *
1471          * Note: The 1.0 specification is a bit ambiguous about this
1472          *       feature so there might be some problems with older
1473          *       controllers.
1474          */
1475         if (cmd->flags & MMC_RSP_BUSY) {
1476                 if (cmd->data) {
1477                         DBG("Cannot wait for busy signal when also doing a data transfer");
1478                 } else if (!(host->quirks & SDHCI_QUIRK_NO_BUSY_IRQ) &&
1479                            cmd == host->data_cmd) {
1480                         /* Command complete before busy is ended */
1481                         return;
1482                 }
1483         }
1484
1485         /* Finished CMD23, now send actual command. */
1486         if (cmd == cmd->mrq->sbc) {
1487                 sdhci_send_command(host, cmd->mrq->cmd);
1488         } else {
1489
1490                 /* Processed actual command. */
1491                 if (host->data && host->data_early)
1492                         sdhci_finish_data(host);
1493
1494                 if (!cmd->data)
1495                         sdhci_finish_mrq(host, cmd->mrq);
1496         }
1497 }
1498
1499 static u16 sdhci_get_preset_value(struct sdhci_host *host)
1500 {
1501         u16 preset = 0;
1502
1503         switch (host->timing) {
1504         case MMC_TIMING_UHS_SDR12:
1505                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1506                 break;
1507         case MMC_TIMING_UHS_SDR25:
1508                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR25);
1509                 break;
1510         case MMC_TIMING_UHS_SDR50:
1511                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR50);
1512                 break;
1513         case MMC_TIMING_UHS_SDR104:
1514         case MMC_TIMING_MMC_HS200:
1515                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR104);
1516                 break;
1517         case MMC_TIMING_UHS_DDR50:
1518         case MMC_TIMING_MMC_DDR52:
1519                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_DDR50);
1520                 break;
1521         case MMC_TIMING_MMC_HS400:
1522                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_HS400);
1523                 break;
1524         default:
1525                 pr_warn("%s: Invalid UHS-I mode selected\n",
1526                         mmc_hostname(host->mmc));
1527                 preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1528                 break;
1529         }
1530         return preset;
1531 }
1532
1533 u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock,
1534                    unsigned int *actual_clock)
1535 {
1536         int div = 0; /* Initialized for compiler warning */
1537         int real_div = div, clk_mul = 1;
1538         u16 clk = 0;
1539         bool switch_base_clk = false;
1540
1541         if (host->version >= SDHCI_SPEC_300) {
1542                 if (host->preset_enabled) {
1543                         u16 pre_val;
1544
1545                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1546                         pre_val = sdhci_get_preset_value(host);
1547                         div = (pre_val & SDHCI_PRESET_SDCLK_FREQ_MASK)
1548                                 >> SDHCI_PRESET_SDCLK_FREQ_SHIFT;
1549                         if (host->clk_mul &&
1550                                 (pre_val & SDHCI_PRESET_CLKGEN_SEL_MASK)) {
1551                                 clk = SDHCI_PROG_CLOCK_MODE;
1552                                 real_div = div + 1;
1553                                 clk_mul = host->clk_mul;
1554                         } else {
1555                                 real_div = max_t(int, 1, div << 1);
1556                         }
1557                         goto clock_set;
1558                 }
1559
1560                 /*
1561                  * Check if the Host Controller supports Programmable Clock
1562                  * Mode.
1563                  */
1564                 if (host->clk_mul) {
1565                         for (div = 1; div <= 1024; div++) {
1566                                 if ((host->max_clk * host->clk_mul / div)
1567                                         <= clock)
1568                                         break;
1569                         }
1570                         if ((host->max_clk * host->clk_mul / div) <= clock) {
1571                                 /*
1572                                  * Set Programmable Clock Mode in the Clock
1573                                  * Control register.
1574                                  */
1575                                 clk = SDHCI_PROG_CLOCK_MODE;
1576                                 real_div = div;
1577                                 clk_mul = host->clk_mul;
1578                                 div--;
1579                         } else {
1580                                 /*
1581                                  * Divisor can be too small to reach clock
1582                                  * speed requirement. Then use the base clock.
1583                                  */
1584                                 switch_base_clk = true;
1585                         }
1586                 }
1587
1588                 if (!host->clk_mul || switch_base_clk) {
1589                         /* Version 3.00 divisors must be a multiple of 2. */
1590                         if (host->max_clk <= clock)
1591                                 div = 1;
1592                         else {
1593                                 for (div = 2; div < SDHCI_MAX_DIV_SPEC_300;
1594                                      div += 2) {
1595                                         if ((host->max_clk / div) <= clock)
1596                                                 break;
1597                                 }
1598                         }
1599                         real_div = div;
1600                         div >>= 1;
1601                         if ((host->quirks2 & SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN)
1602                                 && !div && host->max_clk <= 25000000)
1603                                 div = 1;
1604                 }
1605         } else {
1606                 /* Version 2.00 divisors must be a power of 2. */
1607                 for (div = 1; div < SDHCI_MAX_DIV_SPEC_200; div *= 2) {
1608                         if ((host->max_clk / div) <= clock)
1609                                 break;
1610                 }
1611                 real_div = div;
1612                 div >>= 1;
1613         }
1614
1615 clock_set:
1616         if (real_div)
1617                 *actual_clock = (host->max_clk * clk_mul) / real_div;
1618         clk |= (div & SDHCI_DIV_MASK) << SDHCI_DIVIDER_SHIFT;
1619         clk |= ((div & SDHCI_DIV_HI_MASK) >> SDHCI_DIV_MASK_LEN)
1620                 << SDHCI_DIVIDER_HI_SHIFT;
1621
1622         return clk;
1623 }
1624 EXPORT_SYMBOL_GPL(sdhci_calc_clk);
1625
1626 void sdhci_enable_clk(struct sdhci_host *host, u16 clk)
1627 {
1628         ktime_t timeout;
1629
1630         clk |= SDHCI_CLOCK_INT_EN;
1631         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1632
1633         /* Wait max 20 ms */
1634         timeout = ktime_add_ms(ktime_get(), 20);
1635         while (1) {
1636                 bool timedout = ktime_after(ktime_get(), timeout);
1637
1638                 clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1639                 if (clk & SDHCI_CLOCK_INT_STABLE)
1640                         break;
1641                 if (timedout) {
1642                         pr_err("%s: Internal clock never stabilised.\n",
1643                                mmc_hostname(host->mmc));
1644                         sdhci_dumpregs(host);
1645                         return;
1646                 }
1647                 udelay(10);
1648         }
1649
1650         clk |= SDHCI_CLOCK_CARD_EN;
1651         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1652 }
1653 EXPORT_SYMBOL_GPL(sdhci_enable_clk);
1654
1655 void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
1656 {
1657         u16 clk;
1658
1659         host->mmc->actual_clock = 0;
1660
1661         sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
1662
1663         if (clock == 0)
1664                 return;
1665
1666         clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
1667         sdhci_enable_clk(host, clk);
1668 }
1669 EXPORT_SYMBOL_GPL(sdhci_set_clock);
1670
1671 static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode,
1672                                 unsigned short vdd)
1673 {
1674         struct mmc_host *mmc = host->mmc;
1675
1676         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
1677
1678         if (mode != MMC_POWER_OFF)
1679                 sdhci_writeb(host, SDHCI_POWER_ON, SDHCI_POWER_CONTROL);
1680         else
1681                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1682 }
1683
1684 void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
1685                            unsigned short vdd)
1686 {
1687         u8 pwr = 0;
1688
1689         if (mode != MMC_POWER_OFF) {
1690                 switch (1 << vdd) {
1691                 case MMC_VDD_165_195:
1692                 /*
1693                  * Without a regulator, SDHCI does not support 2.0v
1694                  * so we only get here if the driver deliberately
1695                  * added the 2.0v range to ocr_avail. Map it to 1.8v
1696                  * for the purpose of turning on the power.
1697                  */
1698                 case MMC_VDD_20_21:
1699                         pwr = SDHCI_POWER_180;
1700                         break;
1701                 case MMC_VDD_29_30:
1702                 case MMC_VDD_30_31:
1703                         pwr = SDHCI_POWER_300;
1704                         break;
1705                 case MMC_VDD_32_33:
1706                 case MMC_VDD_33_34:
1707                         pwr = SDHCI_POWER_330;
1708                         break;
1709                 default:
1710                         WARN(1, "%s: Invalid vdd %#x\n",
1711                              mmc_hostname(host->mmc), vdd);
1712                         break;
1713                 }
1714         }
1715
1716         if (host->pwr == pwr)
1717                 return;
1718
1719         host->pwr = pwr;
1720
1721         if (pwr == 0) {
1722                 sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1723                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1724                         sdhci_runtime_pm_bus_off(host);
1725         } else {
1726                 /*
1727                  * Spec says that we should clear the power reg before setting
1728                  * a new value. Some controllers don't seem to like this though.
1729                  */
1730                 if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE))
1731                         sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1732
1733                 /*
1734                  * At least the Marvell CaFe chip gets confused if we set the
1735                  * voltage and set turn on power at the same time, so set the
1736                  * voltage first.
1737                  */
1738                 if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER)
1739                         sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1740
1741                 pwr |= SDHCI_POWER_ON;
1742
1743                 sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1744
1745                 if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1746                         sdhci_runtime_pm_bus_on(host);
1747
1748                 /*
1749                  * Some controllers need an extra 10ms delay of 10ms before
1750                  * they can apply clock after applying power
1751                  */
1752                 if (host->quirks & SDHCI_QUIRK_DELAY_AFTER_POWER)
1753                         mdelay(10);
1754         }
1755 }
1756 EXPORT_SYMBOL_GPL(sdhci_set_power_noreg);
1757
1758 void sdhci_set_power(struct sdhci_host *host, unsigned char mode,
1759                      unsigned short vdd)
1760 {
1761         if (IS_ERR(host->mmc->supply.vmmc))
1762                 sdhci_set_power_noreg(host, mode, vdd);
1763         else
1764                 sdhci_set_power_reg(host, mode, vdd);
1765 }
1766 EXPORT_SYMBOL_GPL(sdhci_set_power);
1767
1768 /*****************************************************************************\
1769  *                                                                           *
1770  * MMC callbacks                                                             *
1771  *                                                                           *
1772 \*****************************************************************************/
1773
1774 void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1775 {
1776         struct sdhci_host *host;
1777         int present;
1778         unsigned long flags;
1779
1780         host = mmc_priv(mmc);
1781
1782         /* Firstly check card presence */
1783         present = mmc->ops->get_cd(mmc);
1784
1785         spin_lock_irqsave(&host->lock, flags);
1786
1787         sdhci_led_activate(host);
1788
1789         /*
1790          * Ensure we don't send the STOP for non-SET_BLOCK_COUNTED
1791          * requests if Auto-CMD12 is enabled.
1792          */
1793         if (sdhci_auto_cmd12(host, mrq)) {
1794                 if (mrq->stop) {
1795                         mrq->data->stop = NULL;
1796                         mrq->stop = NULL;
1797                 }
1798         }
1799
1800         if (!present || host->flags & SDHCI_DEVICE_DEAD) {
1801                 mrq->cmd->error = -ENOMEDIUM;
1802                 sdhci_finish_mrq(host, mrq);
1803         } else {
1804                 if (mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23))
1805                         sdhci_send_command(host, mrq->sbc);
1806                 else
1807                         sdhci_send_command(host, mrq->cmd);
1808         }
1809
1810         spin_unlock_irqrestore(&host->lock, flags);
1811 }
1812 EXPORT_SYMBOL_GPL(sdhci_request);
1813
1814 void sdhci_set_bus_width(struct sdhci_host *host, int width)
1815 {
1816         u8 ctrl;
1817
1818         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1819         if (width == MMC_BUS_WIDTH_8) {
1820                 ctrl &= ~SDHCI_CTRL_4BITBUS;
1821                 ctrl |= SDHCI_CTRL_8BITBUS;
1822         } else {
1823                 if (host->mmc->caps & MMC_CAP_8_BIT_DATA)
1824                         ctrl &= ~SDHCI_CTRL_8BITBUS;
1825                 if (width == MMC_BUS_WIDTH_4)
1826                         ctrl |= SDHCI_CTRL_4BITBUS;
1827                 else
1828                         ctrl &= ~SDHCI_CTRL_4BITBUS;
1829         }
1830         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1831 }
1832 EXPORT_SYMBOL_GPL(sdhci_set_bus_width);
1833
1834 void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
1835 {
1836         u16 ctrl_2;
1837
1838         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1839         /* Select Bus Speed Mode for host */
1840         ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
1841         if ((timing == MMC_TIMING_MMC_HS200) ||
1842             (timing == MMC_TIMING_UHS_SDR104))
1843                 ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
1844         else if (timing == MMC_TIMING_UHS_SDR12)
1845                 ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
1846         else if (timing == MMC_TIMING_UHS_SDR25)
1847                 ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
1848         else if (timing == MMC_TIMING_UHS_SDR50)
1849                 ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
1850         else if ((timing == MMC_TIMING_UHS_DDR50) ||
1851                  (timing == MMC_TIMING_MMC_DDR52))
1852                 ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
1853         else if (timing == MMC_TIMING_MMC_HS400)
1854                 ctrl_2 |= SDHCI_CTRL_HS400; /* Non-standard */
1855         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1856 }
1857 EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling);
1858
1859 void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1860 {
1861         struct sdhci_host *host = mmc_priv(mmc);
1862         u8 ctrl;
1863
1864         if (ios->power_mode == MMC_POWER_UNDEFINED)
1865                 return;
1866
1867         if (host->flags & SDHCI_DEVICE_DEAD) {
1868                 if (!IS_ERR(mmc->supply.vmmc) &&
1869                     ios->power_mode == MMC_POWER_OFF)
1870                         mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
1871                 return;
1872         }
1873
1874         /*
1875          * Reset the chip on each power off.
1876          * Should clear out any weird states.
1877          */
1878         if (ios->power_mode == MMC_POWER_OFF) {
1879                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
1880                 sdhci_reinit(host);
1881         }
1882
1883         if (host->version >= SDHCI_SPEC_300 &&
1884                 (ios->power_mode == MMC_POWER_UP) &&
1885                 !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN))
1886                 sdhci_enable_preset_value(host, false);
1887
1888         if (!ios->clock || ios->clock != host->clock) {
1889                 host->ops->set_clock(host, ios->clock);
1890                 host->clock = ios->clock;
1891
1892                 if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK &&
1893                     host->clock) {
1894                         host->timeout_clk = host->mmc->actual_clock ?
1895                                                 host->mmc->actual_clock / 1000 :
1896                                                 host->clock / 1000;
1897                         host->mmc->max_busy_timeout =
1898                                 host->ops->get_max_timeout_count ?
1899                                 host->ops->get_max_timeout_count(host) :
1900                                 1 << 27;
1901                         host->mmc->max_busy_timeout /= host->timeout_clk;
1902                 }
1903         }
1904
1905         if (host->ops->set_power)
1906                 host->ops->set_power(host, ios->power_mode, ios->vdd);
1907         else
1908                 sdhci_set_power(host, ios->power_mode, ios->vdd);
1909
1910         if (host->ops->platform_send_init_74_clocks)
1911                 host->ops->platform_send_init_74_clocks(host, ios->power_mode);
1912
1913         host->ops->set_bus_width(host, ios->bus_width);
1914
1915         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1916
1917         if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {
1918                 if (ios->timing == MMC_TIMING_SD_HS ||
1919                      ios->timing == MMC_TIMING_MMC_HS ||
1920                      ios->timing == MMC_TIMING_MMC_HS400 ||
1921                      ios->timing == MMC_TIMING_MMC_HS200 ||
1922                      ios->timing == MMC_TIMING_MMC_DDR52 ||
1923                      ios->timing == MMC_TIMING_UHS_SDR50 ||
1924                      ios->timing == MMC_TIMING_UHS_SDR104 ||
1925                      ios->timing == MMC_TIMING_UHS_DDR50 ||
1926                      ios->timing == MMC_TIMING_UHS_SDR25)
1927                         ctrl |= SDHCI_CTRL_HISPD;
1928                 else
1929                         ctrl &= ~SDHCI_CTRL_HISPD;
1930         }
1931
1932         if (host->version >= SDHCI_SPEC_300) {
1933                 u16 clk, ctrl_2;
1934
1935                 if (!host->preset_enabled) {
1936                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1937                         /*
1938                          * We only need to set Driver Strength if the
1939                          * preset value enable is not set.
1940                          */
1941                         ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1942                         ctrl_2 &= ~SDHCI_CTRL_DRV_TYPE_MASK;
1943                         if (ios->drv_type == MMC_SET_DRIVER_TYPE_A)
1944                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_A;
1945                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_B)
1946                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1947                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_C)
1948                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_C;
1949                         else if (ios->drv_type == MMC_SET_DRIVER_TYPE_D)
1950                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_D;
1951                         else {
1952                                 pr_warn("%s: invalid driver type, default to driver type B\n",
1953                                         mmc_hostname(mmc));
1954                                 ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1955                         }
1956
1957                         sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1958                 } else {
1959                         /*
1960                          * According to SDHC Spec v3.00, if the Preset Value
1961                          * Enable in the Host Control 2 register is set, we
1962                          * need to reset SD Clock Enable before changing High
1963                          * Speed Enable to avoid generating clock gliches.
1964                          */
1965
1966                         /* Reset SD Clock Enable */
1967                         clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1968                         clk &= ~SDHCI_CLOCK_CARD_EN;
1969                         sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1970
1971                         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1972
1973                         /* Re-enable SD Clock */
1974                         host->ops->set_clock(host, host->clock);
1975                 }
1976
1977                 /* Reset SD Clock Enable */
1978                 clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1979                 clk &= ~SDHCI_CLOCK_CARD_EN;
1980                 sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1981
1982                 host->ops->set_uhs_signaling(host, ios->timing);
1983                 host->timing = ios->timing;
1984
1985                 if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) &&
1986                                 ((ios->timing == MMC_TIMING_UHS_SDR12) ||
1987                                  (ios->timing == MMC_TIMING_UHS_SDR25) ||
1988                                  (ios->timing == MMC_TIMING_UHS_SDR50) ||
1989                                  (ios->timing == MMC_TIMING_UHS_SDR104) ||
1990                                  (ios->timing == MMC_TIMING_UHS_DDR50) ||
1991                                  (ios->timing == MMC_TIMING_MMC_DDR52))) {
1992                         u16 preset;
1993
1994                         sdhci_enable_preset_value(host, true);
1995                         preset = sdhci_get_preset_value(host);
1996                         ios->drv_type = (preset & SDHCI_PRESET_DRV_MASK)
1997                                 >> SDHCI_PRESET_DRV_SHIFT;
1998                 }
1999
2000                 /* Re-enable SD Clock */
2001                 host->ops->set_clock(host, host->clock);
2002         } else
2003                 sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
2004
2005         /*
2006          * Some (ENE) controllers go apeshit on some ios operation,
2007          * signalling timeout and CRC errors even on CMD0. Resetting
2008          * it on each ios seems to solve the problem.
2009          */
2010         if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)
2011                 sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
2012 }
2013 EXPORT_SYMBOL_GPL(sdhci_set_ios);
2014
2015 static int sdhci_get_cd(struct mmc_host *mmc)
2016 {
2017         struct sdhci_host *host = mmc_priv(mmc);
2018         int gpio_cd = mmc_gpio_get_cd(mmc);
2019
2020         if (host->flags & SDHCI_DEVICE_DEAD)
2021                 return 0;
2022
2023         /* If nonremovable, assume that the card is always present. */
2024         if (!mmc_card_is_removable(host->mmc))
2025                 return 1;
2026
2027         /*
2028          * Try slot gpio detect, if defined it take precedence
2029          * over build in controller functionality
2030          */
2031         if (gpio_cd >= 0)
2032                 return !!gpio_cd;
2033
2034         /* If polling, assume that the card is always present. */
2035         if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)
2036                 return 1;
2037
2038         /* Host native card detect */
2039         return !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT);
2040 }
2041
2042 static int sdhci_check_ro(struct sdhci_host *host)
2043 {
2044         unsigned long flags;
2045         int is_readonly;
2046
2047         spin_lock_irqsave(&host->lock, flags);
2048
2049         if (host->flags & SDHCI_DEVICE_DEAD)
2050                 is_readonly = 0;
2051         else if (host->ops->get_ro)
2052                 is_readonly = host->ops->get_ro(host);
2053         else if (mmc_can_gpio_ro(host->mmc))
2054                 is_readonly = mmc_gpio_get_ro(host->mmc);
2055         else
2056                 is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE)
2057                                 & SDHCI_WRITE_PROTECT);
2058
2059         spin_unlock_irqrestore(&host->lock, flags);
2060
2061         /* This quirk needs to be replaced by a callback-function later */
2062         return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ?
2063                 !is_readonly : is_readonly;
2064 }
2065
2066 #define SAMPLE_COUNT    5
2067
2068 static int sdhci_get_ro(struct mmc_host *mmc)
2069 {
2070         struct sdhci_host *host = mmc_priv(mmc);
2071         int i, ro_count;
2072
2073         if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT))
2074                 return sdhci_check_ro(host);
2075
2076         ro_count = 0;
2077         for (i = 0; i < SAMPLE_COUNT; i++) {
2078                 if (sdhci_check_ro(host)) {
2079                         if (++ro_count > SAMPLE_COUNT / 2)
2080                                 return 1;
2081                 }
2082                 msleep(30);
2083         }
2084         return 0;
2085 }
2086
2087 static void sdhci_hw_reset(struct mmc_host *mmc)
2088 {
2089         struct sdhci_host *host = mmc_priv(mmc);
2090
2091         if (host->ops && host->ops->hw_reset)
2092                 host->ops->hw_reset(host);
2093 }
2094
2095 static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable)
2096 {
2097         if (!(host->flags & SDHCI_DEVICE_DEAD)) {
2098                 if (enable)
2099                         host->ier |= SDHCI_INT_CARD_INT;
2100                 else
2101                         host->ier &= ~SDHCI_INT_CARD_INT;
2102
2103                 sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2104                 sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2105         }
2106 }
2107
2108 void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
2109 {
2110         struct sdhci_host *host = mmc_priv(mmc);
2111         unsigned long flags;
2112
2113         if (enable)
2114                 pm_runtime_get_noresume(host->mmc->parent);
2115
2116         spin_lock_irqsave(&host->lock, flags);
2117         if (enable)
2118                 host->flags |= SDHCI_SDIO_IRQ_ENABLED;
2119         else
2120                 host->flags &= ~SDHCI_SDIO_IRQ_ENABLED;
2121
2122         sdhci_enable_sdio_irq_nolock(host, enable);
2123         spin_unlock_irqrestore(&host->lock, flags);
2124
2125         if (!enable)
2126                 pm_runtime_put_noidle(host->mmc->parent);
2127 }
2128 EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq);
2129
2130 int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
2131                                       struct mmc_ios *ios)
2132 {
2133         struct sdhci_host *host = mmc_priv(mmc);
2134         u16 ctrl;
2135         int ret;
2136
2137         /*
2138          * Signal Voltage Switching is only applicable for Host Controllers
2139          * v3.00 and above.
2140          */
2141         if (host->version < SDHCI_SPEC_300)
2142                 return 0;
2143
2144         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2145
2146         switch (ios->signal_voltage) {
2147         case MMC_SIGNAL_VOLTAGE_330:
2148                 if (!(host->flags & SDHCI_SIGNALING_330))
2149                         return -EINVAL;
2150                 /* Set 1.8V Signal Enable in the Host Control2 register to 0 */
2151                 ctrl &= ~SDHCI_CTRL_VDD_180;
2152                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2153
2154                 if (!IS_ERR(mmc->supply.vqmmc)) {
2155                         ret = mmc_regulator_set_vqmmc(mmc, ios);
2156                         if (ret) {
2157                                 pr_warn("%s: Switching to 3.3V signalling voltage failed\n",
2158                                         mmc_hostname(mmc));
2159                                 return -EIO;
2160                         }
2161                 }
2162                 /* Wait for 5ms */
2163                 usleep_range(5000, 5500);
2164
2165                 /* 3.3V regulator output should be stable within 5 ms */
2166                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2167                 if (!(ctrl & SDHCI_CTRL_VDD_180))
2168                         return 0;
2169
2170                 pr_warn("%s: 3.3V regulator output did not became stable\n",
2171                         mmc_hostname(mmc));
2172
2173                 return -EAGAIN;
2174         case MMC_SIGNAL_VOLTAGE_180:
2175                 if (!(host->flags & SDHCI_SIGNALING_180))
2176                         return -EINVAL;
2177                 if (!IS_ERR(mmc->supply.vqmmc)) {
2178                         ret = mmc_regulator_set_vqmmc(mmc, ios);
2179                         if (ret) {
2180                                 pr_warn("%s: Switching to 1.8V signalling voltage failed\n",
2181                                         mmc_hostname(mmc));
2182                                 return -EIO;
2183                         }
2184                 }
2185
2186                 /*
2187                  * Enable 1.8V Signal Enable in the Host Control2
2188                  * register
2189                  */
2190                 ctrl |= SDHCI_CTRL_VDD_180;
2191                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2192
2193                 /* Some controller need to do more when switching */
2194                 if (host->ops->voltage_switch)
2195                         host->ops->voltage_switch(host);
2196
2197                 /* 1.8V regulator output should be stable within 5 ms */
2198                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2199                 if (ctrl & SDHCI_CTRL_VDD_180)
2200                         return 0;
2201
2202                 pr_warn("%s: 1.8V regulator output did not became stable\n",
2203                         mmc_hostname(mmc));
2204
2205                 return -EAGAIN;
2206         case MMC_SIGNAL_VOLTAGE_120:
2207                 if (!(host->flags & SDHCI_SIGNALING_120))
2208                         return -EINVAL;
2209                 if (!IS_ERR(mmc->supply.vqmmc)) {
2210                         ret = mmc_regulator_set_vqmmc(mmc, ios);
2211                         if (ret) {
2212                                 pr_warn("%s: Switching to 1.2V signalling voltage failed\n",
2213                                         mmc_hostname(mmc));
2214                                 return -EIO;
2215                         }
2216                 }
2217                 return 0;
2218         default:
2219                 /* No signal voltage switch required */
2220                 return 0;
2221         }
2222 }
2223 EXPORT_SYMBOL_GPL(sdhci_start_signal_voltage_switch);
2224
2225 static int sdhci_card_busy(struct mmc_host *mmc)
2226 {
2227         struct sdhci_host *host = mmc_priv(mmc);
2228         u32 present_state;
2229
2230         /* Check whether DAT[0] is 0 */
2231         present_state = sdhci_readl(host, SDHCI_PRESENT_STATE);
2232
2233         return !(present_state & SDHCI_DATA_0_LVL_MASK);
2234 }
2235
2236 static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios)
2237 {
2238         struct sdhci_host *host = mmc_priv(mmc);
2239         unsigned long flags;
2240
2241         spin_lock_irqsave(&host->lock, flags);
2242         host->flags |= SDHCI_HS400_TUNING;
2243         spin_unlock_irqrestore(&host->lock, flags);
2244
2245         return 0;
2246 }
2247
2248 void sdhci_start_tuning(struct sdhci_host *host)
2249 {
2250         u16 ctrl;
2251
2252         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2253         ctrl |= SDHCI_CTRL_EXEC_TUNING;
2254         if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
2255                 ctrl |= SDHCI_CTRL_TUNED_CLK;
2256         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2257
2258         /*
2259          * As per the Host Controller spec v3.00, tuning command
2260          * generates Buffer Read Ready interrupt, so enable that.
2261          *
2262          * Note: The spec clearly says that when tuning sequence
2263          * is being performed, the controller does not generate
2264          * interrupts other than Buffer Read Ready interrupt. But
2265          * to make sure we don't hit a controller bug, we _only_
2266          * enable Buffer Read Ready interrupt here.
2267          */
2268         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
2269         sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
2270 }
2271 EXPORT_SYMBOL_GPL(sdhci_start_tuning);
2272
2273 void sdhci_end_tuning(struct sdhci_host *host)
2274 {
2275         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2276         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2277 }
2278 EXPORT_SYMBOL_GPL(sdhci_end_tuning);
2279
2280 void sdhci_reset_tuning(struct sdhci_host *host)
2281 {
2282         u16 ctrl;
2283
2284         ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2285         ctrl &= ~SDHCI_CTRL_TUNED_CLK;
2286         ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
2287         sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2288 }
2289 EXPORT_SYMBOL_GPL(sdhci_reset_tuning);
2290
2291 static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode)
2292 {
2293         sdhci_reset_tuning(host);
2294
2295         sdhci_do_reset(host, SDHCI_RESET_CMD);
2296         sdhci_do_reset(host, SDHCI_RESET_DATA);
2297
2298         sdhci_end_tuning(host);
2299
2300         mmc_abort_tuning(host->mmc, opcode);
2301 }
2302
2303 /*
2304  * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI
2305  * tuning command does not have a data payload (or rather the hardware does it
2306  * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
2307  * interrupt setup is different to other commands and there is no timeout
2308  * interrupt so special handling is needed.
2309  */
2310 void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
2311 {
2312         struct mmc_host *mmc = host->mmc;
2313         struct mmc_command cmd = {};
2314         struct mmc_request mrq = {};
2315         unsigned long flags;
2316         u32 b = host->sdma_boundary;
2317
2318         spin_lock_irqsave(&host->lock, flags);
2319
2320         cmd.opcode = opcode;
2321         cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
2322         cmd.mrq = &mrq;
2323
2324         mrq.cmd = &cmd;
2325         /*
2326          * In response to CMD19, the card sends 64 bytes of tuning
2327          * block to the Host Controller. So we set the block size
2328          * to 64 here.
2329          */
2330         if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
2331             mmc->ios.bus_width == MMC_BUS_WIDTH_8)
2332                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE);
2333         else
2334                 sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE);
2335
2336         /*
2337          * The tuning block is sent by the card to the host controller.
2338          * So we set the TRNS_READ bit in the Transfer Mode register.
2339          * This also takes care of setting DMA Enable and Multi Block
2340          * Select in the same register to 0.
2341          */
2342         sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
2343
2344         sdhci_send_command(host, &cmd);
2345
2346         host->cmd = NULL;
2347
2348         sdhci_del_timer(host, &mrq);
2349
2350         host->tuning_done = 0;
2351
2352         spin_unlock_irqrestore(&host->lock, flags);
2353
2354         /* Wait for Buffer Read Ready interrupt */
2355         wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),
2356                            msecs_to_jiffies(50));
2357
2358 }
2359 EXPORT_SYMBOL_GPL(sdhci_send_tuning);
2360
2361 static int __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
2362 {
2363         int i;
2364
2365         /*
2366          * Issue opcode repeatedly till Execute Tuning is set to 0 or the number
2367          * of loops reaches 40 times.
2368          */
2369         for (i = 0; i < MAX_TUNING_LOOP; i++) {
2370                 u16 ctrl;
2371
2372                 sdhci_send_tuning(host, opcode);
2373
2374                 if (!host->tuning_done) {
2375                         pr_info("%s: Tuning timeout, falling back to fixed sampling clock\n",
2376                                 mmc_hostname(host->mmc));
2377                         sdhci_abort_tuning(host, opcode);
2378                         return -ETIMEDOUT;
2379                 }
2380
2381                 /* Spec does not require a delay between tuning cycles */
2382                 if (host->tuning_delay > 0)
2383                         mdelay(host->tuning_delay);
2384
2385                 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2386                 if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) {
2387                         if (ctrl & SDHCI_CTRL_TUNED_CLK)
2388                                 return 0; /* Success! */
2389                         break;
2390                 }
2391
2392         }
2393
2394         pr_info("%s: Tuning failed, falling back to fixed sampling clock\n",
2395                 mmc_hostname(host->mmc));
2396         sdhci_reset_tuning(host);
2397         return -EAGAIN;
2398 }
2399
2400 int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
2401 {
2402         struct sdhci_host *host = mmc_priv(mmc);
2403         int err = 0;
2404         unsigned int tuning_count = 0;
2405         bool hs400_tuning;
2406
2407         hs400_tuning = host->flags & SDHCI_HS400_TUNING;
2408
2409         if (host->tuning_mode == SDHCI_TUNING_MODE_1)
2410                 tuning_count = host->tuning_count;
2411
2412         /*
2413          * The Host Controller needs tuning in case of SDR104 and DDR50
2414          * mode, and for SDR50 mode when Use Tuning for SDR50 is set in
2415          * the Capabilities register.
2416          * If the Host Controller supports the HS200 mode then the
2417          * tuning function has to be executed.
2418          */
2419         switch (host->timing) {
2420         /* HS400 tuning is done in HS200 mode */
2421         case MMC_TIMING_MMC_HS400:
2422                 err = -EINVAL;
2423                 goto out;
2424
2425         case MMC_TIMING_MMC_HS200:
2426                 /*
2427                  * Periodic re-tuning for HS400 is not expected to be needed, so
2428                  * disable it here.
2429                  */
2430                 if (hs400_tuning)
2431                         tuning_count = 0;
2432                 break;
2433
2434         case MMC_TIMING_UHS_SDR104:
2435         case MMC_TIMING_UHS_DDR50:
2436                 break;
2437
2438         case MMC_TIMING_UHS_SDR50:
2439                 if (host->flags & SDHCI_SDR50_NEEDS_TUNING)
2440                         break;
2441                 /* FALLTHROUGH */
2442
2443         default:
2444                 goto out;
2445         }
2446
2447         if (host->ops->platform_execute_tuning) {
2448                 err = host->ops->platform_execute_tuning(host, opcode);
2449                 goto out;
2450         }
2451
2452         host->mmc->retune_period = tuning_count;
2453
2454         if (host->tuning_delay < 0)
2455                 host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK;
2456
2457         sdhci_start_tuning(host);
2458
2459         host->tuning_err = __sdhci_execute_tuning(host, opcode);
2460
2461         sdhci_end_tuning(host);
2462 out:
2463         host->flags &= ~SDHCI_HS400_TUNING;
2464
2465         return err;
2466 }
2467 EXPORT_SYMBOL_GPL(sdhci_execute_tuning);
2468
2469 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable)
2470 {
2471         /* Host Controller v3.00 defines preset value registers */
2472         if (host->version < SDHCI_SPEC_300)
2473                 return;
2474
2475         /*
2476          * We only enable or disable Preset Value if they are not already
2477          * enabled or disabled respectively. Otherwise, we bail out.
2478          */
2479         if (host->preset_enabled != enable) {
2480                 u16 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2481
2482                 if (enable)
2483                         ctrl |= SDHCI_CTRL_PRESET_VAL_ENABLE;
2484                 else
2485                         ctrl &= ~SDHCI_CTRL_PRESET_VAL_ENABLE;
2486
2487                 sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2488
2489                 if (enable)
2490                         host->flags |= SDHCI_PV_ENABLED;
2491                 else
2492                         host->flags &= ~SDHCI_PV_ENABLED;
2493
2494                 host->preset_enabled = enable;
2495         }
2496 }
2497
2498 static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
2499                                 int err)
2500 {
2501         struct sdhci_host *host = mmc_priv(mmc);
2502         struct mmc_data *data = mrq->data;
2503
2504         if (data->host_cookie != COOKIE_UNMAPPED)
2505                 dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len,
2506                              mmc_get_dma_dir(data));
2507
2508         data->host_cookie = COOKIE_UNMAPPED;
2509 }
2510
2511 static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
2512 {
2513         struct sdhci_host *host = mmc_priv(mmc);
2514
2515         mrq->data->host_cookie = COOKIE_UNMAPPED;
2516
2517         /*
2518          * No pre-mapping in the pre hook if we're using the bounce buffer,
2519          * for that we would need two bounce buffers since one buffer is
2520          * in flight when this is getting called.
2521          */
2522         if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)
2523                 sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);
2524 }
2525
2526 static inline bool sdhci_has_requests(struct sdhci_host *host)
2527 {
2528         return host->cmd || host->data_cmd;
2529 }
2530
2531 static void sdhci_error_out_mrqs(struct sdhci_host *host, int err)
2532 {
2533         if (host->data_cmd) {
2534                 host->data_cmd->error = err;
2535                 sdhci_finish_mrq(host, host->data_cmd->mrq);
2536         }
2537
2538         if (host->cmd) {
2539                 host->cmd->error = err;
2540                 sdhci_finish_mrq(host, host->cmd->mrq);
2541         }
2542 }
2543
2544 static void sdhci_card_event(struct mmc_host *mmc)
2545 {
2546         struct sdhci_host *host = mmc_priv(mmc);
2547         unsigned long flags;
2548         int present;
2549
2550         /* First check if client has provided their own card event */
2551         if (host->ops->card_event)
2552                 host->ops->card_event(host);
2553
2554         present = mmc->ops->get_cd(mmc);
2555
2556         spin_lock_irqsave(&host->lock, flags);
2557
2558         /* Check sdhci_has_requests() first in case we are runtime suspended */
2559         if (sdhci_has_requests(host) && !present) {
2560                 pr_err("%s: Card removed during transfer!\n",
2561                         mmc_hostname(host->mmc));
2562                 pr_err("%s: Resetting controller.\n",
2563                         mmc_hostname(host->mmc));
2564
2565                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2566                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2567
2568                 sdhci_error_out_mrqs(host, -ENOMEDIUM);
2569         }
2570
2571         spin_unlock_irqrestore(&host->lock, flags);
2572 }
2573
2574 static const struct mmc_host_ops sdhci_ops = {
2575         .request        = sdhci_request,
2576         .post_req       = sdhci_post_req,
2577         .pre_req        = sdhci_pre_req,
2578         .set_ios        = sdhci_set_ios,
2579         .get_cd         = sdhci_get_cd,
2580         .get_ro         = sdhci_get_ro,
2581         .hw_reset       = sdhci_hw_reset,
2582         .enable_sdio_irq = sdhci_enable_sdio_irq,
2583         .start_signal_voltage_switch    = sdhci_start_signal_voltage_switch,
2584         .prepare_hs400_tuning           = sdhci_prepare_hs400_tuning,
2585         .execute_tuning                 = sdhci_execute_tuning,
2586         .card_event                     = sdhci_card_event,
2587         .card_busy      = sdhci_card_busy,
2588 };
2589
2590 /*****************************************************************************\
2591  *                                                                           *
2592  * Tasklets                                                                  *
2593  *                                                                           *
2594 \*****************************************************************************/
2595
2596 static bool sdhci_request_done(struct sdhci_host *host)
2597 {
2598         unsigned long flags;
2599         struct mmc_request *mrq;
2600         int i;
2601
2602         spin_lock_irqsave(&host->lock, flags);
2603
2604         for (i = 0; i < SDHCI_MAX_MRQS; i++) {
2605                 mrq = host->mrqs_done[i];
2606                 if (mrq)
2607                         break;
2608         }
2609
2610         if (!mrq) {
2611                 spin_unlock_irqrestore(&host->lock, flags);
2612                 return true;
2613         }
2614
2615         sdhci_del_timer(host, mrq);
2616
2617         /*
2618          * Always unmap the data buffers if they were mapped by
2619          * sdhci_prepare_data() whenever we finish with a request.
2620          * This avoids leaking DMA mappings on error.
2621          */
2622         if (host->flags & SDHCI_REQ_USE_DMA) {
2623                 struct mmc_data *data = mrq->data;
2624
2625                 if (data && data->host_cookie == COOKIE_MAPPED) {
2626                         if (host->bounce_buffer) {
2627                                 /*
2628                                  * On reads, copy the bounced data into the
2629                                  * sglist
2630                                  */
2631                                 if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) {
2632                                         unsigned int length = data->bytes_xfered;
2633
2634                                         if (length > host->bounce_buffer_size) {
2635                                                 pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n",
2636                                                        mmc_hostname(host->mmc),
2637                                                        host->bounce_buffer_size,
2638                                                        data->bytes_xfered);
2639                                                 /* Cap it down and continue */
2640                                                 length = host->bounce_buffer_size;
2641                                         }
2642                                         dma_sync_single_for_cpu(
2643                                                 host->mmc->parent,
2644                                                 host->bounce_addr,
2645                                                 host->bounce_buffer_size,
2646                                                 DMA_FROM_DEVICE);
2647                                         sg_copy_from_buffer(data->sg,
2648                                                 data->sg_len,
2649                                                 host->bounce_buffer,
2650                                                 length);
2651                                 } else {
2652                                         /* No copying, just switch ownership */
2653                                         dma_sync_single_for_cpu(
2654                                                 host->mmc->parent,
2655                                                 host->bounce_addr,
2656                                                 host->bounce_buffer_size,
2657                                                 mmc_get_dma_dir(data));
2658                                 }
2659                         } else {
2660                                 /* Unmap the raw data */
2661                                 dma_unmap_sg(mmc_dev(host->mmc), data->sg,
2662                                              data->sg_len,
2663                                              mmc_get_dma_dir(data));
2664                         }
2665                         data->host_cookie = COOKIE_UNMAPPED;
2666                 }
2667         }
2668
2669         /*
2670          * The controller needs a reset of internal state machines
2671          * upon error conditions.
2672          */
2673         if (sdhci_needs_reset(host, mrq)) {
2674                 /*
2675                  * Do not finish until command and data lines are available for
2676                  * reset. Note there can only be one other mrq, so it cannot
2677                  * also be in mrqs_done, otherwise host->cmd and host->data_cmd
2678                  * would both be null.
2679                  */
2680                 if (host->cmd || host->data_cmd) {
2681                         spin_unlock_irqrestore(&host->lock, flags);
2682                         return true;
2683                 }
2684
2685                 /* Some controllers need this kick or reset won't work here */
2686                 if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)
2687                         /* This is to force an update */
2688                         host->ops->set_clock(host, host->clock);
2689
2690                 /* Spec says we should do both at the same time, but Ricoh
2691                    controllers do not like that. */
2692                 sdhci_do_reset(host, SDHCI_RESET_CMD);
2693                 sdhci_do_reset(host, SDHCI_RESET_DATA);
2694
2695                 host->pending_reset = false;
2696         }
2697
2698         if (!sdhci_has_requests(host))
2699                 sdhci_led_deactivate(host);
2700
2701         host->mrqs_done[i] = NULL;
2702
2703         spin_unlock_irqrestore(&host->lock, flags);
2704
2705         mmc_request_done(host->mmc, mrq);
2706
2707         return false;
2708 }
2709
2710 static void sdhci_tasklet_finish(unsigned long param)
2711 {
2712         struct sdhci_host *host = (struct sdhci_host *)param;
2713
2714         while (!sdhci_request_done(host))
2715                 ;
2716 }
2717
2718 static void sdhci_timeout_timer(struct timer_list *t)
2719 {
2720         struct sdhci_host *host;
2721         unsigned long flags;
2722
2723         host = from_timer(host, t, timer);
2724
2725         spin_lock_irqsave(&host->lock, flags);
2726
2727         if (host->cmd && !sdhci_data_line_cmd(host->cmd)) {
2728                 pr_err("%s: Timeout waiting for hardware cmd interrupt.\n",
2729                        mmc_hostname(host->mmc));
2730                 sdhci_dumpregs(host);
2731
2732                 host->cmd->error = -ETIMEDOUT;
2733                 sdhci_finish_mrq(host, host->cmd->mrq);
2734         }
2735
2736         spin_unlock_irqrestore(&host->lock, flags);
2737 }
2738
2739 static void sdhci_timeout_data_timer(struct timer_list *t)
2740 {
2741         struct sdhci_host *host;
2742         unsigned long flags;
2743
2744         host = from_timer(host, t, data_timer);
2745
2746         spin_lock_irqsave(&host->lock, flags);
2747
2748         if (host->data || host->data_cmd ||
2749             (host->cmd && sdhci_data_line_cmd(host->cmd))) {
2750                 pr_err("%s: Timeout waiting for hardware interrupt.\n",
2751                        mmc_hostname(host->mmc));
2752                 sdhci_dumpregs(host);
2753
2754                 if (host->data) {
2755                         host->data->error = -ETIMEDOUT;
2756                         sdhci_finish_data(host);
2757                 } else if (host->data_cmd) {
2758                         host->data_cmd->error = -ETIMEDOUT;
2759                         sdhci_finish_mrq(host, host->data_cmd->mrq);
2760                 } else {
2761                         host->cmd->error = -ETIMEDOUT;
2762                         sdhci_finish_mrq(host, host->cmd->mrq);
2763                 }
2764         }
2765
2766         spin_unlock_irqrestore(&host->lock, flags);
2767 }
2768
2769 /*****************************************************************************\
2770  *                                                                           *
2771  * Interrupt handling                                                        *
2772  *                                                                           *
2773 \*****************************************************************************/
2774
2775 static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask, u32 *intmask_p)
2776 {
2777         /* Handle auto-CMD12 error */
2778         if (intmask & SDHCI_INT_AUTO_CMD_ERR && host->data_cmd) {
2779                 struct mmc_request *mrq = host->data_cmd->mrq;
2780                 u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
2781                 int data_err_bit = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
2782                                    SDHCI_INT_DATA_TIMEOUT :
2783                                    SDHCI_INT_DATA_CRC;
2784
2785                 /* Treat auto-CMD12 error the same as data error */
2786                 if (!mrq->sbc && (host->flags & SDHCI_AUTO_CMD12)) {
2787                         *intmask_p |= data_err_bit;
2788                         return;
2789                 }
2790         }
2791
2792         if (!host->cmd) {
2793                 /*
2794                  * SDHCI recovers from errors by resetting the cmd and data
2795                  * circuits.  Until that is done, there very well might be more
2796                  * interrupts, so ignore them in that case.
2797                  */
2798                 if (host->pending_reset)
2799                         return;
2800                 pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n",
2801                        mmc_hostname(host->mmc), (unsigned)intmask);
2802                 sdhci_dumpregs(host);
2803                 return;
2804         }
2805
2806         if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC |
2807                        SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) {
2808                 if (intmask & SDHCI_INT_TIMEOUT)
2809                         host->cmd->error = -ETIMEDOUT;
2810                 else
2811                         host->cmd->error = -EILSEQ;
2812
2813                 /* Treat data command CRC error the same as data CRC error */
2814                 if (host->cmd->data &&
2815                     (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) ==
2816                      SDHCI_INT_CRC) {
2817                         host->cmd = NULL;
2818                         *intmask_p |= SDHCI_INT_DATA_CRC;
2819                         return;
2820                 }
2821
2822                 sdhci_finish_mrq(host, host->cmd->mrq);
2823                 return;
2824         }
2825
2826         /* Handle auto-CMD23 error */
2827         if (intmask & SDHCI_INT_AUTO_CMD_ERR) {
2828                 struct mmc_request *mrq = host->cmd->mrq;
2829                 u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS);
2830                 int err = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ?
2831                           -ETIMEDOUT :
2832                           -EILSEQ;
2833
2834                 if (mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
2835                         mrq->sbc->error = err;
2836                         sdhci_finish_mrq(host, mrq);
2837                         return;
2838                 }
2839         }
2840
2841         if (intmask & SDHCI_INT_RESPONSE)
2842                 sdhci_finish_command(host);
2843 }
2844
2845 static void sdhci_adma_show_error(struct sdhci_host *host)
2846 {
2847         void *desc = host->adma_table;
2848
2849         sdhci_dumpregs(host);
2850
2851         while (true) {
2852                 struct sdhci_adma2_64_desc *dma_desc = desc;
2853
2854                 if (host->flags & SDHCI_USE_64_BIT_DMA)
2855                         DBG("%p: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n",
2856                             desc, le32_to_cpu(dma_desc->addr_hi),
2857                             le32_to_cpu(dma_desc->addr_lo),
2858                             le16_to_cpu(dma_desc->len),
2859                             le16_to_cpu(dma_desc->cmd));
2860                 else
2861                         DBG("%p: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n",
2862                             desc, le32_to_cpu(dma_desc->addr_lo),
2863                             le16_to_cpu(dma_desc->len),
2864                             le16_to_cpu(dma_desc->cmd));
2865
2866                 desc += host->desc_sz;
2867
2868                 if (dma_desc->cmd & cpu_to_le16(ADMA2_END))
2869                         break;
2870         }
2871 }
2872
2873 static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
2874 {
2875         u32 command;
2876
2877         /* CMD19 generates _only_ Buffer Read Ready interrupt */
2878         if (intmask & SDHCI_INT_DATA_AVAIL) {
2879                 command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND));
2880                 if (command == MMC_SEND_TUNING_BLOCK ||
2881                     command == MMC_SEND_TUNING_BLOCK_HS200) {
2882                         host->tuning_done = 1;
2883                         wake_up(&host->buf_ready_int);
2884                         return;
2885                 }
2886         }
2887
2888         if (!host->data) {
2889                 struct mmc_command *data_cmd = host->data_cmd;
2890
2891                 /*
2892                  * The "data complete" interrupt is also used to
2893                  * indicate that a busy state has ended. See comment
2894                  * above in sdhci_cmd_irq().
2895                  */
2896                 if (data_cmd && (data_cmd->flags & MMC_RSP_BUSY)) {
2897                         if (intmask & SDHCI_INT_DATA_TIMEOUT) {
2898                                 host->data_cmd = NULL;
2899                                 data_cmd->error = -ETIMEDOUT;
2900                                 sdhci_finish_mrq(host, data_cmd->mrq);
2901                                 return;
2902                         }
2903                         if (intmask & SDHCI_INT_DATA_END) {
2904                                 host->data_cmd = NULL;
2905                                 /*
2906                                  * Some cards handle busy-end interrupt
2907                                  * before the command completed, so make
2908                                  * sure we do things in the proper order.
2909                                  */
2910                                 if (host->cmd == data_cmd)
2911                                         return;
2912
2913                                 sdhci_finish_mrq(host, data_cmd->mrq);
2914                                 return;
2915                         }
2916                 }
2917
2918                 /*
2919                  * SDHCI recovers from errors by resetting the cmd and data
2920                  * circuits. Until that is done, there very well might be more
2921                  * interrupts, so ignore them in that case.
2922                  */
2923                 if (host->pending_reset)
2924                         return;
2925
2926                 pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n",
2927                        mmc_hostname(host->mmc), (unsigned)intmask);
2928                 sdhci_dumpregs(host);
2929
2930                 return;
2931         }
2932
2933         if (intmask & SDHCI_INT_DATA_TIMEOUT)
2934                 host->data->error = -ETIMEDOUT;
2935         else if (intmask & SDHCI_INT_DATA_END_BIT)
2936                 host->data->error = -EILSEQ;
2937         else if ((intmask & SDHCI_INT_DATA_CRC) &&
2938                 SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))
2939                         != MMC_BUS_TEST_R)
2940                 host->data->error = -EILSEQ;
2941         else if (intmask & SDHCI_INT_ADMA_ERROR) {
2942                 pr_err("%s: ADMA error\n", mmc_hostname(host->mmc));
2943                 sdhci_adma_show_error(host);
2944                 host->data->error = -EIO;
2945                 if (host->ops->adma_workaround)
2946                         host->ops->adma_workaround(host, intmask);
2947         }
2948
2949         if (host->data->error)
2950                 sdhci_finish_data(host);
2951         else {
2952                 if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
2953                         sdhci_transfer_pio(host);
2954
2955                 /*
2956                  * We currently don't do anything fancy with DMA
2957                  * boundaries, but as we can't disable the feature
2958                  * we need to at least restart the transfer.
2959                  *
2960                  * According to the spec sdhci_readl(host, SDHCI_DMA_ADDRESS)
2961                  * should return a valid address to continue from, but as
2962                  * some controllers are faulty, don't trust them.
2963                  */
2964                 if (intmask & SDHCI_INT_DMA_END) {
2965                         dma_addr_t dmastart, dmanow;
2966
2967                         dmastart = sdhci_sdma_address(host);
2968                         dmanow = dmastart + host->data->bytes_xfered;
2969                         /*
2970                          * Force update to the next DMA block boundary.
2971                          */
2972                         dmanow = (dmanow &
2973                                 ~((dma_addr_t)SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) +
2974                                 SDHCI_DEFAULT_BOUNDARY_SIZE;
2975                         host->data->bytes_xfered = dmanow - dmastart;
2976                         DBG("DMA base %pad, transferred 0x%06x bytes, next %pad\n",
2977                             &dmastart, host->data->bytes_xfered, &dmanow);
2978                         sdhci_set_sdma_addr(host, dmanow);
2979                 }
2980
2981                 if (intmask & SDHCI_INT_DATA_END) {
2982                         if (host->cmd == host->data_cmd) {
2983                                 /*
2984                                  * Data managed to finish before the
2985                                  * command completed. Make sure we do
2986                                  * things in the proper order.
2987                                  */
2988                                 host->data_early = 1;
2989                         } else {
2990                                 sdhci_finish_data(host);
2991                         }
2992                 }
2993         }
2994 }
2995
2996 static irqreturn_t sdhci_irq(int irq, void *dev_id)
2997 {
2998         irqreturn_t result = IRQ_NONE;
2999         struct sdhci_host *host = dev_id;
3000         u32 intmask, mask, unexpected = 0;
3001         int max_loops = 16;
3002
3003         spin_lock(&host->lock);
3004
3005         if (host->runtime_suspended && !sdhci_sdio_irq_enabled(host)) {
3006                 spin_unlock(&host->lock);
3007                 return IRQ_NONE;
3008         }
3009
3010         intmask = sdhci_readl(host, SDHCI_INT_STATUS);
3011         if (!intmask || intmask == 0xffffffff) {
3012                 result = IRQ_NONE;
3013                 goto out;
3014         }
3015
3016         do {
3017                 DBG("IRQ status 0x%08x\n", intmask);
3018
3019                 if (host->ops->irq) {
3020                         intmask = host->ops->irq(host, intmask);
3021                         if (!intmask)
3022                                 goto cont;
3023                 }
3024
3025                 /* Clear selected interrupts. */
3026                 mask = intmask & (SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
3027                                   SDHCI_INT_BUS_POWER);
3028                 sdhci_writel(host, mask, SDHCI_INT_STATUS);
3029
3030                 if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
3031                         u32 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
3032                                       SDHCI_CARD_PRESENT;
3033
3034                         /*
3035                          * There is a observation on i.mx esdhc.  INSERT
3036                          * bit will be immediately set again when it gets
3037                          * cleared, if a card is inserted.  We have to mask
3038                          * the irq to prevent interrupt storm which will
3039                          * freeze the system.  And the REMOVE gets the
3040                          * same situation.
3041                          *
3042                          * More testing are needed here to ensure it works
3043                          * for other platforms though.
3044                          */
3045                         host->ier &= ~(SDHCI_INT_CARD_INSERT |
3046                                        SDHCI_INT_CARD_REMOVE);
3047                         host->ier |= present ? SDHCI_INT_CARD_REMOVE :
3048                                                SDHCI_INT_CARD_INSERT;
3049                         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3050                         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3051
3052                         sdhci_writel(host, intmask & (SDHCI_INT_CARD_INSERT |
3053                                      SDHCI_INT_CARD_REMOVE), SDHCI_INT_STATUS);
3054
3055                         host->thread_isr |= intmask & (SDHCI_INT_CARD_INSERT |
3056                                                        SDHCI_INT_CARD_REMOVE);
3057                         result = IRQ_WAKE_THREAD;
3058                 }
3059
3060                 if (intmask & SDHCI_INT_CMD_MASK)
3061                         sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK, &intmask);
3062
3063                 if (intmask & SDHCI_INT_DATA_MASK)
3064                         sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK);
3065
3066                 if (intmask & SDHCI_INT_BUS_POWER)
3067                         pr_err("%s: Card is consuming too much power!\n",
3068                                 mmc_hostname(host->mmc));
3069
3070                 if (intmask & SDHCI_INT_RETUNE)
3071                         mmc_retune_needed(host->mmc);
3072
3073                 if ((intmask & SDHCI_INT_CARD_INT) &&
3074                     (host->ier & SDHCI_INT_CARD_INT)) {
3075                         sdhci_enable_sdio_irq_nolock(host, false);
3076                         host->thread_isr |= SDHCI_INT_CARD_INT;
3077                         result = IRQ_WAKE_THREAD;
3078                 }
3079
3080                 intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
3081                              SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
3082                              SDHCI_INT_ERROR | SDHCI_INT_BUS_POWER |
3083                              SDHCI_INT_RETUNE | SDHCI_INT_CARD_INT);
3084
3085                 if (intmask) {
3086                         unexpected |= intmask;
3087                         sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3088                 }
3089 cont:
3090                 if (result == IRQ_NONE)
3091                         result = IRQ_HANDLED;
3092
3093                 intmask = sdhci_readl(host, SDHCI_INT_STATUS);
3094         } while (intmask && --max_loops);
3095 out:
3096         spin_unlock(&host->lock);
3097
3098         if (unexpected) {
3099                 pr_err("%s: Unexpected interrupt 0x%08x.\n",
3100                            mmc_hostname(host->mmc), unexpected);
3101                 sdhci_dumpregs(host);
3102         }
3103
3104         return result;
3105 }
3106
3107 static irqreturn_t sdhci_thread_irq(int irq, void *dev_id)
3108 {
3109         struct sdhci_host *host = dev_id;
3110         unsigned long flags;
3111         u32 isr;
3112
3113         spin_lock_irqsave(&host->lock, flags);
3114         isr = host->thread_isr;
3115         host->thread_isr = 0;
3116         spin_unlock_irqrestore(&host->lock, flags);
3117
3118         if (isr & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
3119                 struct mmc_host *mmc = host->mmc;
3120
3121                 mmc->ops->card_event(mmc);
3122                 mmc_detect_change(mmc, msecs_to_jiffies(200));
3123         }
3124
3125         if (isr & SDHCI_INT_CARD_INT) {
3126                 sdio_run_irqs(host->mmc);
3127
3128                 spin_lock_irqsave(&host->lock, flags);
3129                 if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
3130                         sdhci_enable_sdio_irq_nolock(host, true);
3131                 spin_unlock_irqrestore(&host->lock, flags);
3132         }
3133
3134         return isr ? IRQ_HANDLED : IRQ_NONE;
3135 }
3136
3137 /*****************************************************************************\
3138  *                                                                           *
3139  * Suspend/resume                                                            *
3140  *                                                                           *
3141 \*****************************************************************************/
3142
3143 #ifdef CONFIG_PM
3144
3145 static bool sdhci_cd_irq_can_wakeup(struct sdhci_host *host)
3146 {
3147         return mmc_card_is_removable(host->mmc) &&
3148                !(host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3149                !mmc_can_gpio_cd(host->mmc);
3150 }
3151
3152 /*
3153  * To enable wakeup events, the corresponding events have to be enabled in
3154  * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal
3155  * Table' in the SD Host Controller Standard Specification.
3156  * It is useless to restore SDHCI_INT_ENABLE state in
3157  * sdhci_disable_irq_wakeups() since it will be set by
3158  * sdhci_enable_card_detection() or sdhci_init().
3159  */
3160 static bool sdhci_enable_irq_wakeups(struct sdhci_host *host)
3161 {
3162         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE |
3163                   SDHCI_WAKE_ON_INT;
3164         u32 irq_val = 0;
3165         u8 wake_val = 0;
3166         u8 val;
3167
3168         if (sdhci_cd_irq_can_wakeup(host)) {
3169                 wake_val |= SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE;
3170                 irq_val |= SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE;
3171         }
3172
3173         if (mmc_card_wake_sdio_irq(host->mmc)) {
3174                 wake_val |= SDHCI_WAKE_ON_INT;
3175                 irq_val |= SDHCI_INT_CARD_INT;
3176         }
3177
3178         if (!irq_val)
3179                 return false;
3180
3181         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3182         val &= ~mask;
3183         val |= wake_val;
3184         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3185
3186         sdhci_writel(host, irq_val, SDHCI_INT_ENABLE);
3187
3188         host->irq_wake_enabled = !enable_irq_wake(host->irq);
3189
3190         return host->irq_wake_enabled;
3191 }
3192
3193 static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
3194 {
3195         u8 val;
3196         u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
3197                         | SDHCI_WAKE_ON_INT;
3198
3199         val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3200         val &= ~mask;
3201         sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3202
3203         disable_irq_wake(host->irq);
3204
3205         host->irq_wake_enabled = false;
3206 }
3207
3208 int sdhci_suspend_host(struct sdhci_host *host)
3209 {
3210         sdhci_disable_card_detection(host);
3211
3212         mmc_retune_timer_stop(host->mmc);
3213
3214         if (!device_may_wakeup(mmc_dev(host->mmc)) ||
3215             !sdhci_enable_irq_wakeups(host)) {
3216                 host->ier = 0;
3217                 sdhci_writel(host, 0, SDHCI_INT_ENABLE);
3218                 sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
3219                 free_irq(host->irq, host);
3220         }
3221
3222         return 0;
3223 }
3224
3225 EXPORT_SYMBOL_GPL(sdhci_suspend_host);
3226
3227 int sdhci_resume_host(struct sdhci_host *host)
3228 {
3229         struct mmc_host *mmc = host->mmc;
3230         int ret = 0;
3231
3232         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3233                 if (host->ops->enable_dma)
3234                         host->ops->enable_dma(host);
3235         }
3236
3237         if ((host->mmc->pm_flags & MMC_PM_KEEP_POWER) &&
3238             (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) {
3239                 /* Card keeps power but host controller does not */
3240                 sdhci_init(host, 0);
3241                 host->pwr = 0;
3242                 host->clock = 0;
3243                 mmc->ops->set_ios(mmc, &mmc->ios);
3244         } else {
3245                 sdhci_init(host, (host->mmc->pm_flags & MMC_PM_KEEP_POWER));
3246         }
3247
3248         if (host->irq_wake_enabled) {
3249                 sdhci_disable_irq_wakeups(host);
3250         } else {
3251                 ret = request_threaded_irq(host->irq, sdhci_irq,
3252                                            sdhci_thread_irq, IRQF_SHARED,
3253                                            mmc_hostname(host->mmc), host);
3254                 if (ret)
3255                         return ret;
3256         }
3257
3258         sdhci_enable_card_detection(host);
3259
3260         return ret;
3261 }
3262
3263 EXPORT_SYMBOL_GPL(sdhci_resume_host);
3264
3265 int sdhci_runtime_suspend_host(struct sdhci_host *host)
3266 {
3267         unsigned long flags;
3268
3269         mmc_retune_timer_stop(host->mmc);
3270
3271         spin_lock_irqsave(&host->lock, flags);
3272         host->ier &= SDHCI_INT_CARD_INT;
3273         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3274         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3275         spin_unlock_irqrestore(&host->lock, flags);
3276
3277         synchronize_hardirq(host->irq);
3278
3279         spin_lock_irqsave(&host->lock, flags);
3280         host->runtime_suspended = true;
3281         spin_unlock_irqrestore(&host->lock, flags);
3282
3283         return 0;
3284 }
3285 EXPORT_SYMBOL_GPL(sdhci_runtime_suspend_host);
3286
3287 int sdhci_runtime_resume_host(struct sdhci_host *host)
3288 {
3289         struct mmc_host *mmc = host->mmc;
3290         unsigned long flags;
3291         int host_flags = host->flags;
3292
3293         if (host_flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3294                 if (host->ops->enable_dma)
3295                         host->ops->enable_dma(host);
3296         }
3297
3298         sdhci_init(host, 0);
3299
3300         if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&
3301             mmc->ios.power_mode != MMC_POWER_OFF) {
3302                 /* Force clock and power re-program */
3303                 host->pwr = 0;
3304                 host->clock = 0;
3305                 mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);
3306                 mmc->ops->set_ios(mmc, &mmc->ios);
3307
3308                 if ((host_flags & SDHCI_PV_ENABLED) &&
3309                     !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) {
3310                         spin_lock_irqsave(&host->lock, flags);
3311                         sdhci_enable_preset_value(host, true);
3312                         spin_unlock_irqrestore(&host->lock, flags);
3313                 }
3314
3315                 if ((mmc->caps2 & MMC_CAP2_HS400_ES) &&
3316                     mmc->ops->hs400_enhanced_strobe)
3317                         mmc->ops->hs400_enhanced_strobe(mmc, &mmc->ios);
3318         }
3319
3320         spin_lock_irqsave(&host->lock, flags);
3321
3322         host->runtime_suspended = false;
3323
3324         /* Enable SDIO IRQ */
3325         if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
3326                 sdhci_enable_sdio_irq_nolock(host, true);
3327
3328         /* Enable Card Detection */
3329         sdhci_enable_card_detection(host);
3330
3331         spin_unlock_irqrestore(&host->lock, flags);
3332
3333         return 0;
3334 }
3335 EXPORT_SYMBOL_GPL(sdhci_runtime_resume_host);
3336
3337 #endif /* CONFIG_PM */
3338
3339 /*****************************************************************************\
3340  *                                                                           *
3341  * Command Queue Engine (CQE) helpers                                        *
3342  *                                                                           *
3343 \*****************************************************************************/
3344
3345 void sdhci_cqe_enable(struct mmc_host *mmc)
3346 {
3347         struct sdhci_host *host = mmc_priv(mmc);
3348         unsigned long flags;
3349         u8 ctrl;
3350
3351         spin_lock_irqsave(&host->lock, flags);
3352
3353         ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
3354         ctrl &= ~SDHCI_CTRL_DMA_MASK;
3355         /*
3356          * Host from V4.10 supports ADMA3 DMA type.
3357          * ADMA3 performs integrated descriptor which is more suitable
3358          * for cmd queuing to fetch both command and transfer descriptors.
3359          */
3360         if (host->v4_mode && (host->caps1 & SDHCI_CAN_DO_ADMA3))
3361                 ctrl |= SDHCI_CTRL_ADMA3;
3362         else if (host->flags & SDHCI_USE_64_BIT_DMA)
3363                 ctrl |= SDHCI_CTRL_ADMA64;
3364         else
3365                 ctrl |= SDHCI_CTRL_ADMA32;
3366         sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
3367
3368         sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),
3369                      SDHCI_BLOCK_SIZE);
3370
3371         /* Set maximum timeout */
3372         sdhci_set_timeout(host, NULL);
3373
3374         host->ier = host->cqe_ier;
3375
3376         sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3377         sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3378
3379         host->cqe_on = true;
3380
3381         pr_debug("%s: sdhci: CQE on, IRQ mask %#x, IRQ status %#x\n",
3382                  mmc_hostname(mmc), host->ier,
3383                  sdhci_readl(host, SDHCI_INT_STATUS));
3384
3385         spin_unlock_irqrestore(&host->lock, flags);
3386 }
3387 EXPORT_SYMBOL_GPL(sdhci_cqe_enable);
3388
3389 void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery)
3390 {
3391         struct sdhci_host *host = mmc_priv(mmc);
3392         unsigned long flags;
3393
3394         spin_lock_irqsave(&host->lock, flags);
3395
3396         sdhci_set_default_irqs(host);
3397
3398         host->cqe_on = false;
3399
3400         if (recovery) {
3401                 sdhci_do_reset(host, SDHCI_RESET_CMD);
3402                 sdhci_do_reset(host, SDHCI_RESET_DATA);
3403         }
3404
3405         pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n",
3406                  mmc_hostname(mmc), host->ier,
3407                  sdhci_readl(host, SDHCI_INT_STATUS));
3408
3409         spin_unlock_irqrestore(&host->lock, flags);
3410 }
3411 EXPORT_SYMBOL_GPL(sdhci_cqe_disable);
3412
3413 bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
3414                    int *data_error)
3415 {
3416         u32 mask;
3417
3418         if (!host->cqe_on)
3419                 return false;
3420
3421         if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC))
3422                 *cmd_error = -EILSEQ;
3423         else if (intmask & SDHCI_INT_TIMEOUT)
3424                 *cmd_error = -ETIMEDOUT;
3425         else
3426                 *cmd_error = 0;
3427
3428         if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC))
3429                 *data_error = -EILSEQ;
3430         else if (intmask & SDHCI_INT_DATA_TIMEOUT)
3431                 *data_error = -ETIMEDOUT;
3432         else if (intmask & SDHCI_INT_ADMA_ERROR)
3433                 *data_error = -EIO;
3434         else
3435                 *data_error = 0;
3436
3437         /* Clear selected interrupts. */
3438         mask = intmask & host->cqe_ier;
3439         sdhci_writel(host, mask, SDHCI_INT_STATUS);
3440
3441         if (intmask & SDHCI_INT_BUS_POWER)
3442                 pr_err("%s: Card is consuming too much power!\n",
3443                        mmc_hostname(host->mmc));
3444
3445         intmask &= ~(host->cqe_ier | SDHCI_INT_ERROR);
3446         if (intmask) {
3447                 sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3448                 pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n",
3449                        mmc_hostname(host->mmc), intmask);
3450                 sdhci_dumpregs(host);
3451         }
3452
3453         return true;
3454 }
3455 EXPORT_SYMBOL_GPL(sdhci_cqe_irq);
3456
3457 /*****************************************************************************\
3458  *                                                                           *
3459  * Device allocation/registration                                            *
3460  *                                                                           *
3461 \*****************************************************************************/
3462
3463 struct sdhci_host *sdhci_alloc_host(struct device *dev,
3464         size_t priv_size)
3465 {
3466         struct mmc_host *mmc;
3467         struct sdhci_host *host;
3468
3469         WARN_ON(dev == NULL);
3470
3471         mmc = mmc_alloc_host(sizeof(struct sdhci_host) + priv_size, dev);
3472         if (!mmc)
3473                 return ERR_PTR(-ENOMEM);
3474
3475         host = mmc_priv(mmc);
3476         host->mmc = mmc;
3477         host->mmc_host_ops = sdhci_ops;
3478         mmc->ops = &host->mmc_host_ops;
3479
3480         host->flags = SDHCI_SIGNALING_330;
3481
3482         host->cqe_ier     = SDHCI_CQE_INT_MASK;
3483         host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK;
3484
3485         host->tuning_delay = -1;
3486
3487         host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;
3488
3489         /*
3490          * The DMA table descriptor count is calculated as the maximum
3491          * number of segments times 2, to allow for an alignment
3492          * descriptor for each segment, plus 1 for a nop end descriptor.
3493          */
3494         host->adma_table_cnt = SDHCI_MAX_SEGS * 2 + 1;
3495
3496         return host;
3497 }
3498
3499 EXPORT_SYMBOL_GPL(sdhci_alloc_host);
3500
3501 static int sdhci_set_dma_mask(struct sdhci_host *host)
3502 {
3503         struct mmc_host *mmc = host->mmc;
3504         struct device *dev = mmc_dev(mmc);
3505         int ret = -EINVAL;
3506
3507         if (host->quirks2 & SDHCI_QUIRK2_BROKEN_64_BIT_DMA)
3508                 host->flags &= ~SDHCI_USE_64_BIT_DMA;
3509
3510         /* Try 64-bit mask if hardware is capable  of it */
3511         if (host->flags & SDHCI_USE_64_BIT_DMA) {
3512                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
3513                 if (ret) {
3514                         pr_warn("%s: Failed to set 64-bit DMA mask.\n",
3515                                 mmc_hostname(mmc));
3516                         host->flags &= ~SDHCI_USE_64_BIT_DMA;
3517                 }
3518         }
3519
3520         /* 32-bit mask as default & fallback */
3521         if (ret) {
3522                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
3523                 if (ret)
3524                         pr_warn("%s: Failed to set 32-bit DMA mask.\n",
3525                                 mmc_hostname(mmc));
3526         }
3527
3528         return ret;
3529 }
3530
3531 void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
3532 {
3533         u16 v;
3534         u64 dt_caps_mask = 0;
3535         u64 dt_caps = 0;
3536
3537         if (host->read_caps)
3538                 return;
3539
3540         host->read_caps = true;
3541
3542         if (debug_quirks)
3543                 host->quirks = debug_quirks;
3544
3545         if (debug_quirks2)
3546                 host->quirks2 = debug_quirks2;
3547
3548         sdhci_do_reset(host, SDHCI_RESET_ALL);
3549
3550         if (host->v4_mode)
3551                 sdhci_do_enable_v4_mode(host);
3552
3553         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3554                              "sdhci-caps-mask", &dt_caps_mask);
3555         of_property_read_u64(mmc_dev(host->mmc)->of_node,
3556                              "sdhci-caps", &dt_caps);
3557
3558         v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION);
3559         host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;
3560
3561         if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)
3562                 return;
3563
3564         if (caps) {
3565                 host->caps = *caps;
3566         } else {
3567                 host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
3568                 host->caps &= ~lower_32_bits(dt_caps_mask);
3569                 host->caps |= lower_32_bits(dt_caps);
3570         }
3571
3572         if (host->version < SDHCI_SPEC_300)
3573                 return;
3574
3575         if (caps1) {
3576                 host->caps1 = *caps1;
3577         } else {
3578                 host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
3579                 host->caps1 &= ~upper_32_bits(dt_caps_mask);
3580                 host->caps1 |= upper_32_bits(dt_caps);
3581         }
3582 }
3583 EXPORT_SYMBOL_GPL(__sdhci_read_caps);
3584
3585 static void sdhci_allocate_bounce_buffer(struct sdhci_host *host)
3586 {
3587         struct mmc_host *mmc = host->mmc;
3588         unsigned int max_blocks;
3589         unsigned int bounce_size;
3590         int ret;
3591
3592         /*
3593          * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
3594          * has diminishing returns, this is probably because SD/MMC
3595          * cards are usually optimized to handle this size of requests.
3596          */
3597         bounce_size = SZ_64K;
3598         /*
3599          * Adjust downwards to maximum request size if this is less
3600          * than our segment size, else hammer down the maximum
3601          * request size to the maximum buffer size.
3602          */
3603         if (mmc->max_req_size < bounce_size)
3604                 bounce_size = mmc->max_req_size;
3605         max_blocks = bounce_size / 512;
3606
3607         /*
3608          * When we just support one segment, we can get significant
3609          * speedups by the help of a bounce buffer to group scattered
3610          * reads/writes together.
3611          */
3612         host->bounce_buffer = devm_kmalloc(mmc->parent,
3613                                            bounce_size,
3614                                            GFP_KERNEL);
3615         if (!host->bounce_buffer) {
3616                 pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n",
3617                        mmc_hostname(mmc),
3618                        bounce_size);
3619                 /*
3620                  * Exiting with zero here makes sure we proceed with
3621                  * mmc->max_segs == 1.
3622                  */
3623                 return;
3624         }
3625
3626         host->bounce_addr = dma_map_single(mmc->parent,
3627                                            host->bounce_buffer,
3628                                            bounce_size,
3629                                            DMA_BIDIRECTIONAL);
3630         ret = dma_mapping_error(mmc->parent, host->bounce_addr);
3631         if (ret)
3632                 /* Again fall back to max_segs == 1 */
3633                 return;
3634         host->bounce_buffer_size = bounce_size;
3635
3636         /* Lie about this since we're bouncing */
3637         mmc->max_segs = max_blocks;
3638         mmc->max_seg_size = bounce_size;
3639         mmc->max_req_size = bounce_size;
3640
3641         pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n",
3642                 mmc_hostname(mmc), max_blocks, bounce_size);
3643 }
3644
3645 static inline bool sdhci_can_64bit_dma(struct sdhci_host *host)
3646 {
3647         /*
3648          * According to SD Host Controller spec v4.10, bit[27] added from
3649          * version 4.10 in Capabilities Register is used as 64-bit System
3650          * Address support for V4 mode.
3651          */
3652         if (host->version >= SDHCI_SPEC_410 && host->v4_mode)
3653                 return host->caps & SDHCI_CAN_64BIT_V4;
3654
3655         return host->caps & SDHCI_CAN_64BIT;
3656 }
3657
3658 int sdhci_setup_host(struct sdhci_host *host)
3659 {
3660         struct mmc_host *mmc;
3661         u32 max_current_caps;
3662         unsigned int ocr_avail;
3663         unsigned int override_timeout_clk;
3664         u32 max_clk;
3665         int ret;
3666
3667         WARN_ON(host == NULL);
3668         if (host == NULL)
3669                 return -EINVAL;
3670
3671         mmc = host->mmc;
3672
3673         /*
3674          * If there are external regulators, get them. Note this must be done
3675          * early before resetting the host and reading the capabilities so that
3676          * the host can take the appropriate action if regulators are not
3677          * available.
3678          */
3679         ret = mmc_regulator_get_supply(mmc);
3680         if (ret)
3681                 return ret;
3682
3683         DBG("Version:   0x%08x | Present:  0x%08x\n",
3684             sdhci_readw(host, SDHCI_HOST_VERSION),
3685             sdhci_readl(host, SDHCI_PRESENT_STATE));
3686         DBG("Caps:      0x%08x | Caps_1:   0x%08x\n",
3687             sdhci_readl(host, SDHCI_CAPABILITIES),
3688             sdhci_readl(host, SDHCI_CAPABILITIES_1));
3689
3690         sdhci_read_caps(host);
3691
3692         override_timeout_clk = host->timeout_clk;
3693
3694         if (host->version > SDHCI_SPEC_420) {
3695                 pr_err("%s: Unknown controller version (%d). You may experience problems.\n",
3696                        mmc_hostname(mmc), host->version);
3697         }
3698
3699         if (host->quirks & SDHCI_QUIRK_FORCE_DMA)
3700                 host->flags |= SDHCI_USE_SDMA;
3701         else if (!(host->caps & SDHCI_CAN_DO_SDMA))
3702                 DBG("Controller doesn't have SDMA capability\n");
3703         else
3704                 host->flags |= SDHCI_USE_SDMA;
3705
3706         if ((host->quirks & SDHCI_QUIRK_BROKEN_DMA) &&
3707                 (host->flags & SDHCI_USE_SDMA)) {
3708                 DBG("Disabling DMA as it is marked broken\n");
3709                 host->flags &= ~SDHCI_USE_SDMA;
3710         }
3711
3712         if ((host->version >= SDHCI_SPEC_200) &&
3713                 (host->caps & SDHCI_CAN_DO_ADMA2))
3714                 host->flags |= SDHCI_USE_ADMA;
3715
3716         if ((host->quirks & SDHCI_QUIRK_BROKEN_ADMA) &&
3717                 (host->flags & SDHCI_USE_ADMA)) {
3718                 DBG("Disabling ADMA as it is marked broken\n");
3719                 host->flags &= ~SDHCI_USE_ADMA;
3720         }
3721
3722         /*
3723          * It is assumed that a 64-bit capable device has set a 64-bit DMA mask
3724          * and *must* do 64-bit DMA.  A driver has the opportunity to change
3725          * that during the first call to ->enable_dma().  Similarly
3726          * SDHCI_QUIRK2_BROKEN_64_BIT_DMA must be left to the drivers to
3727          * implement.
3728          */
3729         if (sdhci_can_64bit_dma(host))
3730                 host->flags |= SDHCI_USE_64_BIT_DMA;
3731
3732         if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3733                 ret = sdhci_set_dma_mask(host);
3734
3735                 if (!ret && host->ops->enable_dma)
3736                         ret = host->ops->enable_dma(host);
3737
3738                 if (ret) {
3739                         pr_warn("%s: No suitable DMA available - falling back to PIO\n",
3740                                 mmc_hostname(mmc));
3741                         host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);
3742
3743                         ret = 0;
3744                 }
3745         }
3746
3747         /* SDMA does not support 64-bit DMA if v4 mode not set */
3748         if ((host->flags & SDHCI_USE_64_BIT_DMA) && !host->v4_mode)
3749                 host->flags &= ~SDHCI_USE_SDMA;
3750
3751         if (host->flags & SDHCI_USE_ADMA) {
3752                 dma_addr_t dma;
3753                 void *buf;
3754
3755                 if (host->flags & SDHCI_USE_64_BIT_DMA) {
3756                         host->adma_table_sz = host->adma_table_cnt *
3757                                               SDHCI_ADMA2_64_DESC_SZ(host);
3758                         host->desc_sz = SDHCI_ADMA2_64_DESC_SZ(host);
3759                 } else {
3760                         host->adma_table_sz = host->adma_table_cnt *
3761                                               SDHCI_ADMA2_32_DESC_SZ;
3762                         host->desc_sz = SDHCI_ADMA2_32_DESC_SZ;
3763                 }
3764
3765                 host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN;
3766                 /*
3767                  * Use zalloc to zero the reserved high 32-bits of 128-bit
3768                  * descriptors so that they never need to be written.
3769                  */
3770                 buf = dma_alloc_coherent(mmc_dev(mmc),
3771                                          host->align_buffer_sz + host->adma_table_sz,
3772                                          &dma, GFP_KERNEL);
3773                 if (!buf) {
3774                         pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n",
3775                                 mmc_hostname(mmc));
3776                         host->flags &= ~SDHCI_USE_ADMA;
3777                 } else if ((dma + host->align_buffer_sz) &
3778                            (SDHCI_ADMA2_DESC_ALIGN - 1)) {
3779                         pr_warn("%s: unable to allocate aligned ADMA descriptor\n",
3780                                 mmc_hostname(mmc));
3781                         host->flags &= ~SDHCI_USE_ADMA;
3782                         dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3783                                           host->adma_table_sz, buf, dma);
3784                 } else {
3785                         host->align_buffer = buf;
3786                         host->align_addr = dma;
3787
3788                         host->adma_table = buf + host->align_buffer_sz;
3789                         host->adma_addr = dma + host->align_buffer_sz;
3790                 }
3791         }
3792
3793         /*
3794          * If we use DMA, then it's up to the caller to set the DMA
3795          * mask, but PIO does not need the hw shim so we set a new
3796          * mask here in that case.
3797          */
3798         if (!(host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA))) {
3799                 host->dma_mask = DMA_BIT_MASK(64);
3800                 mmc_dev(mmc)->dma_mask = &host->dma_mask;
3801         }
3802
3803         if (host->version >= SDHCI_SPEC_300)
3804                 host->max_clk = (host->caps & SDHCI_CLOCK_V3_BASE_MASK)
3805                         >> SDHCI_CLOCK_BASE_SHIFT;
3806         else
3807                 host->max_clk = (host->caps & SDHCI_CLOCK_BASE_MASK)
3808                         >> SDHCI_CLOCK_BASE_SHIFT;
3809
3810         host->max_clk *= 1000000;
3811         if (host->max_clk == 0 || host->quirks &
3812                         SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN) {
3813                 if (!host->ops->get_max_clock) {
3814                         pr_err("%s: Hardware doesn't specify base clock frequency.\n",
3815                                mmc_hostname(mmc));
3816                         ret = -ENODEV;
3817                         goto undma;
3818                 }
3819                 host->max_clk = host->ops->get_max_clock(host);
3820         }
3821
3822         /*
3823          * In case of Host Controller v3.00, find out whether clock
3824          * multiplier is supported.
3825          */
3826         host->clk_mul = (host->caps1 & SDHCI_CLOCK_MUL_MASK) >>
3827                         SDHCI_CLOCK_MUL_SHIFT;
3828
3829         /*
3830          * In case the value in Clock Multiplier is 0, then programmable
3831          * clock mode is not supported, otherwise the actual clock
3832          * multiplier is one more than the value of Clock Multiplier
3833          * in the Capabilities Register.
3834          */
3835         if (host->clk_mul)
3836                 host->clk_mul += 1;
3837
3838         /*
3839          * Set host parameters.
3840          */
3841         max_clk = host->max_clk;
3842
3843         if (host->ops->get_min_clock)
3844                 mmc->f_min = host->ops->get_min_clock(host);
3845         else if (host->version >= SDHCI_SPEC_300) {
3846                 if (host->clk_mul) {
3847                         mmc->f_min = (host->max_clk * host->clk_mul) / 1024;
3848                         max_clk = host->max_clk * host->clk_mul;
3849                 } else
3850                         mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300;
3851         } else
3852                 mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200;
3853
3854         if (!mmc->f_max || mmc->f_max > max_clk)
3855                 mmc->f_max = max_clk;
3856
3857         if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) {
3858                 host->timeout_clk = (host->caps & SDHCI_TIMEOUT_CLK_MASK) >>
3859                                         SDHCI_TIMEOUT_CLK_SHIFT;
3860
3861                 if (host->caps & SDHCI_TIMEOUT_CLK_UNIT)
3862                         host->timeout_clk *= 1000;
3863
3864                 if (host->timeout_clk == 0) {
3865                         if (!host->ops->get_timeout_clock) {
3866                                 pr_err("%s: Hardware doesn't specify timeout clock frequency.\n",
3867                                         mmc_hostname(mmc));
3868                                 ret = -ENODEV;
3869                                 goto undma;
3870                         }
3871
3872                         host->timeout_clk =
3873                                 DIV_ROUND_UP(host->ops->get_timeout_clock(host),
3874                                              1000);
3875                 }
3876
3877                 if (override_timeout_clk)
3878                         host->timeout_clk = override_timeout_clk;
3879
3880                 mmc->max_busy_timeout = host->ops->get_max_timeout_count ?
3881                         host->ops->get_max_timeout_count(host) : 1 << 27;
3882                 mmc->max_busy_timeout /= host->timeout_clk;
3883         }
3884
3885         if (host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT &&
3886             !host->ops->get_max_timeout_count)
3887                 mmc->max_busy_timeout = 0;
3888
3889         mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23;
3890         mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
3891
3892         if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12)
3893                 host->flags |= SDHCI_AUTO_CMD12;
3894
3895         /*
3896          * For v3 mode, Auto-CMD23 stuff only works in ADMA or PIO.
3897          * For v4 mode, SDMA may use Auto-CMD23 as well.
3898          */
3899         if ((host->version >= SDHCI_SPEC_300) &&
3900             ((host->flags & SDHCI_USE_ADMA) ||
3901              !(host->flags & SDHCI_USE_SDMA) || host->v4_mode) &&
3902              !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) {
3903                 host->flags |= SDHCI_AUTO_CMD23;
3904                 DBG("Auto-CMD23 available\n");
3905         } else {
3906                 DBG("Auto-CMD23 unavailable\n");
3907         }
3908
3909         /*
3910          * A controller may support 8-bit width, but the board itself
3911          * might not have the pins brought out.  Boards that support
3912          * 8-bit width must set "mmc->caps |= MMC_CAP_8_BIT_DATA;" in
3913          * their platform code before calling sdhci_add_host(), and we
3914          * won't assume 8-bit width for hosts without that CAP.
3915          */
3916         if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))
3917                 mmc->caps |= MMC_CAP_4_BIT_DATA;
3918
3919         if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)
3920                 mmc->caps &= ~MMC_CAP_CMD23;
3921
3922         if (host->caps & SDHCI_CAN_DO_HISPD)
3923                 mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;
3924
3925         if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3926             mmc_card_is_removable(mmc) &&
3927             mmc_gpio_get_cd(host->mmc) < 0)
3928                 mmc->caps |= MMC_CAP_NEEDS_POLL;
3929
3930         if (!IS_ERR(mmc->supply.vqmmc)) {
3931                 ret = regulator_enable(mmc->supply.vqmmc);
3932
3933                 /* If vqmmc provides no 1.8V signalling, then there's no UHS */
3934                 if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000,
3935                                                     1950000))
3936                         host->caps1 &= ~(SDHCI_SUPPORT_SDR104 |
3937                                          SDHCI_SUPPORT_SDR50 |
3938                                          SDHCI_SUPPORT_DDR50);
3939
3940                 /* In eMMC case vqmmc might be a fixed 1.8V regulator */
3941                 if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 2700000,
3942                                                     3600000))
3943                         host->flags &= ~SDHCI_SIGNALING_330;
3944
3945                 if (ret) {
3946                         pr_warn("%s: Failed to enable vqmmc regulator: %d\n",
3947                                 mmc_hostname(mmc), ret);
3948                         mmc->supply.vqmmc = ERR_PTR(-EINVAL);
3949                 }
3950         }
3951
3952         if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {
3953                 host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3954                                  SDHCI_SUPPORT_DDR50);
3955                 /*
3956                  * The SDHCI controller in a SoC might support HS200/HS400
3957                  * (indicated using mmc-hs200-1_8v/mmc-hs400-1_8v dt property),
3958                  * but if the board is modeled such that the IO lines are not
3959                  * connected to 1.8v then HS200/HS400 cannot be supported.
3960                  * Disable HS200/HS400 if the board does not have 1.8v connected
3961                  * to the IO lines. (Applicable for other modes in 1.8v)
3962                  */
3963                 mmc->caps2 &= ~(MMC_CAP2_HSX00_1_8V | MMC_CAP2_HS400_ES);
3964                 mmc->caps &= ~(MMC_CAP_1_8V_DDR | MMC_CAP_UHS);
3965         }
3966
3967         /* Any UHS-I mode in caps implies SDR12 and SDR25 support. */
3968         if (host->caps1 & (SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3969                            SDHCI_SUPPORT_DDR50))
3970                 mmc->caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
3971
3972         /* SDR104 supports also implies SDR50 support */
3973         if (host->caps1 & SDHCI_SUPPORT_SDR104) {
3974                 mmc->caps |= MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_SDR50;
3975                 /* SD3.0: SDR104 is supported so (for eMMC) the caps2
3976                  * field can be promoted to support HS200.
3977                  */
3978                 if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_HS200))
3979                         mmc->caps2 |= MMC_CAP2_HS200;
3980         } else if (host->caps1 & SDHCI_SUPPORT_SDR50) {
3981                 mmc->caps |= MMC_CAP_UHS_SDR50;
3982         }
3983
3984         if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400 &&
3985             (host->caps1 & SDHCI_SUPPORT_HS400))
3986                 mmc->caps2 |= MMC_CAP2_HS400;
3987
3988         if ((mmc->caps2 & MMC_CAP2_HSX00_1_2V) &&
3989             (IS_ERR(mmc->supply.vqmmc) ||
3990              !regulator_is_supported_voltage(mmc->supply.vqmmc, 1100000,
3991                                              1300000)))
3992                 mmc->caps2 &= ~MMC_CAP2_HSX00_1_2V;
3993
3994         if ((host->caps1 & SDHCI_SUPPORT_DDR50) &&
3995             !(host->quirks2 & SDHCI_QUIRK2_BROKEN_DDR50))
3996                 mmc->caps |= MMC_CAP_UHS_DDR50;
3997
3998         /* Does the host need tuning for SDR50? */
3999         if (host->caps1 & SDHCI_USE_SDR50_TUNING)
4000                 host->flags |= SDHCI_SDR50_NEEDS_TUNING;
4001
4002         /* Driver Type(s) (A, C, D) supported by the host */
4003         if (host->caps1 & SDHCI_DRIVER_TYPE_A)
4004                 mmc->caps |= MMC_CAP_DRIVER_TYPE_A;
4005         if (host->caps1 & SDHCI_DRIVER_TYPE_C)
4006                 mmc->caps |= MMC_CAP_DRIVER_TYPE_C;
4007         if (host->caps1 & SDHCI_DRIVER_TYPE_D)
4008                 mmc->caps |= MMC_CAP_DRIVER_TYPE_D;
4009
4010         /* Initial value for re-tuning timer count */
4011         host->tuning_count = (host->caps1 & SDHCI_RETUNING_TIMER_COUNT_MASK) >>
4012                              SDHCI_RETUNING_TIMER_COUNT_SHIFT;
4013
4014         /*
4015          * In case Re-tuning Timer is not disabled, the actual value of
4016          * re-tuning timer will be 2 ^ (n - 1).
4017          */
4018         if (host->tuning_count)
4019                 host->tuning_count = 1 << (host->tuning_count - 1);
4020
4021         /* Re-tuning mode supported by the Host Controller */
4022         host->tuning_mode = (host->caps1 & SDHCI_RETUNING_MODE_MASK) >>
4023                              SDHCI_RETUNING_MODE_SHIFT;
4024
4025         ocr_avail = 0;
4026
4027         /*
4028          * According to SD Host Controller spec v3.00, if the Host System
4029          * can afford more than 150mA, Host Driver should set XPC to 1. Also
4030          * the value is meaningful only if Voltage Support in the Capabilities
4031          * register is set. The actual current value is 4 times the register
4032          * value.
4033          */
4034         max_current_caps = sdhci_readl(host, SDHCI_MAX_CURRENT);
4035         if (!max_current_caps && !IS_ERR(mmc->supply.vmmc)) {
4036                 int curr = regulator_get_current_limit(mmc->supply.vmmc);
4037                 if (curr > 0) {
4038
4039                         /* convert to SDHCI_MAX_CURRENT format */
4040                         curr = curr/1000;  /* convert to mA */
4041                         curr = curr/SDHCI_MAX_CURRENT_MULTIPLIER;
4042
4043                         curr = min_t(u32, curr, SDHCI_MAX_CURRENT_LIMIT);
4044                         max_current_caps =
4045                                 (curr << SDHCI_MAX_CURRENT_330_SHIFT) |
4046                                 (curr << SDHCI_MAX_CURRENT_300_SHIFT) |
4047                                 (curr << SDHCI_MAX_CURRENT_180_SHIFT);
4048                 }
4049         }
4050
4051         if (host->caps & SDHCI_CAN_VDD_330) {
4052                 ocr_avail |= MMC_VDD_32_33 | MMC_VDD_33_34;
4053
4054                 mmc->max_current_330 = ((max_current_caps &
4055                                    SDHCI_MAX_CURRENT_330_MASK) >>
4056                                    SDHCI_MAX_CURRENT_330_SHIFT) *
4057                                    SDHCI_MAX_CURRENT_MULTIPLIER;
4058         }
4059         if (host->caps & SDHCI_CAN_VDD_300) {
4060                 ocr_avail |= MMC_VDD_29_30 | MMC_VDD_30_31;
4061
4062                 mmc->max_current_300 = ((max_current_caps &
4063                                    SDHCI_MAX_CURRENT_300_MASK) >>
4064                                    SDHCI_MAX_CURRENT_300_SHIFT) *
4065                                    SDHCI_MAX_CURRENT_MULTIPLIER;
4066         }
4067         if (host->caps & SDHCI_CAN_VDD_180) {
4068                 ocr_avail |= MMC_VDD_165_195;
4069
4070                 mmc->max_current_180 = ((max_current_caps &
4071                                    SDHCI_MAX_CURRENT_180_MASK) >>
4072                                    SDHCI_MAX_CURRENT_180_SHIFT) *
4073                                    SDHCI_MAX_CURRENT_MULTIPLIER;
4074         }
4075
4076         /* If OCR set by host, use it instead. */
4077         if (host->ocr_mask)
4078                 ocr_avail = host->ocr_mask;
4079
4080         /* If OCR set by external regulators, give it highest prio. */
4081         if (mmc->ocr_avail)
4082                 ocr_avail = mmc->ocr_avail;
4083
4084         mmc->ocr_avail = ocr_avail;
4085         mmc->ocr_avail_sdio = ocr_avail;
4086         if (host->ocr_avail_sdio)
4087                 mmc->ocr_avail_sdio &= host->ocr_avail_sdio;
4088         mmc->ocr_avail_sd = ocr_avail;
4089         if (host->ocr_avail_sd)
4090                 mmc->ocr_avail_sd &= host->ocr_avail_sd;
4091         else /* normal SD controllers don't support 1.8V */
4092                 mmc->ocr_avail_sd &= ~MMC_VDD_165_195;
4093         mmc->ocr_avail_mmc = ocr_avail;
4094         if (host->ocr_avail_mmc)
4095                 mmc->ocr_avail_mmc &= host->ocr_avail_mmc;
4096
4097         if (mmc->ocr_avail == 0) {
4098                 pr_err("%s: Hardware doesn't report any support voltages.\n",
4099                        mmc_hostname(mmc));
4100                 ret = -ENODEV;
4101                 goto unreg;
4102         }
4103
4104         if ((mmc->caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
4105                           MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 |
4106                           MMC_CAP_UHS_DDR50 | MMC_CAP_1_8V_DDR)) ||
4107             (mmc->caps2 & (MMC_CAP2_HS200_1_8V_SDR | MMC_CAP2_HS400_1_8V)))
4108                 host->flags |= SDHCI_SIGNALING_180;
4109
4110         if (mmc->caps2 & MMC_CAP2_HSX00_1_2V)
4111                 host->flags |= SDHCI_SIGNALING_120;
4112
4113         spin_lock_init(&host->lock);
4114
4115         /*
4116          * Maximum number of sectors in one transfer. Limited by SDMA boundary
4117          * size (512KiB). Note some tuning modes impose a 4MiB limit, but this
4118          * is less anyway.
4119          */
4120         mmc->max_req_size = 524288;
4121
4122         /*
4123          * Maximum number of segments. Depends on if the hardware
4124          * can do scatter/gather or not.
4125          */
4126         if (host->flags & SDHCI_USE_ADMA) {
4127                 mmc->max_segs = SDHCI_MAX_SEGS;
4128         } else if (host->flags & SDHCI_USE_SDMA) {
4129                 mmc->max_segs = 1;
4130                 if (swiotlb_max_segment()) {
4131                         unsigned int max_req_size = (1 << IO_TLB_SHIFT) *
4132                                                 IO_TLB_SEGSIZE;
4133                         mmc->max_req_size = min(mmc->max_req_size,
4134                                                 max_req_size);
4135                 }
4136         } else { /* PIO */
4137                 mmc->max_segs = SDHCI_MAX_SEGS;
4138         }
4139
4140         /*
4141          * Maximum segment size. Could be one segment with the maximum number
4142          * of bytes. When doing hardware scatter/gather, each entry cannot
4143          * be larger than 64 KiB though.
4144          */
4145         if (host->flags & SDHCI_USE_ADMA) {
4146                 if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC)
4147                         mmc->max_seg_size = 65535;
4148                 else
4149                         mmc->max_seg_size = 65536;
4150         } else {
4151                 mmc->max_seg_size = mmc->max_req_size;
4152         }
4153
4154         /*
4155          * Maximum block size. This varies from controller to controller and
4156          * is specified in the capabilities register.
4157          */
4158         if (host->quirks & SDHCI_QUIRK_FORCE_BLK_SZ_2048) {
4159                 mmc->max_blk_size = 2;
4160         } else {
4161                 mmc->max_blk_size = (host->caps & SDHCI_MAX_BLOCK_MASK) >>
4162                                 SDHCI_MAX_BLOCK_SHIFT;
4163                 if (mmc->max_blk_size >= 3) {
4164                         pr_warn("%s: Invalid maximum block size, assuming 512 bytes\n",
4165                                 mmc_hostname(mmc));
4166                         mmc->max_blk_size = 0;
4167                 }
4168         }
4169
4170         mmc->max_blk_size = 512 << mmc->max_blk_size;
4171
4172         /*
4173          * Maximum block count.
4174          */
4175         mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535;
4176
4177         if (mmc->max_segs == 1)
4178                 /* This may alter mmc->*_blk_* parameters */
4179                 sdhci_allocate_bounce_buffer(host);
4180
4181         return 0;
4182
4183 unreg:
4184         if (!IS_ERR(mmc->supply.vqmmc))
4185                 regulator_disable(mmc->supply.vqmmc);
4186 undma:
4187         if (host->align_buffer)
4188                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4189                                   host->adma_table_sz, host->align_buffer,
4190                                   host->align_addr);
4191         host->adma_table = NULL;
4192         host->align_buffer = NULL;
4193
4194         return ret;
4195 }
4196 EXPORT_SYMBOL_GPL(sdhci_setup_host);
4197
4198 void sdhci_cleanup_host(struct sdhci_host *host)
4199 {
4200         struct mmc_host *mmc = host->mmc;
4201
4202         if (!IS_ERR(mmc->supply.vqmmc))
4203                 regulator_disable(mmc->supply.vqmmc);
4204
4205         if (host->align_buffer)
4206                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4207                                   host->adma_table_sz, host->align_buffer,
4208                                   host->align_addr);
4209         host->adma_table = NULL;
4210         host->align_buffer = NULL;
4211 }
4212 EXPORT_SYMBOL_GPL(sdhci_cleanup_host);
4213
4214 int __sdhci_add_host(struct sdhci_host *host)
4215 {
4216         struct mmc_host *mmc = host->mmc;
4217         int ret;
4218
4219         /*
4220          * Init tasklets.
4221          */
4222         tasklet_init(&host->finish_tasklet,
4223                 sdhci_tasklet_finish, (unsigned long)host);
4224
4225         timer_setup(&host->timer, sdhci_timeout_timer, 0);
4226         timer_setup(&host->data_timer, sdhci_timeout_data_timer, 0);
4227
4228         init_waitqueue_head(&host->buf_ready_int);
4229
4230         sdhci_init(host, 0);
4231
4232         ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq,
4233                                    IRQF_SHARED, mmc_hostname(mmc), host);
4234         if (ret) {
4235                 pr_err("%s: Failed to request IRQ %d: %d\n",
4236                        mmc_hostname(mmc), host->irq, ret);
4237                 goto untasklet;
4238         }
4239
4240         ret = sdhci_led_register(host);
4241         if (ret) {
4242                 pr_err("%s: Failed to register LED device: %d\n",
4243                        mmc_hostname(mmc), ret);
4244                 goto unirq;
4245         }
4246
4247         ret = mmc_add_host(mmc);
4248         if (ret)
4249                 goto unled;
4250
4251         pr_info("%s: SDHCI controller on %s [%s] using %s\n",
4252                 mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)),
4253                 (host->flags & SDHCI_USE_ADMA) ?
4254                 (host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" :
4255                 (host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO");
4256
4257         sdhci_enable_card_detection(host);
4258
4259         return 0;
4260
4261 unled:
4262         sdhci_led_unregister(host);
4263 unirq:
4264         sdhci_do_reset(host, SDHCI_RESET_ALL);
4265         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4266         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4267         free_irq(host->irq, host);
4268 untasklet:
4269         tasklet_kill(&host->finish_tasklet);
4270
4271         return ret;
4272 }
4273 EXPORT_SYMBOL_GPL(__sdhci_add_host);
4274
4275 int sdhci_add_host(struct sdhci_host *host)
4276 {
4277         int ret;
4278
4279         ret = sdhci_setup_host(host);
4280         if (ret)
4281                 return ret;
4282
4283         ret = __sdhci_add_host(host);
4284         if (ret)
4285                 goto cleanup;
4286
4287         return 0;
4288
4289 cleanup:
4290         sdhci_cleanup_host(host);
4291
4292         return ret;
4293 }
4294 EXPORT_SYMBOL_GPL(sdhci_add_host);
4295
4296 void sdhci_remove_host(struct sdhci_host *host, int dead)
4297 {
4298         struct mmc_host *mmc = host->mmc;
4299         unsigned long flags;
4300
4301         if (dead) {
4302                 spin_lock_irqsave(&host->lock, flags);
4303
4304                 host->flags |= SDHCI_DEVICE_DEAD;
4305
4306                 if (sdhci_has_requests(host)) {
4307                         pr_err("%s: Controller removed during "
4308                                 " transfer!\n", mmc_hostname(mmc));
4309                         sdhci_error_out_mrqs(host, -ENOMEDIUM);
4310                 }
4311
4312                 spin_unlock_irqrestore(&host->lock, flags);
4313         }
4314
4315         sdhci_disable_card_detection(host);
4316
4317         mmc_remove_host(mmc);
4318
4319         sdhci_led_unregister(host);
4320
4321         if (!dead)
4322                 sdhci_do_reset(host, SDHCI_RESET_ALL);
4323
4324         sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4325         sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4326         free_irq(host->irq, host);
4327
4328         del_timer_sync(&host->timer);
4329         del_timer_sync(&host->data_timer);
4330
4331         tasklet_kill(&host->finish_tasklet);
4332
4333         if (!IS_ERR(mmc->supply.vqmmc))
4334                 regulator_disable(mmc->supply.vqmmc);
4335
4336         if (host->align_buffer)
4337                 dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4338                                   host->adma_table_sz, host->align_buffer,
4339                                   host->align_addr);
4340
4341         host->adma_table = NULL;
4342         host->align_buffer = NULL;
4343 }
4344
4345 EXPORT_SYMBOL_GPL(sdhci_remove_host);
4346
4347 void sdhci_free_host(struct sdhci_host *host)
4348 {
4349         mmc_free_host(host->mmc);
4350 }
4351
4352 EXPORT_SYMBOL_GPL(sdhci_free_host);
4353
4354 /*****************************************************************************\
4355  *                                                                           *
4356  * Driver init/exit                                                          *
4357  *                                                                           *
4358 \*****************************************************************************/
4359
4360 static int __init sdhci_drv_init(void)
4361 {
4362         pr_info(DRIVER_NAME
4363                 ": Secure Digital Host Controller Interface driver\n");
4364         pr_info(DRIVER_NAME ": Copyright(c) Pierre Ossman\n");
4365
4366         return 0;
4367 }
4368
4369 static void __exit sdhci_drv_exit(void)
4370 {
4371 }
4372
4373 module_init(sdhci_drv_init);
4374 module_exit(sdhci_drv_exit);
4375
4376 module_param(debug_quirks, uint, 0444);
4377 module_param(debug_quirks2, uint, 0444);
4378
4379 MODULE_AUTHOR("Pierre Ossman <pierre@ossman.eu>");
4380 MODULE_DESCRIPTION("Secure Digital Host Controller Interface core driver");
4381 MODULE_LICENSE("GPL");
4382
4383 MODULE_PARM_DESC(debug_quirks, "Force certain quirks.");
4384 MODULE_PARM_DESC(debug_quirks2, "Force certain other quirks.");