drivers/ufs/host/ufs-mediatek.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2019 MediaTek Inc.
   4  * Authors:
   5  *      Stanley Chu <stanley.chu@mediatek.com>
   6  *      Peter Wang <peter.wang@mediatek.com>
   7  */
   8
   9 #include <linux/arm-smccc.h>
  10 #include <linux/bitfield.h>
  11 #include <linux/clk.h>
  12 #include <linux/delay.h>
  13 #include <linux/module.h>
  14 #include <linux/of.h>
  15 #include <linux/of_address.h>
  16 #include <linux/of_device.h>
  17 #include <linux/of_platform.h>
  18 #include <linux/phy/phy.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/pm_qos.h>
  21 #include <linux/regulator/consumer.h>
  22 #include <linux/reset.h>
  23 #include <linux/soc/mediatek/mtk_sip_svc.h>
  24
  25 #include <ufs/ufshcd.h>
  26 #include "ufshcd-pltfrm.h"
  27 #include <ufs/ufs_quirks.h>
  28 #include <ufs/unipro.h>
  29 #include "ufs-mediatek.h"
  30
  31 static int  ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq);
  32
  33 #define CREATE_TRACE_POINTS
  34 #include "ufs-mediatek-trace.h"
  35 #undef CREATE_TRACE_POINTS
  36
  37 #define MAX_SUPP_MAC 64
  38 #define MCQ_QUEUE_OFFSET(c) ((((c) >> 16) & 0xFF) * 0x200)
  39
  40 static const struct ufs_dev_quirk ufs_mtk_dev_fixups[] = {
  41         { .wmanufacturerid = UFS_ANY_VENDOR,
  42           .model = UFS_ANY_MODEL,
  43           .quirk = UFS_DEVICE_QUIRK_DELAY_AFTER_LPM |
  44                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
  45         { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
  46           .model = "H9HQ21AFAMZDAR",
  47           .quirk = UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES },
  48         {}
  49 };
  50
  51 static const struct of_device_id ufs_mtk_of_match[] = {
  52         { .compatible = "mediatek,mt8183-ufshci" },
  53         {},
  54 };
  55
  56 /*
  57  * Details of UIC Errors
  58  */
  59 static const char *const ufs_uic_err_str[] = {
  60         "PHY Adapter Layer",
  61         "Data Link Layer",
  62         "Network Link Layer",
  63         "Transport Link Layer",
  64         "DME"
  65 };
  66
  67 static const char *const ufs_uic_pa_err_str[] = {
  68         "PHY error on Lane 0",
  69         "PHY error on Lane 1",
  70         "PHY error on Lane 2",
  71         "PHY error on Lane 3",
  72         "Generic PHY Adapter Error. This should be the LINERESET indication"
  73 };
  74
  75 static const char *const ufs_uic_dl_err_str[] = {
  76         "NAC_RECEIVED",
  77         "TCx_REPLAY_TIMER_EXPIRED",
  78         "AFCx_REQUEST_TIMER_EXPIRED",
  79         "FCx_PROTECTION_TIMER_EXPIRED",
  80         "CRC_ERROR",
  81         "RX_BUFFER_OVERFLOW",
  82         "MAX_FRAME_LENGTH_EXCEEDED",
  83         "WRONG_SEQUENCE_NUMBER",
  84         "AFC_FRAME_SYNTAX_ERROR",
  85         "NAC_FRAME_SYNTAX_ERROR",
  86         "EOF_SYNTAX_ERROR",
  87         "FRAME_SYNTAX_ERROR",
  88         "BAD_CTRL_SYMBOL_TYPE",
  89         "PA_INIT_ERROR",
  90         "PA_ERROR_IND_RECEIVED",
  91         "PA_INIT"
  92 };
  93
  94 static bool ufs_mtk_is_boost_crypt_enabled(struct ufs_hba *hba)
  95 {
  96         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
  97
  98         return !!(host->caps & UFS_MTK_CAP_BOOST_CRYPT_ENGINE);
  99 }
 100
 101 static bool ufs_mtk_is_va09_supported(struct ufs_hba *hba)
 102 {
 103         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 104
 105         return !!(host->caps & UFS_MTK_CAP_VA09_PWR_CTRL);
 106 }
 107
 108 static bool ufs_mtk_is_broken_vcc(struct ufs_hba *hba)
 109 {
 110         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 111
 112         return !!(host->caps & UFS_MTK_CAP_BROKEN_VCC);
 113 }
 114
 115 static bool ufs_mtk_is_pmc_via_fastauto(struct ufs_hba *hba)
 116 {
 117         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 118
 119         return !!(host->caps & UFS_MTK_CAP_PMC_VIA_FASTAUTO);
 120 }
 121
 122 static void ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable)
 123 {
 124         u32 tmp;
 125
 126         if (enable) {
 127                 ufshcd_dme_get(hba,
 128                                UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
 129                 tmp = tmp |
 130                       (1 << RX_SYMBOL_CLK_GATE_EN) |
 131                       (1 << SYS_CLK_GATE_EN) |
 132                       (1 << TX_CLK_GATE_EN);
 133                 ufshcd_dme_set(hba,
 134                                UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
 135
 136                 ufshcd_dme_get(hba,
 137                                UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp);
 138                 tmp = tmp & ~(1 << TX_SYMBOL_CLK_REQ_FORCE);
 139                 ufshcd_dme_set(hba,
 140                                UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp);
 141         } else {
 142                 ufshcd_dme_get(hba,
 143                                UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
 144                 tmp = tmp & ~((1 << RX_SYMBOL_CLK_GATE_EN) |
 145                               (1 << SYS_CLK_GATE_EN) |
 146                               (1 << TX_CLK_GATE_EN));
 147                 ufshcd_dme_set(hba,
 148                                UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
 149
 150                 ufshcd_dme_get(hba,
 151                                UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp);
 152                 tmp = tmp | (1 << TX_SYMBOL_CLK_REQ_FORCE);
 153                 ufshcd_dme_set(hba,
 154                                UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp);
 155         }
 156 }
 157
 158 static void ufs_mtk_crypto_enable(struct ufs_hba *hba)
 159 {
 160         struct arm_smccc_res res;
 161
 162         ufs_mtk_crypto_ctrl(res, 1);
 163         if (res.a0) {
 164                 dev_info(hba->dev, "%s: crypto enable failed, err: %lu\n",
 165                          __func__, res.a0);
 166                 hba->caps &= ~UFSHCD_CAP_CRYPTO;
 167         }
 168 }
 169
 170 static void ufs_mtk_host_reset(struct ufs_hba *hba)
 171 {
 172         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 173
 174         reset_control_assert(host->hci_reset);
 175         reset_control_assert(host->crypto_reset);
 176         reset_control_assert(host->unipro_reset);
 177
 178         usleep_range(100, 110);
 179
 180         reset_control_deassert(host->unipro_reset);
 181         reset_control_deassert(host->crypto_reset);
 182         reset_control_deassert(host->hci_reset);
 183 }
 184
 185 static void ufs_mtk_init_reset_control(struct ufs_hba *hba,
 186                                        struct reset_control **rc,
 187                                        char *str)
 188 {
 189         *rc = devm_reset_control_get(hba->dev, str);
 190         if (IS_ERR(*rc)) {
 191                 dev_info(hba->dev, "Failed to get reset control %s: %ld\n",
 192                          str, PTR_ERR(*rc));
 193                 *rc = NULL;
 194         }
 195 }
 196
 197 static void ufs_mtk_init_reset(struct ufs_hba *hba)
 198 {
 199         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 200
 201         ufs_mtk_init_reset_control(hba, &host->hci_reset,
 202                                    "hci_rst");
 203         ufs_mtk_init_reset_control(hba, &host->unipro_reset,
 204                                    "unipro_rst");
 205         ufs_mtk_init_reset_control(hba, &host->crypto_reset,
 206                                    "crypto_rst");
 207 }
 208
 209 static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba,
 210                                      enum ufs_notify_change_status status)
 211 {
 212         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 213
 214         if (status == PRE_CHANGE) {
 215                 if (host->unipro_lpm) {
 216                         hba->vps->hba_enable_delay_us = 0;
 217                 } else {
 218                         hba->vps->hba_enable_delay_us = 600;
 219                         ufs_mtk_host_reset(hba);
 220                 }
 221
 222                 if (hba->caps & UFSHCD_CAP_CRYPTO)
 223                         ufs_mtk_crypto_enable(hba);
 224
 225                 if (host->caps & UFS_MTK_CAP_DISABLE_AH8) {
 226                         ufshcd_writel(hba, 0,
 227                                       REG_AUTO_HIBERNATE_IDLE_TIMER);
 228                         hba->capabilities &= ~MASK_AUTO_HIBERN8_SUPPORT;
 229                         hba->ahit = 0;
 230                 }
 231
 232                 /*
 233                  * Turn on CLK_CG early to bypass abnormal ERR_CHK signal
 234                  * to prevent host hang issue
 235                  */
 236                 ufshcd_writel(hba,
 237                               ufshcd_readl(hba, REG_UFS_XOUFS_CTRL) | 0x80,
 238                               REG_UFS_XOUFS_CTRL);
 239         }
 240
 241         return 0;
 242 }
 243
 244 static int ufs_mtk_bind_mphy(struct ufs_hba *hba)
 245 {
 246         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 247         struct device *dev = hba->dev;
 248         struct device_node *np = dev->of_node;
 249         int err = 0;
 250
 251         host->mphy = devm_of_phy_get_by_index(dev, np, 0);
 252
 253         if (host->mphy == ERR_PTR(-EPROBE_DEFER)) {
 254                 /*
 255                  * UFS driver might be probed before the phy driver does.
 256                  * In that case we would like to return EPROBE_DEFER code.
 257                  */
 258                 err = -EPROBE_DEFER;
 259                 dev_info(dev,
 260                          "%s: required phy hasn't probed yet. err = %d\n",
 261                         __func__, err);
 262         } else if (IS_ERR(host->mphy)) {
 263                 err = PTR_ERR(host->mphy);
 264                 if (err != -ENODEV) {
 265                         dev_info(dev, "%s: PHY get failed %d\n", __func__,
 266                                  err);
 267                 }
 268         }
 269
 270         if (err)
 271                 host->mphy = NULL;
 272         /*
 273          * Allow unbound mphy because not every platform needs specific
 274          * mphy control.
 275          */
 276         if (err == -ENODEV)
 277                 err = 0;
 278
 279         return err;
 280 }
 281
 282 static int ufs_mtk_setup_ref_clk(struct ufs_hba *hba, bool on)
 283 {
 284         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 285         struct arm_smccc_res res;
 286         ktime_t timeout, time_checked;
 287         u32 value;
 288
 289         if (host->ref_clk_enabled == on)
 290                 return 0;
 291
 292         ufs_mtk_ref_clk_notify(on, PRE_CHANGE, res);
 293
 294         if (on) {
 295                 ufshcd_writel(hba, REFCLK_REQUEST, REG_UFS_REFCLK_CTRL);
 296         } else {
 297                 ufshcd_delay_us(host->ref_clk_gating_wait_us, 10);
 298                 ufshcd_writel(hba, REFCLK_RELEASE, REG_UFS_REFCLK_CTRL);
 299         }
 300
 301         /* Wait for ack */
 302         timeout = ktime_add_us(ktime_get(), REFCLK_REQ_TIMEOUT_US);
 303         do {
 304                 time_checked = ktime_get();
 305                 value = ufshcd_readl(hba, REG_UFS_REFCLK_CTRL);
 306
 307                 /* Wait until ack bit equals to req bit */
 308                 if (((value & REFCLK_ACK) >> 1) == (value & REFCLK_REQUEST))
 309                         goto out;
 310
 311                 usleep_range(100, 200);
 312         } while (ktime_before(time_checked, timeout));
 313
 314         dev_err(hba->dev, "missing ack of refclk req, reg: 0x%x\n", value);
 315
 316         ufs_mtk_ref_clk_notify(host->ref_clk_enabled, POST_CHANGE, res);
 317
 318         return -ETIMEDOUT;
 319
 320 out:
 321         host->ref_clk_enabled = on;
 322         if (on)
 323                 ufshcd_delay_us(host->ref_clk_ungating_wait_us, 10);
 324
 325         ufs_mtk_ref_clk_notify(on, POST_CHANGE, res);
 326
 327         return 0;
 328 }
 329
 330 static void ufs_mtk_setup_ref_clk_wait_us(struct ufs_hba *hba,
 331                                           u16 gating_us)
 332 {
 333         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 334
 335         if (hba->dev_info.clk_gating_wait_us) {
 336                 host->ref_clk_gating_wait_us =
 337                         hba->dev_info.clk_gating_wait_us;
 338         } else {
 339                 host->ref_clk_gating_wait_us = gating_us;
 340         }
 341
 342         host->ref_clk_ungating_wait_us = REFCLK_DEFAULT_WAIT_US;
 343 }
 344
 345 static void ufs_mtk_dbg_sel(struct ufs_hba *hba)
 346 {
 347         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 348
 349         if (((host->ip_ver >> 16) & 0xFF) >= 0x36) {
 350                 ufshcd_writel(hba, 0x820820, REG_UFS_DEBUG_SEL);
 351                 ufshcd_writel(hba, 0x0, REG_UFS_DEBUG_SEL_B0);
 352                 ufshcd_writel(hba, 0x55555555, REG_UFS_DEBUG_SEL_B1);
 353                 ufshcd_writel(hba, 0xaaaaaaaa, REG_UFS_DEBUG_SEL_B2);
 354                 ufshcd_writel(hba, 0xffffffff, REG_UFS_DEBUG_SEL_B3);
 355         } else {
 356                 ufshcd_writel(hba, 0x20, REG_UFS_DEBUG_SEL);
 357         }
 358 }
 359
 360 static void ufs_mtk_wait_idle_state(struct ufs_hba *hba,
 361                             unsigned long retry_ms)
 362 {
 363         u64 timeout, time_checked;
 364         u32 val, sm;
 365         bool wait_idle;
 366
 367         /* cannot use plain ktime_get() in suspend */
 368         timeout = ktime_get_mono_fast_ns() + retry_ms * 1000000UL;
 369
 370         /* wait a specific time after check base */
 371         udelay(10);
 372         wait_idle = false;
 373
 374         do {
 375                 time_checked = ktime_get_mono_fast_ns();
 376                 ufs_mtk_dbg_sel(hba);
 377                 val = ufshcd_readl(hba, REG_UFS_PROBE);
 378
 379                 sm = val & 0x1f;
 380
 381                 /*
 382                  * if state is in H8 enter and H8 enter confirm
 383                  * wait until return to idle state.
 384                  */
 385                 if ((sm >= VS_HIB_ENTER) && (sm <= VS_HIB_EXIT)) {
 386                         wait_idle = true;
 387                         udelay(50);
 388                         continue;
 389                 } else if (!wait_idle)
 390                         break;
 391
 392                 if (wait_idle && (sm == VS_HCE_BASE))
 393                         break;
 394         } while (time_checked < timeout);
 395
 396         if (wait_idle && sm != VS_HCE_BASE)
 397                 dev_info(hba->dev, "wait idle tmo: 0x%x\n", val);
 398 }
 399
 400 static int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 state,
 401                                    unsigned long max_wait_ms)
 402 {
 403         ktime_t timeout, time_checked;
 404         u32 val;
 405
 406         timeout = ktime_add_ms(ktime_get(), max_wait_ms);
 407         do {
 408                 time_checked = ktime_get();
 409                 ufs_mtk_dbg_sel(hba);
 410                 val = ufshcd_readl(hba, REG_UFS_PROBE);
 411                 val = val >> 28;
 412
 413                 if (val == state)
 414                         return 0;
 415
 416                 /* Sleep for max. 200us */
 417                 usleep_range(100, 200);
 418         } while (ktime_before(time_checked, timeout));
 419
 420         return -ETIMEDOUT;
 421 }
 422
 423 static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on)
 424 {
 425         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 426         struct phy *mphy = host->mphy;
 427         struct arm_smccc_res res;
 428         int ret = 0;
 429
 430         if (!mphy || !(on ^ host->mphy_powered_on))
 431                 return 0;
 432
 433         if (on) {
 434                 if (ufs_mtk_is_va09_supported(hba)) {
 435                         ret = regulator_enable(host->reg_va09);
 436                         if (ret < 0)
 437                                 goto out;
 438                         /* wait 200 us to stablize VA09 */
 439                         usleep_range(200, 210);
 440                         ufs_mtk_va09_pwr_ctrl(res, 1);
 441                 }
 442                 phy_power_on(mphy);
 443         } else {
 444                 phy_power_off(mphy);
 445                 if (ufs_mtk_is_va09_supported(hba)) {
 446                         ufs_mtk_va09_pwr_ctrl(res, 0);
 447                         ret = regulator_disable(host->reg_va09);
 448                 }
 449         }
 450 out:
 451         if (ret) {
 452                 dev_info(hba->dev,
 453                          "failed to %s va09: %d\n",
 454                          on ? "enable" : "disable",
 455                          ret);
 456         } else {
 457                 host->mphy_powered_on = on;
 458         }
 459
 460         return ret;
 461 }
 462
 463 static int ufs_mtk_get_host_clk(struct device *dev, const char *name,
 464                                 struct clk **clk_out)
 465 {
 466         struct clk *clk;
 467         int err = 0;
 468
 469         clk = devm_clk_get(dev, name);
 470         if (IS_ERR(clk))
 471                 err = PTR_ERR(clk);
 472         else
 473                 *clk_out = clk;
 474
 475         return err;
 476 }
 477
 478 static void ufs_mtk_boost_crypt(struct ufs_hba *hba, bool boost)
 479 {
 480         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 481         struct ufs_mtk_crypt_cfg *cfg;
 482         struct regulator *reg;
 483         int volt, ret;
 484
 485         if (!ufs_mtk_is_boost_crypt_enabled(hba))
 486                 return;
 487
 488         cfg = host->crypt;
 489         volt = cfg->vcore_volt;
 490         reg = cfg->reg_vcore;
 491
 492         ret = clk_prepare_enable(cfg->clk_crypt_mux);
 493         if (ret) {
 494                 dev_info(hba->dev, "clk_prepare_enable(): %d\n",
 495                          ret);
 496                 return;
 497         }
 498
 499         if (boost) {
 500                 ret = regulator_set_voltage(reg, volt, INT_MAX);
 501                 if (ret) {
 502                         dev_info(hba->dev,
 503                                  "failed to set vcore to %d\n", volt);
 504                         goto out;
 505                 }
 506
 507                 ret = clk_set_parent(cfg->clk_crypt_mux,
 508                                      cfg->clk_crypt_perf);
 509                 if (ret) {
 510                         dev_info(hba->dev,
 511                                  "failed to set clk_crypt_perf\n");
 512                         regulator_set_voltage(reg, 0, INT_MAX);
 513                         goto out;
 514                 }
 515         } else {
 516                 ret = clk_set_parent(cfg->clk_crypt_mux,
 517                                      cfg->clk_crypt_lp);
 518                 if (ret) {
 519                         dev_info(hba->dev,
 520                                  "failed to set clk_crypt_lp\n");
 521                         goto out;
 522                 }
 523
 524                 ret = regulator_set_voltage(reg, 0, INT_MAX);
 525                 if (ret) {
 526                         dev_info(hba->dev,
 527                                  "failed to set vcore to MIN\n");
 528                 }
 529         }
 530 out:
 531         clk_disable_unprepare(cfg->clk_crypt_mux);
 532 }
 533
 534 static int ufs_mtk_init_host_clk(struct ufs_hba *hba, const char *name,
 535                                  struct clk **clk)
 536 {
 537         int ret;
 538
 539         ret = ufs_mtk_get_host_clk(hba->dev, name, clk);
 540         if (ret) {
 541                 dev_info(hba->dev, "%s: failed to get %s: %d", __func__,
 542                          name, ret);
 543         }
 544
 545         return ret;
 546 }
 547
 548 static void ufs_mtk_init_boost_crypt(struct ufs_hba *hba)
 549 {
 550         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 551         struct ufs_mtk_crypt_cfg *cfg;
 552         struct device *dev = hba->dev;
 553         struct regulator *reg;
 554         u32 volt;
 555
 556         host->crypt = devm_kzalloc(dev, sizeof(*(host->crypt)),
 557                                    GFP_KERNEL);
 558         if (!host->crypt)
 559                 goto disable_caps;
 560
 561         reg = devm_regulator_get_optional(dev, "dvfsrc-vcore");
 562         if (IS_ERR(reg)) {
 563                 dev_info(dev, "failed to get dvfsrc-vcore: %ld",
 564                          PTR_ERR(reg));
 565                 goto disable_caps;
 566         }
 567
 568         if (of_property_read_u32(dev->of_node, "boost-crypt-vcore-min",
 569                                  &volt)) {
 570                 dev_info(dev, "failed to get boost-crypt-vcore-min");
 571                 goto disable_caps;
 572         }
 573
 574         cfg = host->crypt;
 575         if (ufs_mtk_init_host_clk(hba, "crypt_mux",
 576                                   &cfg->clk_crypt_mux))
 577                 goto disable_caps;
 578
 579         if (ufs_mtk_init_host_clk(hba, "crypt_lp",
 580                                   &cfg->clk_crypt_lp))
 581                 goto disable_caps;
 582
 583         if (ufs_mtk_init_host_clk(hba, "crypt_perf",
 584                                   &cfg->clk_crypt_perf))
 585                 goto disable_caps;
 586
 587         cfg->reg_vcore = reg;
 588         cfg->vcore_volt = volt;
 589         host->caps |= UFS_MTK_CAP_BOOST_CRYPT_ENGINE;
 590
 591 disable_caps:
 592         return;
 593 }
 594
 595 static void ufs_mtk_init_va09_pwr_ctrl(struct ufs_hba *hba)
 596 {
 597         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 598
 599         host->reg_va09 = regulator_get(hba->dev, "va09");
 600         if (IS_ERR(host->reg_va09))
 601                 dev_info(hba->dev, "failed to get va09");
 602         else
 603                 host->caps |= UFS_MTK_CAP_VA09_PWR_CTRL;
 604 }
 605
 606 static void ufs_mtk_init_host_caps(struct ufs_hba *hba)
 607 {
 608         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 609         struct device_node *np = hba->dev->of_node;
 610
 611         if (of_property_read_bool(np, "mediatek,ufs-boost-crypt"))
 612                 ufs_mtk_init_boost_crypt(hba);
 613
 614         if (of_property_read_bool(np, "mediatek,ufs-support-va09"))
 615                 ufs_mtk_init_va09_pwr_ctrl(hba);
 616
 617         if (of_property_read_bool(np, "mediatek,ufs-disable-ah8"))
 618                 host->caps |= UFS_MTK_CAP_DISABLE_AH8;
 619
 620         if (of_property_read_bool(np, "mediatek,ufs-broken-vcc"))
 621                 host->caps |= UFS_MTK_CAP_BROKEN_VCC;
 622
 623         if (of_property_read_bool(np, "mediatek,ufs-pmc-via-fastauto"))
 624                 host->caps |= UFS_MTK_CAP_PMC_VIA_FASTAUTO;
 625
 626         dev_info(hba->dev, "caps: 0x%x", host->caps);
 627 }
 628
 629 static void ufs_mtk_boost_pm_qos(struct ufs_hba *hba, bool boost)
 630 {
 631         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 632
 633         if (!host || !host->pm_qos_init)
 634                 return;
 635
 636         cpu_latency_qos_update_request(&host->pm_qos_req,
 637                                        boost ? 0 : PM_QOS_DEFAULT_VALUE);
 638 }
 639
 640 static void ufs_mtk_scale_perf(struct ufs_hba *hba, bool scale_up)
 641 {
 642         ufs_mtk_boost_crypt(hba, scale_up);
 643         ufs_mtk_boost_pm_qos(hba, scale_up);
 644 }
 645
 646 static void ufs_mtk_pwr_ctrl(struct ufs_hba *hba, bool on)
 647 {
 648         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 649
 650         if (on) {
 651                 phy_power_on(host->mphy);
 652                 ufs_mtk_setup_ref_clk(hba, on);
 653                 if (!ufshcd_is_clkscaling_supported(hba))
 654                         ufs_mtk_scale_perf(hba, on);
 655         } else {
 656                 if (!ufshcd_is_clkscaling_supported(hba))
 657                         ufs_mtk_scale_perf(hba, on);
 658                 ufs_mtk_setup_ref_clk(hba, on);
 659                 phy_power_off(host->mphy);
 660         }
 661 }
 662
 663 /**
 664  * ufs_mtk_setup_clocks - enables/disable clocks
 665  * @hba: host controller instance
 666  * @on: If true, enable clocks else disable them.
 667  * @status: PRE_CHANGE or POST_CHANGE notify
 668  *
 669  * Return: 0 on success, non-zero on failure.
 670  */
 671 static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on,
 672                                 enum ufs_notify_change_status status)
 673 {
 674         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 675         bool clk_pwr_off = false;
 676         int ret = 0;
 677
 678         /*
 679          * In case ufs_mtk_init() is not yet done, simply ignore.
 680          * This ufs_mtk_setup_clocks() shall be called from
 681          * ufs_mtk_init() after init is done.
 682          */
 683         if (!host)
 684                 return 0;
 685
 686         if (!on && status == PRE_CHANGE) {
 687                 if (ufshcd_is_link_off(hba)) {
 688                         clk_pwr_off = true;
 689                 } else if (ufshcd_is_link_hibern8(hba) ||
 690                          (!ufshcd_can_hibern8_during_gating(hba) &&
 691                          ufshcd_is_auto_hibern8_enabled(hba))) {
 692                         /*
 693                          * Gate ref-clk and poweroff mphy if link state is in
 694                          * OFF or Hibern8 by either Auto-Hibern8 or
 695                          * ufshcd_link_state_transition().
 696                          */
 697                         ret = ufs_mtk_wait_link_state(hba,
 698                                                       VS_LINK_HIBERN8,
 699                                                       15);
 700                         if (!ret)
 701                                 clk_pwr_off = true;
 702                 }
 703
 704                 if (clk_pwr_off)
 705                         ufs_mtk_pwr_ctrl(hba, false);
 706         } else if (on && status == POST_CHANGE) {
 707                 ufs_mtk_pwr_ctrl(hba, true);
 708         }
 709
 710         return ret;
 711 }
 712
 713 static void ufs_mtk_get_controller_version(struct ufs_hba *hba)
 714 {
 715         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 716         int ret, ver = 0;
 717
 718         if (host->hw_ver.major)
 719                 return;
 720
 721         /* Set default (minimum) version anyway */
 722         host->hw_ver.major = 2;
 723
 724         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), &ver);
 725         if (!ret) {
 726                 if (ver >= UFS_UNIPRO_VER_1_8) {
 727                         host->hw_ver.major = 3;
 728                         /*
 729                          * Fix HCI version for some platforms with
 730                          * incorrect version
 731                          */
 732                         if (hba->ufs_version < ufshci_version(3, 0))
 733                                 hba->ufs_version = ufshci_version(3, 0);
 734                 }
 735         }
 736 }
 737
 738 static u32 ufs_mtk_get_ufs_hci_version(struct ufs_hba *hba)
 739 {
 740         return hba->ufs_version;
 741 }
 742
 743 /**
 744  * ufs_mtk_init_clocks - Init mtk driver private clocks
 745  *
 746  * @hba: per adapter instance
 747  */
 748 static void ufs_mtk_init_clocks(struct ufs_hba *hba)
 749 {
 750         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 751         struct list_head *head = &hba->clk_list_head;
 752         struct ufs_mtk_clk *mclk = &host->mclk;
 753         struct ufs_clk_info *clki, *clki_tmp;
 754
 755         /*
 756          * Find private clocks and store them in struct ufs_mtk_clk.
 757          * Remove "ufs_sel_min_src" and "ufs_sel_min_src" from list to avoid
 758          * being switched on/off in clock gating.
 759          */
 760         list_for_each_entry_safe(clki, clki_tmp, head, list) {
 761                 if (!strcmp(clki->name, "ufs_sel")) {
 762                         host->mclk.ufs_sel_clki = clki;
 763                 } else if (!strcmp(clki->name, "ufs_sel_max_src")) {
 764                         host->mclk.ufs_sel_max_clki = clki;
 765                         clk_disable_unprepare(clki->clk);
 766                         list_del(&clki->list);
 767                 } else if (!strcmp(clki->name, "ufs_sel_min_src")) {
 768                         host->mclk.ufs_sel_min_clki = clki;
 769                         clk_disable_unprepare(clki->clk);
 770                         list_del(&clki->list);
 771                 }
 772         }
 773
 774         if (!mclk->ufs_sel_clki || !mclk->ufs_sel_max_clki ||
 775             !mclk->ufs_sel_min_clki) {
 776                 hba->caps &= ~UFSHCD_CAP_CLK_SCALING;
 777                 dev_info(hba->dev,
 778                          "%s: Clk-scaling not ready. Feature disabled.",
 779                          __func__);
 780         }
 781 }
 782
 783 #define MAX_VCC_NAME 30
 784 static int ufs_mtk_vreg_fix_vcc(struct ufs_hba *hba)
 785 {
 786         struct ufs_vreg_info *info = &hba->vreg_info;
 787         struct device_node *np = hba->dev->of_node;
 788         struct device *dev = hba->dev;
 789         char vcc_name[MAX_VCC_NAME];
 790         struct arm_smccc_res res;
 791         int err, ver;
 792
 793         if (hba->vreg_info.vcc)
 794                 return 0;
 795
 796         if (of_property_read_bool(np, "mediatek,ufs-vcc-by-num")) {
 797                 ufs_mtk_get_vcc_num(res);
 798                 if (res.a1 > UFS_VCC_NONE && res.a1 < UFS_VCC_MAX)
 799                         snprintf(vcc_name, MAX_VCC_NAME, "vcc-opt%lu", res.a1);
 800                 else
 801                         return -ENODEV;
 802         } else if (of_property_read_bool(np, "mediatek,ufs-vcc-by-ver")) {
 803                 ver = (hba->dev_info.wspecversion & 0xF00) >> 8;
 804                 snprintf(vcc_name, MAX_VCC_NAME, "vcc-ufs%u", ver);
 805         } else {
 806                 return 0;
 807         }
 808
 809         err = ufshcd_populate_vreg(dev, vcc_name, &info->vcc, false);
 810         if (err)
 811                 return err;
 812
 813         err = ufshcd_get_vreg(dev, info->vcc);
 814         if (err)
 815                 return err;
 816
 817         err = regulator_enable(info->vcc->reg);
 818         if (!err) {
 819                 info->vcc->enabled = true;
 820                 dev_info(dev, "%s: %s enabled\n", __func__, vcc_name);
 821         }
 822
 823         return err;
 824 }
 825
 826 static void ufs_mtk_vreg_fix_vccqx(struct ufs_hba *hba)
 827 {
 828         struct ufs_vreg_info *info = &hba->vreg_info;
 829         struct ufs_vreg **vreg_on, **vreg_off;
 830
 831         if (hba->dev_info.wspecversion >= 0x0300) {
 832                 vreg_on = &info->vccq;
 833                 vreg_off = &info->vccq2;
 834         } else {
 835                 vreg_on = &info->vccq2;
 836                 vreg_off = &info->vccq;
 837         }
 838
 839         if (*vreg_on)
 840                 (*vreg_on)->always_on = true;
 841
 842         if (*vreg_off) {
 843                 regulator_disable((*vreg_off)->reg);
 844                 devm_kfree(hba->dev, (*vreg_off)->name);
 845                 devm_kfree(hba->dev, *vreg_off);
 846                 *vreg_off = NULL;
 847         }
 848 }
 849
 850 static void ufs_mtk_init_mcq_irq(struct ufs_hba *hba)
 851 {
 852         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 853         struct platform_device *pdev;
 854         int i;
 855         int irq;
 856
 857         host->mcq_nr_intr = UFSHCD_MAX_Q_NR;
 858         pdev = container_of(hba->dev, struct platform_device, dev);
 859
 860         for (i = 0; i < host->mcq_nr_intr; i++) {
 861                 /* irq index 0 is legacy irq, sq/cq irq start from index 1 */
 862                 irq = platform_get_irq(pdev, i + 1);
 863                 if (irq < 0) {
 864                         host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ;
 865                         goto failed;
 866                 }
 867                 host->mcq_intr_info[i].hba = hba;
 868                 host->mcq_intr_info[i].irq = irq;
 869                 dev_info(hba->dev, "get platform mcq irq: %d, %d\n", i, irq);
 870         }
 871
 872         return;
 873 failed:
 874        /* invalidate irq info */
 875         for (i = 0; i < host->mcq_nr_intr; i++)
 876                 host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ;
 877
 878         host->mcq_nr_intr = 0;
 879 }
 880
 881 /**
 882  * ufs_mtk_init - find other essential mmio bases
 883  * @hba: host controller instance
 884  *
 885  * Binds PHY with controller and powers up PHY enabling clocks
 886  * and regulators.
 887  *
 888  * Return: -EPROBE_DEFER if binding fails, returns negative error
 889  * on phy power up failure and returns zero on success.
 890  */
 891 static int ufs_mtk_init(struct ufs_hba *hba)
 892 {
 893         const struct of_device_id *id;
 894         struct device *dev = hba->dev;
 895         struct ufs_mtk_host *host;
 896         int err = 0;
 897
 898         host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
 899         if (!host) {
 900                 err = -ENOMEM;
 901                 dev_info(dev, "%s: no memory for mtk ufs host\n", __func__);
 902                 goto out;
 903         }
 904
 905         host->hba = hba;
 906         ufshcd_set_variant(hba, host);
 907
 908         id = of_match_device(ufs_mtk_of_match, dev);
 909         if (!id) {
 910                 err = -EINVAL;
 911                 goto out;
 912         }
 913
 914         /* Initialize host capability */
 915         ufs_mtk_init_host_caps(hba);
 916
 917         ufs_mtk_init_mcq_irq(hba);
 918
 919         err = ufs_mtk_bind_mphy(hba);
 920         if (err)
 921                 goto out_variant_clear;
 922
 923         ufs_mtk_init_reset(hba);
 924
 925         /* Enable runtime autosuspend */
 926         hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND;
 927
 928         /* Enable clock-gating */
 929         hba->caps |= UFSHCD_CAP_CLK_GATING;
 930
 931         /* Enable inline encryption */
 932         hba->caps |= UFSHCD_CAP_CRYPTO;
 933
 934         /* Enable WriteBooster */
 935         hba->caps |= UFSHCD_CAP_WB_EN;
 936
 937         /* Enable clk scaling*/
 938         hba->caps |= UFSHCD_CAP_CLK_SCALING;
 939
 940         hba->quirks |= UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL;
 941         hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_INTR;
 942         hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_RTC;
 943         hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80);
 944
 945         if (host->caps & UFS_MTK_CAP_DISABLE_AH8)
 946                 hba->caps |= UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
 947
 948         ufs_mtk_init_clocks(hba);
 949
 950         /*
 951          * ufshcd_vops_init() is invoked after
 952          * ufshcd_setup_clock(true) in ufshcd_hba_init() thus
 953          * phy clock setup is skipped.
 954          *
 955          * Enable phy clocks specifically here.
 956          */
 957         ufs_mtk_mphy_power_on(hba, true);
 958         ufs_mtk_setup_clocks(hba, true, POST_CHANGE);
 959
 960         host->ip_ver = ufshcd_readl(hba, REG_UFS_MTK_IP_VER);
 961
 962         /* Initialize pm-qos request */
 963         cpu_latency_qos_add_request(&host->pm_qos_req, PM_QOS_DEFAULT_VALUE);
 964         host->pm_qos_init = true;
 965
 966         goto out;
 967
 968 out_variant_clear:
 969         ufshcd_set_variant(hba, NULL);
 970 out:
 971         return err;
 972 }
 973
 974 static bool ufs_mtk_pmc_via_fastauto(struct ufs_hba *hba,
 975                                      struct ufs_pa_layer_attr *dev_req_params)
 976 {
 977         if (!ufs_mtk_is_pmc_via_fastauto(hba))
 978                 return false;
 979
 980         if (dev_req_params->hs_rate == hba->pwr_info.hs_rate)
 981                 return false;
 982
 983         if (dev_req_params->pwr_tx != FAST_MODE &&
 984             dev_req_params->gear_tx < UFS_HS_G4)
 985                 return false;
 986
 987         if (dev_req_params->pwr_rx != FAST_MODE &&
 988             dev_req_params->gear_rx < UFS_HS_G4)
 989                 return false;
 990
 991         return true;
 992 }
 993
 994 static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba,
 995                                   struct ufs_pa_layer_attr *dev_max_params,
 996                                   struct ufs_pa_layer_attr *dev_req_params)
 997 {
 998         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 999         struct ufs_host_params host_params;
1000         int ret;
1001
1002         ufshcd_init_host_params(&host_params);
1003         host_params.hs_rx_gear = UFS_HS_G5;
1004         host_params.hs_tx_gear = UFS_HS_G5;
1005
1006         ret = ufshcd_negotiate_pwr_params(&host_params, dev_max_params, dev_req_params);
1007         if (ret) {
1008                 pr_info("%s: failed to determine capabilities\n",
1009                         __func__);
1010         }
1011
1012         if (ufs_mtk_pmc_via_fastauto(hba, dev_req_params)) {
1013                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
1014                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), UFS_HS_G1);
1015
1016                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true);
1017                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), UFS_HS_G1);
1018
1019                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
1020                                dev_req_params->lane_tx);
1021                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
1022                                dev_req_params->lane_rx);
1023                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
1024                                dev_req_params->hs_rate);
1025
1026                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE),
1027                                PA_NO_ADAPT);
1028
1029                 ret = ufshcd_uic_change_pwr_mode(hba,
1030                                         FASTAUTO_MODE << 4 | FASTAUTO_MODE);
1031
1032                 if (ret) {
1033                         dev_err(hba->dev, "%s: HSG1B FASTAUTO failed ret=%d\n",
1034                                 __func__, ret);
1035                 }
1036         }
1037
1038         if (host->hw_ver.major >= 3) {
1039                 ret = ufshcd_dme_configure_adapt(hba,
1040                                            dev_req_params->gear_tx,
1041                                            PA_INITIAL_ADAPT);
1042         }
1043
1044         return ret;
1045 }
1046
1047 static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba,
1048                                      enum ufs_notify_change_status stage,
1049                                      struct ufs_pa_layer_attr *dev_max_params,
1050                                      struct ufs_pa_layer_attr *dev_req_params)
1051 {
1052         int ret = 0;
1053
1054         switch (stage) {
1055         case PRE_CHANGE:
1056                 ret = ufs_mtk_pre_pwr_change(hba, dev_max_params,
1057                                              dev_req_params);
1058                 break;
1059         case POST_CHANGE:
1060                 break;
1061         default:
1062                 ret = -EINVAL;
1063                 break;
1064         }
1065
1066         return ret;
1067 }
1068
1069 static int ufs_mtk_unipro_set_lpm(struct ufs_hba *hba, bool lpm)
1070 {
1071         int ret;
1072         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1073
1074         ret = ufshcd_dme_set(hba,
1075                              UIC_ARG_MIB_SEL(VS_UNIPROPOWERDOWNCONTROL, 0),
1076                              lpm ? 1 : 0);
1077         if (!ret || !lpm) {
1078                 /*
1079                  * Forcibly set as non-LPM mode if UIC commands is failed
1080                  * to use default hba_enable_delay_us value for re-enabling
1081                  * the host.
1082                  */
1083                 host->unipro_lpm = lpm;
1084         }
1085
1086         return ret;
1087 }
1088
1089 static int ufs_mtk_pre_link(struct ufs_hba *hba)
1090 {
1091         int ret;
1092         u32 tmp;
1093
1094         ufs_mtk_get_controller_version(hba);
1095
1096         ret = ufs_mtk_unipro_set_lpm(hba, false);
1097         if (ret)
1098                 return ret;
1099
1100         /*
1101          * Setting PA_Local_TX_LCC_Enable to 0 before link startup
1102          * to make sure that both host and device TX LCC are disabled
1103          * once link startup is completed.
1104          */
1105         ret = ufshcd_disable_host_tx_lcc(hba);
1106         if (ret)
1107                 return ret;
1108
1109         /* disable deep stall */
1110         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
1111         if (ret)
1112                 return ret;
1113
1114         tmp &= ~(1 << 6);
1115
1116         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
1117
1118         return ret;
1119 }
1120
1121 static void ufs_mtk_setup_clk_gating(struct ufs_hba *hba)
1122 {
1123         u32 ah_ms;
1124
1125         if (ufshcd_is_clkgating_allowed(hba)) {
1126                 if (ufshcd_is_auto_hibern8_supported(hba) && hba->ahit)
1127                         ah_ms = FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK,
1128                                           hba->ahit);
1129                 else
1130                         ah_ms = 10;
1131                 ufshcd_clkgate_delay_set(hba->dev, ah_ms + 5);
1132         }
1133 }
1134
1135 static void ufs_mtk_post_link(struct ufs_hba *hba)
1136 {
1137         /* enable unipro clock gating feature */
1138         ufs_mtk_cfg_unipro_cg(hba, true);
1139
1140         /* will be configured during probe hba */
1141         if (ufshcd_is_auto_hibern8_supported(hba))
1142                 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 10) |
1143                         FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
1144
1145         ufs_mtk_setup_clk_gating(hba);
1146 }
1147
1148 static int ufs_mtk_link_startup_notify(struct ufs_hba *hba,
1149                                        enum ufs_notify_change_status stage)
1150 {
1151         int ret = 0;
1152
1153         switch (stage) {
1154         case PRE_CHANGE:
1155                 ret = ufs_mtk_pre_link(hba);
1156                 break;
1157         case POST_CHANGE:
1158                 ufs_mtk_post_link(hba);
1159                 break;
1160         default:
1161                 ret = -EINVAL;
1162                 break;
1163         }
1164
1165         return ret;
1166 }
1167
1168 static int ufs_mtk_device_reset(struct ufs_hba *hba)
1169 {
1170         struct arm_smccc_res res;
1171
1172         /* disable hba before device reset */
1173         ufshcd_hba_stop(hba);
1174
1175         ufs_mtk_device_reset_ctrl(0, res);
1176
1177         /*
1178          * The reset signal is active low. UFS devices shall detect
1179          * more than or equal to 1us of positive or negative RST_n
1180          * pulse width.
1181          *
1182          * To be on safe side, keep the reset low for at least 10us.
1183          */
1184         usleep_range(10, 15);
1185
1186         ufs_mtk_device_reset_ctrl(1, res);
1187
1188         /* Some devices may need time to respond to rst_n */
1189         usleep_range(10000, 15000);
1190
1191         dev_info(hba->dev, "device reset done\n");
1192
1193         return 0;
1194 }
1195
1196 static int ufs_mtk_link_set_hpm(struct ufs_hba *hba)
1197 {
1198         int err;
1199
1200         err = ufshcd_hba_enable(hba);
1201         if (err)
1202                 return err;
1203
1204         err = ufs_mtk_unipro_set_lpm(hba, false);
1205         if (err)
1206                 return err;
1207
1208         err = ufshcd_uic_hibern8_exit(hba);
1209         if (!err)
1210                 ufshcd_set_link_active(hba);
1211         else
1212                 return err;
1213
1214         if (!hba->mcq_enabled) {
1215                 err = ufshcd_make_hba_operational(hba);
1216         } else {
1217                 ufs_mtk_config_mcq(hba, false);
1218                 ufshcd_mcq_make_queues_operational(hba);
1219                 ufshcd_mcq_config_mac(hba, hba->nutrs);
1220                 /* Enable MCQ mode */
1221                 ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1,
1222                               REG_UFS_MEM_CFG);
1223         }
1224
1225         if (err)
1226                 return err;
1227
1228         return 0;
1229 }
1230
1231 static int ufs_mtk_link_set_lpm(struct ufs_hba *hba)
1232 {
1233         int err;
1234
1235         /* Disable reset confirm feature by UniPro */
1236         ufshcd_writel(hba,
1237                       (ufshcd_readl(hba, REG_UFS_XOUFS_CTRL) & ~0x100),
1238                       REG_UFS_XOUFS_CTRL);
1239
1240         err = ufs_mtk_unipro_set_lpm(hba, true);
1241         if (err) {
1242                 /* Resume UniPro state for following error recovery */
1243                 ufs_mtk_unipro_set_lpm(hba, false);
1244                 return err;
1245         }
1246
1247         return 0;
1248 }
1249
1250 static void ufs_mtk_vccqx_set_lpm(struct ufs_hba *hba, bool lpm)
1251 {
1252         struct ufs_vreg *vccqx = NULL;
1253
1254         if (hba->vreg_info.vccq)
1255                 vccqx = hba->vreg_info.vccq;
1256         else
1257                 vccqx = hba->vreg_info.vccq2;
1258
1259         regulator_set_mode(vccqx->reg,
1260                            lpm ? REGULATOR_MODE_IDLE : REGULATOR_MODE_NORMAL);
1261 }
1262
1263 static void ufs_mtk_vsx_set_lpm(struct ufs_hba *hba, bool lpm)
1264 {
1265         struct arm_smccc_res res;
1266
1267         ufs_mtk_device_pwr_ctrl(!lpm,
1268                                 (unsigned long)hba->dev_info.wspecversion,
1269                                 res);
1270 }
1271
1272 static void ufs_mtk_dev_vreg_set_lpm(struct ufs_hba *hba, bool lpm)
1273 {
1274         if (!hba->vreg_info.vccq && !hba->vreg_info.vccq2)
1275                 return;
1276
1277         /* Skip if VCC is assumed always-on */
1278         if (!hba->vreg_info.vcc)
1279                 return;
1280
1281         /* Bypass LPM when device is still active */
1282         if (lpm && ufshcd_is_ufs_dev_active(hba))
1283                 return;
1284
1285         /* Bypass LPM if VCC is enabled */
1286         if (lpm && hba->vreg_info.vcc->enabled)
1287                 return;
1288
1289         if (lpm) {
1290                 ufs_mtk_vccqx_set_lpm(hba, lpm);
1291                 ufs_mtk_vsx_set_lpm(hba, lpm);
1292         } else {
1293                 ufs_mtk_vsx_set_lpm(hba, lpm);
1294                 ufs_mtk_vccqx_set_lpm(hba, lpm);
1295         }
1296 }
1297
1298 static void ufs_mtk_auto_hibern8_disable(struct ufs_hba *hba)
1299 {
1300         int ret;
1301
1302         /* disable auto-hibern8 */
1303         ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER);
1304
1305         /* wait host return to idle state when auto-hibern8 off */
1306         ufs_mtk_wait_idle_state(hba, 5);
1307
1308         ret = ufs_mtk_wait_link_state(hba, VS_LINK_UP, 100);
1309         if (ret)
1310                 dev_warn(hba->dev, "exit h8 state fail, ret=%d\n", ret);
1311 }
1312
1313 static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
1314         enum ufs_notify_change_status status)
1315 {
1316         int err;
1317         struct arm_smccc_res res;
1318
1319         if (status == PRE_CHANGE) {
1320                 if (ufshcd_is_auto_hibern8_supported(hba))
1321                         ufs_mtk_auto_hibern8_disable(hba);
1322                 return 0;
1323         }
1324
1325         if (ufshcd_is_link_hibern8(hba)) {
1326                 err = ufs_mtk_link_set_lpm(hba);
1327                 if (err)
1328                         goto fail;
1329         }
1330
1331         if (!ufshcd_is_link_active(hba)) {
1332                 /*
1333                  * Make sure no error will be returned to prevent
1334                  * ufshcd_suspend() re-enabling regulators while vreg is still
1335                  * in low-power mode.
1336                  */
1337                 err = ufs_mtk_mphy_power_on(hba, false);
1338                 if (err)
1339                         goto fail;
1340         }
1341
1342         if (ufshcd_is_link_off(hba))
1343                 ufs_mtk_device_reset_ctrl(0, res);
1344
1345         ufs_mtk_host_pwr_ctrl(HOST_PWR_HCI, false, res);
1346
1347         return 0;
1348 fail:
1349         /*
1350          * Set link as off state enforcedly to trigger
1351          * ufshcd_host_reset_and_restore() in ufshcd_suspend()
1352          * for completed host reset.
1353          */
1354         ufshcd_set_link_off(hba);
1355         return -EAGAIN;
1356 }
1357
1358 static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
1359 {
1360         int err;
1361         struct arm_smccc_res res;
1362
1363         if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
1364                 ufs_mtk_dev_vreg_set_lpm(hba, false);
1365
1366         ufs_mtk_host_pwr_ctrl(HOST_PWR_HCI, true, res);
1367
1368         err = ufs_mtk_mphy_power_on(hba, true);
1369         if (err)
1370                 goto fail;
1371
1372         if (ufshcd_is_link_hibern8(hba)) {
1373                 err = ufs_mtk_link_set_hpm(hba);
1374                 if (err)
1375                         goto fail;
1376         }
1377
1378         return 0;
1379 fail:
1380         return ufshcd_link_recovery(hba);
1381 }
1382
1383 static void ufs_mtk_dbg_register_dump(struct ufs_hba *hba)
1384 {
1385         /* Dump ufshci register 0x140 ~ 0x14C */
1386         ufshcd_dump_regs(hba, REG_UFS_XOUFS_CTRL, 0x10,
1387                          "XOUFS Ctrl (0x140): ");
1388
1389         ufshcd_dump_regs(hba, REG_UFS_EXTREG, 0x4, "Ext Reg ");
1390
1391         /* Dump ufshci register 0x2200 ~ 0x22AC */
1392         ufshcd_dump_regs(hba, REG_UFS_MPHYCTRL,
1393                          REG_UFS_REJECT_MON - REG_UFS_MPHYCTRL + 4,
1394                          "MPHY Ctrl (0x2200): ");
1395
1396         /* Direct debugging information to REG_MTK_PROBE */
1397         ufs_mtk_dbg_sel(hba);
1398         ufshcd_dump_regs(hba, REG_UFS_PROBE, 0x4, "Debug Probe ");
1399 }
1400
1401 static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba)
1402 {
1403         struct ufs_dev_info *dev_info = &hba->dev_info;
1404         u16 mid = dev_info->wmanufacturerid;
1405
1406         if (mid == UFS_VENDOR_SAMSUNG) {
1407                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 6);
1408                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), 10);
1409         }
1410
1411         /*
1412          * Decide waiting time before gating reference clock and
1413          * after ungating reference clock according to vendors'
1414          * requirements.
1415          */
1416         if (mid == UFS_VENDOR_SAMSUNG)
1417                 ufs_mtk_setup_ref_clk_wait_us(hba, 1);
1418         else if (mid == UFS_VENDOR_SKHYNIX)
1419                 ufs_mtk_setup_ref_clk_wait_us(hba, 30);
1420         else if (mid == UFS_VENDOR_TOSHIBA)
1421                 ufs_mtk_setup_ref_clk_wait_us(hba, 100);
1422         else
1423                 ufs_mtk_setup_ref_clk_wait_us(hba,
1424                                               REFCLK_DEFAULT_WAIT_US);
1425         return 0;
1426 }
1427
1428 static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba)
1429 {
1430         ufshcd_fixup_dev_quirks(hba, ufs_mtk_dev_fixups);
1431
1432         if (ufs_mtk_is_broken_vcc(hba) && hba->vreg_info.vcc &&
1433             (hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)) {
1434                 hba->vreg_info.vcc->always_on = true;
1435                 /*
1436                  * VCC will be kept always-on thus we don't
1437                  * need any delay during regulator operations
1438                  */
1439                 hba->dev_quirks &= ~(UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
1440                         UFS_DEVICE_QUIRK_DELAY_AFTER_LPM);
1441         }
1442
1443         ufs_mtk_vreg_fix_vcc(hba);
1444         ufs_mtk_vreg_fix_vccqx(hba);
1445 }
1446
1447 static void ufs_mtk_event_notify(struct ufs_hba *hba,
1448                                  enum ufs_event_type evt, void *data)
1449 {
1450         unsigned int val = *(u32 *)data;
1451         unsigned long reg;
1452         u8 bit;
1453
1454         trace_ufs_mtk_event(evt, val);
1455
1456         /* Print details of UIC Errors */
1457         if (evt <= UFS_EVT_DME_ERR) {
1458                 dev_info(hba->dev,
1459                          "Host UIC Error Code (%s): %08x\n",
1460                          ufs_uic_err_str[evt], val);
1461                 reg = val;
1462         }
1463
1464         if (evt == UFS_EVT_PA_ERR) {
1465                 for_each_set_bit(bit, &reg, ARRAY_SIZE(ufs_uic_pa_err_str))
1466                         dev_info(hba->dev, "%s\n", ufs_uic_pa_err_str[bit]);
1467         }
1468
1469         if (evt == UFS_EVT_DL_ERR) {
1470                 for_each_set_bit(bit, &reg, ARRAY_SIZE(ufs_uic_dl_err_str))
1471                         dev_info(hba->dev, "%s\n", ufs_uic_dl_err_str[bit]);
1472         }
1473 }
1474
1475 static void ufs_mtk_config_scaling_param(struct ufs_hba *hba,
1476                                 struct devfreq_dev_profile *profile,
1477                                 struct devfreq_simple_ondemand_data *data)
1478 {
1479         /* Customize min gear in clk scaling */
1480         hba->clk_scaling.min_gear = UFS_HS_G4;
1481
1482         hba->vps->devfreq_profile.polling_ms = 200;
1483         hba->vps->ondemand_data.upthreshold = 50;
1484         hba->vps->ondemand_data.downdifferential = 20;
1485 }
1486
1487 /**
1488  * ufs_mtk_clk_scale - Internal clk scaling operation
1489  *
1490  * MTK platform supports clk scaling by switching parent of ufs_sel(mux).
1491  * The ufs_sel downstream to ufs_ck which feeds directly to UFS hardware.
1492  * Max and min clocks rate of ufs_sel defined in dts should match rate of
1493  * "ufs_sel_max_src" and "ufs_sel_min_src" respectively.
1494  * This prevent changing rate of pll clock that is shared between modules.
1495  *
1496  * @hba: per adapter instance
1497  * @scale_up: True for scaling up and false for scaling down
1498  */
1499 static void ufs_mtk_clk_scale(struct ufs_hba *hba, bool scale_up)
1500 {
1501         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1502         struct ufs_mtk_clk *mclk = &host->mclk;
1503         struct ufs_clk_info *clki = mclk->ufs_sel_clki;
1504         int ret = 0;
1505
1506         ret = clk_prepare_enable(clki->clk);
1507         if (ret) {
1508                 dev_info(hba->dev,
1509                          "clk_prepare_enable() fail, ret: %d\n", ret);
1510                 return;
1511         }
1512
1513         if (scale_up) {
1514                 ret = clk_set_parent(clki->clk, mclk->ufs_sel_max_clki->clk);
1515                 clki->curr_freq = clki->max_freq;
1516         } else {
1517                 ret = clk_set_parent(clki->clk, mclk->ufs_sel_min_clki->clk);
1518                 clki->curr_freq = clki->min_freq;
1519         }
1520
1521         if (ret) {
1522                 dev_info(hba->dev,
1523                          "Failed to set ufs_sel_clki, ret: %d\n", ret);
1524         }
1525
1526         clk_disable_unprepare(clki->clk);
1527
1528         trace_ufs_mtk_clk_scale(clki->name, scale_up, clk_get_rate(clki->clk));
1529 }
1530
1531 static int ufs_mtk_clk_scale_notify(struct ufs_hba *hba, bool scale_up,
1532                                     enum ufs_notify_change_status status)
1533 {
1534         if (!ufshcd_is_clkscaling_supported(hba))
1535                 return 0;
1536
1537         if (status == PRE_CHANGE) {
1538                 /* Switch parent before clk_set_rate() */
1539                 ufs_mtk_clk_scale(hba, scale_up);
1540         } else {
1541                 /* Request interrupt latency QoS accordingly */
1542                 ufs_mtk_scale_perf(hba, scale_up);
1543         }
1544
1545         return 0;
1546 }
1547
1548 static int ufs_mtk_get_hba_mac(struct ufs_hba *hba)
1549 {
1550         return MAX_SUPP_MAC;
1551 }
1552
1553 static int ufs_mtk_op_runtime_config(struct ufs_hba *hba)
1554 {
1555         struct ufshcd_mcq_opr_info_t *opr;
1556         int i;
1557
1558         hba->mcq_opr[OPR_SQD].offset = REG_UFS_MTK_SQD;
1559         hba->mcq_opr[OPR_SQIS].offset = REG_UFS_MTK_SQIS;
1560         hba->mcq_opr[OPR_CQD].offset = REG_UFS_MTK_CQD;
1561         hba->mcq_opr[OPR_CQIS].offset = REG_UFS_MTK_CQIS;
1562
1563         for (i = 0; i < OPR_MAX; i++) {
1564                 opr = &hba->mcq_opr[i];
1565                 opr->stride = REG_UFS_MCQ_STRIDE;
1566                 opr->base = hba->mmio_base + opr->offset;
1567         }
1568
1569         return 0;
1570 }
1571
1572 static int ufs_mtk_mcq_config_resource(struct ufs_hba *hba)
1573 {
1574         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1575
1576         /* fail mcq initialization if interrupt is not filled properly */
1577         if (!host->mcq_nr_intr) {
1578                 dev_info(hba->dev, "IRQs not ready. MCQ disabled.");
1579                 return -EINVAL;
1580         }
1581
1582         hba->mcq_base = hba->mmio_base + MCQ_QUEUE_OFFSET(hba->mcq_capabilities);
1583         return 0;
1584 }
1585
1586 static irqreturn_t ufs_mtk_mcq_intr(int irq, void *__intr_info)
1587 {
1588         struct ufs_mtk_mcq_intr_info *mcq_intr_info = __intr_info;
1589         struct ufs_hba *hba = mcq_intr_info->hba;
1590         struct ufs_hw_queue *hwq;
1591         u32 events;
1592         int qid = mcq_intr_info->qid;
1593
1594         hwq = &hba->uhq[qid];
1595
1596         events = ufshcd_mcq_read_cqis(hba, qid);
1597         if (events)
1598                 ufshcd_mcq_write_cqis(hba, events, qid);
1599
1600         if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
1601                 ufshcd_mcq_poll_cqe_lock(hba, hwq);
1602
1603         return IRQ_HANDLED;
1604 }
1605
1606 static int ufs_mtk_config_mcq_irq(struct ufs_hba *hba)
1607 {
1608         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1609         u32 irq, i;
1610         int ret;
1611
1612         for (i = 0; i < host->mcq_nr_intr; i++) {
1613                 irq = host->mcq_intr_info[i].irq;
1614                 if (irq == MTK_MCQ_INVALID_IRQ) {
1615                         dev_err(hba->dev, "invalid irq. %d\n", i);
1616                         return -ENOPARAM;
1617                 }
1618
1619                 host->mcq_intr_info[i].qid = i;
1620                 ret = devm_request_irq(hba->dev, irq, ufs_mtk_mcq_intr, 0, UFSHCD,
1621                                        &host->mcq_intr_info[i]);
1622
1623                 dev_dbg(hba->dev, "request irq %d intr %s\n", irq, ret ? "failed" : "");
1624
1625                 if (ret) {
1626                         dev_err(hba->dev, "Cannot request irq %d\n", ret);
1627                         return ret;
1628                 }
1629         }
1630
1631         return 0;
1632 }
1633
1634 static int ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq)
1635 {
1636         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1637         int ret = 0;
1638
1639         if (!host->mcq_set_intr) {
1640                 /* Disable irq option register */
1641                 ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, 0, REG_UFS_MMIO_OPT_CTRL_0);
1642
1643                 if (irq) {
1644                         ret = ufs_mtk_config_mcq_irq(hba);
1645                         if (ret)
1646                                 return ret;
1647                 }
1648
1649                 host->mcq_set_intr = true;
1650         }
1651
1652         ufshcd_rmwl(hba, MCQ_AH8, MCQ_AH8, REG_UFS_MMIO_OPT_CTRL_0);
1653         ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, MCQ_MULTI_INTR_EN, REG_UFS_MMIO_OPT_CTRL_0);
1654
1655         return 0;
1656 }
1657
1658 static int ufs_mtk_config_esi(struct ufs_hba *hba)
1659 {
1660         return ufs_mtk_config_mcq(hba, true);
1661 }
1662
1663 /*
1664  * struct ufs_hba_mtk_vops - UFS MTK specific variant operations
1665  *
1666  * The variant operations configure the necessary controller and PHY
1667  * handshake during initialization.
1668  */
1669 static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
1670         .name                = "mediatek.ufshci",
1671         .init                = ufs_mtk_init,
1672         .get_ufs_hci_version = ufs_mtk_get_ufs_hci_version,
1673         .setup_clocks        = ufs_mtk_setup_clocks,
1674         .hce_enable_notify   = ufs_mtk_hce_enable_notify,
1675         .link_startup_notify = ufs_mtk_link_startup_notify,
1676         .pwr_change_notify   = ufs_mtk_pwr_change_notify,
1677         .apply_dev_quirks    = ufs_mtk_apply_dev_quirks,
1678         .fixup_dev_quirks    = ufs_mtk_fixup_dev_quirks,
1679         .suspend             = ufs_mtk_suspend,
1680         .resume              = ufs_mtk_resume,
1681         .dbg_register_dump   = ufs_mtk_dbg_register_dump,
1682         .device_reset        = ufs_mtk_device_reset,
1683         .event_notify        = ufs_mtk_event_notify,
1684         .config_scaling_param = ufs_mtk_config_scaling_param,
1685         .clk_scale_notify    = ufs_mtk_clk_scale_notify,
1686         /* mcq vops */
1687         .get_hba_mac         = ufs_mtk_get_hba_mac,
1688         .op_runtime_config   = ufs_mtk_op_runtime_config,
1689         .mcq_config_resource = ufs_mtk_mcq_config_resource,
1690         .config_esi          = ufs_mtk_config_esi,
1691 };
1692
1693 /**
1694  * ufs_mtk_probe - probe routine of the driver
1695  * @pdev: pointer to Platform device handle
1696  *
1697  * Return: zero for success and non-zero for failure.
1698  */
1699 static int ufs_mtk_probe(struct platform_device *pdev)
1700 {
1701         int err;
1702         struct device *dev = &pdev->dev;
1703         struct device_node *reset_node;
1704         struct platform_device *reset_pdev;
1705         struct device_link *link;
1706
1707         reset_node = of_find_compatible_node(NULL, NULL,
1708                                              "ti,syscon-reset");
1709         if (!reset_node) {
1710                 dev_notice(dev, "find ti,syscon-reset fail\n");
1711                 goto skip_reset;
1712         }
1713         reset_pdev = of_find_device_by_node(reset_node);
1714         if (!reset_pdev) {
1715                 dev_notice(dev, "find reset_pdev fail\n");
1716                 goto skip_reset;
1717         }
1718         link = device_link_add(dev, &reset_pdev->dev,
1719                 DL_FLAG_AUTOPROBE_CONSUMER);
1720         put_device(&reset_pdev->dev);
1721         if (!link) {
1722                 dev_notice(dev, "add reset device_link fail\n");
1723                 goto skip_reset;
1724         }
1725         /* supplier is not probed */
1726         if (link->status == DL_STATE_DORMANT) {
1727                 err = -EPROBE_DEFER;
1728                 goto out;
1729         }
1730
1731 skip_reset:
1732         /* perform generic probe */
1733         err = ufshcd_pltfrm_init(pdev, &ufs_hba_mtk_vops);
1734
1735 out:
1736         if (err)
1737                 dev_err(dev, "probe failed %d\n", err);
1738
1739         of_node_put(reset_node);
1740         return err;
1741 }
1742
1743 /**
1744  * ufs_mtk_remove - set driver_data of the device to NULL
1745  * @pdev: pointer to platform device handle
1746  *
1747  * Always return 0
1748  */
1749 static void ufs_mtk_remove(struct platform_device *pdev)
1750 {
1751         struct ufs_hba *hba =  platform_get_drvdata(pdev);
1752
1753         pm_runtime_get_sync(&(pdev)->dev);
1754         ufshcd_remove(hba);
1755 }
1756
1757 #ifdef CONFIG_PM_SLEEP
1758 static int ufs_mtk_system_suspend(struct device *dev)
1759 {
1760         struct ufs_hba *hba = dev_get_drvdata(dev);
1761         int ret;
1762
1763         ret = ufshcd_system_suspend(dev);
1764         if (ret)
1765                 return ret;
1766
1767         ufs_mtk_dev_vreg_set_lpm(hba, true);
1768
1769         return 0;
1770 }
1771
1772 static int ufs_mtk_system_resume(struct device *dev)
1773 {
1774         struct ufs_hba *hba = dev_get_drvdata(dev);
1775
1776         ufs_mtk_dev_vreg_set_lpm(hba, false);
1777
1778         return ufshcd_system_resume(dev);
1779 }
1780 #endif
1781
1782 #ifdef CONFIG_PM
1783 static int ufs_mtk_runtime_suspend(struct device *dev)
1784 {
1785         struct ufs_hba *hba = dev_get_drvdata(dev);
1786         int ret = 0;
1787
1788         ret = ufshcd_runtime_suspend(dev);
1789         if (ret)
1790                 return ret;
1791
1792         ufs_mtk_dev_vreg_set_lpm(hba, true);
1793
1794         return 0;
1795 }
1796
1797 static int ufs_mtk_runtime_resume(struct device *dev)
1798 {
1799         struct ufs_hba *hba = dev_get_drvdata(dev);
1800
1801         ufs_mtk_dev_vreg_set_lpm(hba, false);
1802
1803         return ufshcd_runtime_resume(dev);
1804 }
1805 #endif
1806
1807 static const struct dev_pm_ops ufs_mtk_pm_ops = {
1808         SET_SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend,
1809                                 ufs_mtk_system_resume)
1810         SET_RUNTIME_PM_OPS(ufs_mtk_runtime_suspend,
1811                            ufs_mtk_runtime_resume, NULL)
1812         .prepare         = ufshcd_suspend_prepare,
1813         .complete        = ufshcd_resume_complete,
1814 };
1815
1816 static struct platform_driver ufs_mtk_pltform = {
1817         .probe      = ufs_mtk_probe,
1818         .remove_new = ufs_mtk_remove,
1819         .driver = {
1820                 .name   = "ufshcd-mtk",
1821                 .pm     = &ufs_mtk_pm_ops,
1822                 .of_match_table = ufs_mtk_of_match,
1823         },
1824 };
1825
1826 MODULE_AUTHOR("Stanley Chu <stanley.chu@mediatek.com>");
1827 MODULE_AUTHOR("Peter Wang <peter.wang@mediatek.com>");
1828 MODULE_DESCRIPTION("MediaTek UFS Host Driver");
1829 MODULE_LICENSE("GPL v2");
1830
1831 module_platform_driver(ufs_mtk_pltform);