drivers/slimbus/qcom-ngd-ctrl.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (c) 2011-2017, The Linux Foundation. All rights reserved.
   3 // Copyright (c) 2018, Linaro Limited
   4
   5 #include <linux/irq.h>
   6 #include <linux/kernel.h>
   7 #include <linux/init.h>
   8 #include <linux/slab.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/platform_device.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmaengine.h>
  13 #include <linux/slimbus.h>
  14 #include <linux/delay.h>
  15 #include <linux/pm_runtime.h>
  16 #include <linux/mutex.h>
  17 #include <linux/notifier.h>
  18 #include <linux/remoteproc/qcom_rproc.h>
  19 #include <linux/of.h>
  20 #include <linux/io.h>
  21 #include <linux/soc/qcom/qmi.h>
  22 #include <linux/soc/qcom/pdr.h>
  23 #include <net/sock.h>
  24 #include "slimbus.h"
  25
  26 /* NGD (Non-ported Generic Device) registers */
  27 #define NGD_CFG                 0x0
  28 #define NGD_CFG_ENABLE          BIT(0)
  29 #define NGD_CFG_RX_MSGQ_EN      BIT(1)
  30 #define NGD_CFG_TX_MSGQ_EN      BIT(2)
  31 #define NGD_STATUS              0x4
  32 #define NGD_LADDR               BIT(1)
  33 #define NGD_RX_MSGQ_CFG         0x8
  34 #define NGD_INT_EN              0x10
  35 #define NGD_INT_RECFG_DONE      BIT(24)
  36 #define NGD_INT_TX_NACKED_2     BIT(25)
  37 #define NGD_INT_MSG_BUF_CONTE   BIT(26)
  38 #define NGD_INT_MSG_TX_INVAL    BIT(27)
  39 #define NGD_INT_IE_VE_CHG       BIT(28)
  40 #define NGD_INT_DEV_ERR         BIT(29)
  41 #define NGD_INT_RX_MSG_RCVD     BIT(30)
  42 #define NGD_INT_TX_MSG_SENT     BIT(31)
  43 #define NGD_INT_STAT            0x14
  44 #define NGD_INT_CLR             0x18
  45 #define DEF_NGD_INT_MASK (NGD_INT_TX_NACKED_2 | NGD_INT_MSG_BUF_CONTE | \
  46                                 NGD_INT_MSG_TX_INVAL | NGD_INT_IE_VE_CHG | \
  47                                 NGD_INT_DEV_ERR | NGD_INT_TX_MSG_SENT | \
  48                                 NGD_INT_RX_MSG_RCVD)
  49
  50 /* Slimbus QMI service */
  51 #define SLIMBUS_QMI_SVC_ID      0x0301
  52 #define SLIMBUS_QMI_SVC_V1      1
  53 #define SLIMBUS_QMI_INS_ID      0
  54 #define SLIMBUS_QMI_SELECT_INSTANCE_REQ_V01     0x0020
  55 #define SLIMBUS_QMI_SELECT_INSTANCE_RESP_V01    0x0020
  56 #define SLIMBUS_QMI_POWER_REQ_V01               0x0021
  57 #define SLIMBUS_QMI_POWER_RESP_V01              0x0021
  58 #define SLIMBUS_QMI_CHECK_FRAMER_STATUS_REQ     0x0022
  59 #define SLIMBUS_QMI_CHECK_FRAMER_STATUS_RESP    0x0022
  60 #define SLIMBUS_QMI_POWER_REQ_MAX_MSG_LEN       14
  61 #define SLIMBUS_QMI_POWER_RESP_MAX_MSG_LEN      7
  62 #define SLIMBUS_QMI_SELECT_INSTANCE_REQ_MAX_MSG_LEN     14
  63 #define SLIMBUS_QMI_SELECT_INSTANCE_RESP_MAX_MSG_LEN    7
  64 #define SLIMBUS_QMI_CHECK_FRAMER_STAT_RESP_MAX_MSG_LEN  7
  65 /* QMI response timeout of 500ms */
  66 #define SLIMBUS_QMI_RESP_TOUT   1000
  67
  68 /* User defined commands */
  69 #define SLIM_USR_MC_GENERIC_ACK 0x25
  70 #define SLIM_USR_MC_MASTER_CAPABILITY   0x0
  71 #define SLIM_USR_MC_REPORT_SATELLITE    0x1
  72 #define SLIM_USR_MC_ADDR_QUERY          0xD
  73 #define SLIM_USR_MC_ADDR_REPLY          0xE
  74 #define SLIM_USR_MC_DEFINE_CHAN         0x20
  75 #define SLIM_USR_MC_DEF_ACT_CHAN        0x21
  76 #define SLIM_USR_MC_CHAN_CTRL           0x23
  77 #define SLIM_USR_MC_RECONFIG_NOW        0x24
  78 #define SLIM_USR_MC_REQ_BW              0x28
  79 #define SLIM_USR_MC_CONNECT_SRC         0x2C
  80 #define SLIM_USR_MC_CONNECT_SINK        0x2D
  81 #define SLIM_USR_MC_DISCONNECT_PORT     0x2E
  82 #define SLIM_USR_MC_REPEAT_CHANGE_VALUE 0x0
  83
  84 #define SLIM_RX_MSGQ_TIMEOUT_VAL        0x10000
  85
  86 #define SLIM_LA_MGR     0xFF
  87 #define SLIM_ROOT_FREQ  24576000
  88 #define LADDR_RETRY     5
  89
  90 /* Per spec.max 40 bytes per received message */
  91 #define SLIM_MSGQ_BUF_LEN       40
  92 #define QCOM_SLIM_NGD_DESC_NUM  32
  93
  94 #define SLIM_MSG_ASM_FIRST_WORD(l, mt, mc, dt, ad) \
  95                 ((l) | ((mt) << 5) | ((mc) << 8) | ((dt) << 15) | ((ad) << 16))
  96
  97 #define INIT_MX_RETRIES 10
  98 #define DEF_RETRY_MS    10
  99 #define SAT_MAGIC_LSB   0xD9
 100 #define SAT_MAGIC_MSB   0xC5
 101 #define SAT_MSG_VER     0x1
 102 #define SAT_MSG_PROT    0x1
 103 #define to_ngd(d)       container_of(d, struct qcom_slim_ngd, dev)
 104
 105 struct ngd_reg_offset_data {
 106         u32 offset, size;
 107 };
 108
 109 static const struct ngd_reg_offset_data ngd_v1_5_offset_info = {
 110         .offset = 0x1000,
 111         .size = 0x1000,
 112 };
 113
 114 enum qcom_slim_ngd_state {
 115         QCOM_SLIM_NGD_CTRL_AWAKE,
 116         QCOM_SLIM_NGD_CTRL_IDLE,
 117         QCOM_SLIM_NGD_CTRL_ASLEEP,
 118         QCOM_SLIM_NGD_CTRL_DOWN,
 119 };
 120
 121 struct qcom_slim_ngd_qmi {
 122         struct qmi_handle qmi;
 123         struct sockaddr_qrtr svc_info;
 124         struct qmi_handle svc_event_hdl;
 125         struct qmi_response_type_v01 resp;
 126         struct qmi_handle *handle;
 127         struct completion qmi_comp;
 128 };
 129
 130 struct qcom_slim_ngd_ctrl;
 131 struct qcom_slim_ngd;
 132
 133 struct qcom_slim_ngd_dma_desc {
 134         struct dma_async_tx_descriptor *desc;
 135         struct qcom_slim_ngd_ctrl *ctrl;
 136         struct completion *comp;
 137         dma_cookie_t cookie;
 138         dma_addr_t phys;
 139         void *base;
 140 };
 141
 142 struct qcom_slim_ngd {
 143         struct platform_device *pdev;
 144         void __iomem *base;
 145         int id;
 146 };
 147
 148 struct qcom_slim_ngd_ctrl {
 149         struct slim_framer framer;
 150         struct slim_controller ctrl;
 151         struct qcom_slim_ngd_qmi qmi;
 152         struct qcom_slim_ngd *ngd;
 153         struct device *dev;
 154         void __iomem *base;
 155         struct dma_chan *dma_rx_channel;
 156         struct dma_chan *dma_tx_channel;
 157         struct qcom_slim_ngd_dma_desc rx_desc[QCOM_SLIM_NGD_DESC_NUM];
 158         struct qcom_slim_ngd_dma_desc txdesc[QCOM_SLIM_NGD_DESC_NUM];
 159         struct completion reconf;
 160         struct work_struct m_work;
 161         struct work_struct ngd_up_work;
 162         struct workqueue_struct *mwq;
 163         struct completion qmi_up;
 164         spinlock_t tx_buf_lock;
 165         struct mutex tx_lock;
 166         struct mutex ssr_lock;
 167         struct notifier_block nb;
 168         void *notifier;
 169         struct pdr_handle *pdr;
 170         enum qcom_slim_ngd_state state;
 171         dma_addr_t rx_phys_base;
 172         dma_addr_t tx_phys_base;
 173         void *rx_base;
 174         void *tx_base;
 175         int tx_tail;
 176         int tx_head;
 177         u32 ver;
 178 };
 179
 180 enum slimbus_mode_enum_type_v01 {
 181         /* To force a 32 bit signed enum. Do not change or use*/
 182         SLIMBUS_MODE_ENUM_TYPE_MIN_ENUM_VAL_V01 = INT_MIN,
 183         SLIMBUS_MODE_SATELLITE_V01 = 1,
 184         SLIMBUS_MODE_MASTER_V01 = 2,
 185         SLIMBUS_MODE_ENUM_TYPE_MAX_ENUM_VAL_V01 = INT_MAX,
 186 };
 187
 188 enum slimbus_pm_enum_type_v01 {
 189         /* To force a 32 bit signed enum. Do not change or use*/
 190         SLIMBUS_PM_ENUM_TYPE_MIN_ENUM_VAL_V01 = INT_MIN,
 191         SLIMBUS_PM_INACTIVE_V01 = 1,
 192         SLIMBUS_PM_ACTIVE_V01 = 2,
 193         SLIMBUS_PM_ENUM_TYPE_MAX_ENUM_VAL_V01 = INT_MAX,
 194 };
 195
 196 enum slimbus_resp_enum_type_v01 {
 197         SLIMBUS_RESP_ENUM_TYPE_MIN_VAL_V01 = INT_MIN,
 198         SLIMBUS_RESP_SYNCHRONOUS_V01 = 1,
 199         SLIMBUS_RESP_ENUM_TYPE_MAX_VAL_V01 = INT_MAX,
 200 };
 201
 202 struct slimbus_select_inst_req_msg_v01 {
 203         uint32_t instance;
 204         uint8_t mode_valid;
 205         enum slimbus_mode_enum_type_v01 mode;
 206 };
 207
 208 struct slimbus_select_inst_resp_msg_v01 {
 209         struct qmi_response_type_v01 resp;
 210 };
 211
 212 struct slimbus_power_req_msg_v01 {
 213         enum slimbus_pm_enum_type_v01 pm_req;
 214         uint8_t resp_type_valid;
 215         enum slimbus_resp_enum_type_v01 resp_type;
 216 };
 217
 218 struct slimbus_power_resp_msg_v01 {
 219         struct qmi_response_type_v01 resp;
 220 };
 221
 222 static const struct qmi_elem_info slimbus_select_inst_req_msg_v01_ei[] = {
 223         {
 224                 .data_type  = QMI_UNSIGNED_4_BYTE,
 225                 .elem_len   = 1,
 226                 .elem_size  = sizeof(uint32_t),
 227                 .array_type = NO_ARRAY,
 228                 .tlv_type   = 0x01,
 229                 .offset     = offsetof(struct slimbus_select_inst_req_msg_v01,
 230                                        instance),
 231                 .ei_array   = NULL,
 232         },
 233         {
 234                 .data_type  = QMI_OPT_FLAG,
 235                 .elem_len   = 1,
 236                 .elem_size  = sizeof(uint8_t),
 237                 .array_type = NO_ARRAY,
 238                 .tlv_type   = 0x10,
 239                 .offset     = offsetof(struct slimbus_select_inst_req_msg_v01,
 240                                        mode_valid),
 241                 .ei_array   = NULL,
 242         },
 243         {
 244                 .data_type  = QMI_UNSIGNED_4_BYTE,
 245                 .elem_len   = 1,
 246                 .elem_size  = sizeof(enum slimbus_mode_enum_type_v01),
 247                 .array_type = NO_ARRAY,
 248                 .tlv_type   = 0x10,
 249                 .offset     = offsetof(struct slimbus_select_inst_req_msg_v01,
 250                                        mode),
 251                 .ei_array   = NULL,
 252         },
 253         {
 254                 .data_type  = QMI_EOTI,
 255                 .elem_len   = 0,
 256                 .elem_size  = 0,
 257                 .array_type = NO_ARRAY,
 258                 .tlv_type   = 0x00,
 259                 .offset     = 0,
 260                 .ei_array   = NULL,
 261         },
 262 };
 263
 264 static const struct qmi_elem_info slimbus_select_inst_resp_msg_v01_ei[] = {
 265         {
 266                 .data_type  = QMI_STRUCT,
 267                 .elem_len   = 1,
 268                 .elem_size  = sizeof(struct qmi_response_type_v01),
 269                 .array_type = NO_ARRAY,
 270                 .tlv_type   = 0x02,
 271                 .offset     = offsetof(struct slimbus_select_inst_resp_msg_v01,
 272                                        resp),
 273                 .ei_array   = qmi_response_type_v01_ei,
 274         },
 275         {
 276                 .data_type  = QMI_EOTI,
 277                 .elem_len   = 0,
 278                 .elem_size  = 0,
 279                 .array_type = NO_ARRAY,
 280                 .tlv_type   = 0x00,
 281                 .offset     = 0,
 282                 .ei_array   = NULL,
 283         },
 284 };
 285
 286 static const struct qmi_elem_info slimbus_power_req_msg_v01_ei[] = {
 287         {
 288                 .data_type  = QMI_UNSIGNED_4_BYTE,
 289                 .elem_len   = 1,
 290                 .elem_size  = sizeof(enum slimbus_pm_enum_type_v01),
 291                 .array_type = NO_ARRAY,
 292                 .tlv_type   = 0x01,
 293                 .offset     = offsetof(struct slimbus_power_req_msg_v01,
 294                                        pm_req),
 295                 .ei_array   = NULL,
 296         },
 297         {
 298                 .data_type  = QMI_OPT_FLAG,
 299                 .elem_len   = 1,
 300                 .elem_size  = sizeof(uint8_t),
 301                 .array_type = NO_ARRAY,
 302                 .tlv_type   = 0x10,
 303                 .offset     = offsetof(struct slimbus_power_req_msg_v01,
 304                                        resp_type_valid),
 305         },
 306         {
 307                 .data_type  = QMI_SIGNED_4_BYTE_ENUM,
 308                 .elem_len   = 1,
 309                 .elem_size  = sizeof(enum slimbus_resp_enum_type_v01),
 310                 .array_type = NO_ARRAY,
 311                 .tlv_type   = 0x10,
 312                 .offset     = offsetof(struct slimbus_power_req_msg_v01,
 313                                        resp_type),
 314         },
 315         {
 316                 .data_type  = QMI_EOTI,
 317                 .elem_len   = 0,
 318                 .elem_size  = 0,
 319                 .array_type = NO_ARRAY,
 320                 .tlv_type   = 0x00,
 321                 .offset     = 0,
 322                 .ei_array   = NULL,
 323         },
 324 };
 325
 326 static const struct qmi_elem_info slimbus_power_resp_msg_v01_ei[] = {
 327         {
 328                 .data_type  = QMI_STRUCT,
 329                 .elem_len   = 1,
 330                 .elem_size  = sizeof(struct qmi_response_type_v01),
 331                 .array_type = NO_ARRAY,
 332                 .tlv_type   = 0x02,
 333                 .offset     = offsetof(struct slimbus_power_resp_msg_v01, resp),
 334                 .ei_array   = qmi_response_type_v01_ei,
 335         },
 336         {
 337                 .data_type  = QMI_EOTI,
 338                 .elem_len   = 0,
 339                 .elem_size  = 0,
 340                 .array_type = NO_ARRAY,
 341                 .tlv_type   = 0x00,
 342                 .offset     = 0,
 343                 .ei_array   = NULL,
 344         },
 345 };
 346
 347 static int qcom_slim_qmi_send_select_inst_req(struct qcom_slim_ngd_ctrl *ctrl,
 348                                 struct slimbus_select_inst_req_msg_v01 *req)
 349 {
 350         struct slimbus_select_inst_resp_msg_v01 resp = { { 0, 0 } };
 351         struct qmi_txn txn;
 352         int rc;
 353
 354         rc = qmi_txn_init(ctrl->qmi.handle, &txn,
 355                                 slimbus_select_inst_resp_msg_v01_ei, &resp);
 356         if (rc < 0) {
 357                 dev_err(ctrl->dev, "QMI TXN init fail: %d\n", rc);
 358                 return rc;
 359         }
 360
 361         rc = qmi_send_request(ctrl->qmi.handle, NULL, &txn,
 362                                 SLIMBUS_QMI_SELECT_INSTANCE_REQ_V01,
 363                                 SLIMBUS_QMI_SELECT_INSTANCE_REQ_MAX_MSG_LEN,
 364                                 slimbus_select_inst_req_msg_v01_ei, req);
 365         if (rc < 0) {
 366                 dev_err(ctrl->dev, "QMI send req fail %d\n", rc);
 367                 qmi_txn_cancel(&txn);
 368                 return rc;
 369         }
 370
 371         rc = qmi_txn_wait(&txn, SLIMBUS_QMI_RESP_TOUT);
 372         if (rc < 0) {
 373                 dev_err(ctrl->dev, "QMI TXN wait fail: %d\n", rc);
 374                 return rc;
 375         }
 376         /* Check the response */
 377         if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
 378                 dev_err(ctrl->dev, "QMI request failed 0x%x\n",
 379                         resp.resp.result);
 380                 return -EREMOTEIO;
 381         }
 382
 383         return 0;
 384 }
 385
 386 static void qcom_slim_qmi_power_resp_cb(struct qmi_handle *handle,
 387                                         struct sockaddr_qrtr *sq,
 388                                         struct qmi_txn *txn, const void *data)
 389 {
 390         struct slimbus_power_resp_msg_v01 *resp;
 391
 392         resp = (struct slimbus_power_resp_msg_v01 *)data;
 393         if (resp->resp.result != QMI_RESULT_SUCCESS_V01)
 394                 pr_err("QMI power request failed 0x%x\n",
 395                                 resp->resp.result);
 396
 397         complete(&txn->completion);
 398 }
 399
 400 static int qcom_slim_qmi_send_power_request(struct qcom_slim_ngd_ctrl *ctrl,
 401                                         struct slimbus_power_req_msg_v01 *req)
 402 {
 403         struct slimbus_power_resp_msg_v01 resp = { { 0, 0 } };
 404         struct qmi_txn txn;
 405         int rc;
 406
 407         rc = qmi_txn_init(ctrl->qmi.handle, &txn,
 408                                 slimbus_power_resp_msg_v01_ei, &resp);
 409
 410         rc = qmi_send_request(ctrl->qmi.handle, NULL, &txn,
 411                                 SLIMBUS_QMI_POWER_REQ_V01,
 412                                 SLIMBUS_QMI_POWER_REQ_MAX_MSG_LEN,
 413                                 slimbus_power_req_msg_v01_ei, req);
 414         if (rc < 0) {
 415                 dev_err(ctrl->dev, "QMI send req fail %d\n", rc);
 416                 qmi_txn_cancel(&txn);
 417                 return rc;
 418         }
 419
 420         rc = qmi_txn_wait(&txn, SLIMBUS_QMI_RESP_TOUT);
 421         if (rc < 0) {
 422                 dev_err(ctrl->dev, "QMI TXN wait fail: %d\n", rc);
 423                 return rc;
 424         }
 425
 426         /* Check the response */
 427         if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
 428                 dev_err(ctrl->dev, "QMI request failed 0x%x\n",
 429                         resp.resp.result);
 430                 return -EREMOTEIO;
 431         }
 432
 433         return 0;
 434 }
 435
 436 static const struct qmi_msg_handler qcom_slim_qmi_msg_handlers[] = {
 437         {
 438                 .type = QMI_RESPONSE,
 439                 .msg_id = SLIMBUS_QMI_POWER_RESP_V01,
 440                 .ei = slimbus_power_resp_msg_v01_ei,
 441                 .decoded_size = sizeof(struct slimbus_power_resp_msg_v01),
 442                 .fn = qcom_slim_qmi_power_resp_cb,
 443         },
 444         {}
 445 };
 446
 447 static int qcom_slim_qmi_init(struct qcom_slim_ngd_ctrl *ctrl,
 448                               bool apps_is_master)
 449 {
 450         struct slimbus_select_inst_req_msg_v01 req;
 451         struct qmi_handle *handle;
 452         int rc;
 453
 454         handle = devm_kzalloc(ctrl->dev, sizeof(*handle), GFP_KERNEL);
 455         if (!handle)
 456                 return -ENOMEM;
 457
 458         rc = qmi_handle_init(handle, SLIMBUS_QMI_POWER_REQ_MAX_MSG_LEN,
 459                                 NULL, qcom_slim_qmi_msg_handlers);
 460         if (rc < 0) {
 461                 dev_err(ctrl->dev, "QMI client init failed: %d\n", rc);
 462                 goto qmi_handle_init_failed;
 463         }
 464
 465         rc = kernel_connect(handle->sock,
 466                                 (struct sockaddr *)&ctrl->qmi.svc_info,
 467                                 sizeof(ctrl->qmi.svc_info), 0);
 468         if (rc < 0) {
 469                 dev_err(ctrl->dev, "Remote Service connect failed: %d\n", rc);
 470                 goto qmi_connect_to_service_failed;
 471         }
 472
 473         /* Instance is 0 based */
 474         req.instance = (ctrl->ngd->id >> 1);
 475         req.mode_valid = 1;
 476
 477         /* Mode indicates the role of the ADSP */
 478         if (apps_is_master)
 479                 req.mode = SLIMBUS_MODE_SATELLITE_V01;
 480         else
 481                 req.mode = SLIMBUS_MODE_MASTER_V01;
 482
 483         ctrl->qmi.handle = handle;
 484
 485         rc = qcom_slim_qmi_send_select_inst_req(ctrl, &req);
 486         if (rc) {
 487                 dev_err(ctrl->dev, "failed to select h/w instance\n");
 488                 goto qmi_select_instance_failed;
 489         }
 490
 491         return 0;
 492
 493 qmi_select_instance_failed:
 494         ctrl->qmi.handle = NULL;
 495 qmi_connect_to_service_failed:
 496         qmi_handle_release(handle);
 497 qmi_handle_init_failed:
 498         devm_kfree(ctrl->dev, handle);
 499         return rc;
 500 }
 501
 502 static void qcom_slim_qmi_exit(struct qcom_slim_ngd_ctrl *ctrl)
 503 {
 504         if (!ctrl->qmi.handle)
 505                 return;
 506
 507         qmi_handle_release(ctrl->qmi.handle);
 508         devm_kfree(ctrl->dev, ctrl->qmi.handle);
 509         ctrl->qmi.handle = NULL;
 510 }
 511
 512 static int qcom_slim_qmi_power_request(struct qcom_slim_ngd_ctrl *ctrl,
 513                                        bool active)
 514 {
 515         struct slimbus_power_req_msg_v01 req;
 516
 517         if (active)
 518                 req.pm_req = SLIMBUS_PM_ACTIVE_V01;
 519         else
 520                 req.pm_req = SLIMBUS_PM_INACTIVE_V01;
 521
 522         req.resp_type_valid = 0;
 523
 524         return qcom_slim_qmi_send_power_request(ctrl, &req);
 525 }
 526
 527 static u32 *qcom_slim_ngd_tx_msg_get(struct qcom_slim_ngd_ctrl *ctrl, int len,
 528                                      struct completion *comp)
 529 {
 530         struct qcom_slim_ngd_dma_desc *desc;
 531         unsigned long flags;
 532
 533         spin_lock_irqsave(&ctrl->tx_buf_lock, flags);
 534
 535         if ((ctrl->tx_tail + 1) % QCOM_SLIM_NGD_DESC_NUM == ctrl->tx_head) {
 536                 spin_unlock_irqrestore(&ctrl->tx_buf_lock, flags);
 537                 return NULL;
 538         }
 539         desc  = &ctrl->txdesc[ctrl->tx_tail];
 540         desc->base = ctrl->tx_base + ctrl->tx_tail * SLIM_MSGQ_BUF_LEN;
 541         desc->comp = comp;
 542         ctrl->tx_tail = (ctrl->tx_tail + 1) % QCOM_SLIM_NGD_DESC_NUM;
 543
 544         spin_unlock_irqrestore(&ctrl->tx_buf_lock, flags);
 545
 546         return desc->base;
 547 }
 548
 549 static void qcom_slim_ngd_tx_msg_dma_cb(void *args)
 550 {
 551         struct qcom_slim_ngd_dma_desc *desc = args;
 552         struct qcom_slim_ngd_ctrl *ctrl = desc->ctrl;
 553         unsigned long flags;
 554
 555         spin_lock_irqsave(&ctrl->tx_buf_lock, flags);
 556
 557         if (desc->comp) {
 558                 complete(desc->comp);
 559                 desc->comp = NULL;
 560         }
 561
 562         ctrl->tx_head = (ctrl->tx_head + 1) % QCOM_SLIM_NGD_DESC_NUM;
 563         spin_unlock_irqrestore(&ctrl->tx_buf_lock, flags);
 564 }
 565
 566 static int qcom_slim_ngd_tx_msg_post(struct qcom_slim_ngd_ctrl *ctrl,
 567                                      void *buf, int len)
 568 {
 569         struct qcom_slim_ngd_dma_desc *desc;
 570         unsigned long flags;
 571         int index, offset;
 572
 573         spin_lock_irqsave(&ctrl->tx_buf_lock, flags);
 574         offset = buf - ctrl->tx_base;
 575         index = offset/SLIM_MSGQ_BUF_LEN;
 576
 577         desc = &ctrl->txdesc[index];
 578         desc->phys = ctrl->tx_phys_base + offset;
 579         desc->base = ctrl->tx_base + offset;
 580         desc->ctrl = ctrl;
 581         len = (len + 3) & 0xfc;
 582
 583         desc->desc = dmaengine_prep_slave_single(ctrl->dma_tx_channel,
 584                                                 desc->phys, len,
 585                                                 DMA_MEM_TO_DEV,
 586                                                 DMA_PREP_INTERRUPT);
 587         if (!desc->desc) {
 588                 dev_err(ctrl->dev, "unable to prepare channel\n");
 589                 spin_unlock_irqrestore(&ctrl->tx_buf_lock, flags);
 590                 return -EINVAL;
 591         }
 592
 593         desc->desc->callback = qcom_slim_ngd_tx_msg_dma_cb;
 594         desc->desc->callback_param = desc;
 595         desc->desc->cookie = dmaengine_submit(desc->desc);
 596         dma_async_issue_pending(ctrl->dma_tx_channel);
 597         spin_unlock_irqrestore(&ctrl->tx_buf_lock, flags);
 598
 599         return 0;
 600 }
 601
 602 static void qcom_slim_ngd_rx(struct qcom_slim_ngd_ctrl *ctrl, u8 *buf)
 603 {
 604         u8 mc, mt, len;
 605
 606         mt = SLIM_HEADER_GET_MT(buf[0]);
 607         len = SLIM_HEADER_GET_RL(buf[0]);
 608         mc = SLIM_HEADER_GET_MC(buf[1]);
 609
 610         if (mc == SLIM_USR_MC_MASTER_CAPABILITY &&
 611                 mt == SLIM_MSG_MT_SRC_REFERRED_USER)
 612                 queue_work(ctrl->mwq, &ctrl->m_work);
 613
 614         if (mc == SLIM_MSG_MC_REPLY_INFORMATION ||
 615             mc == SLIM_MSG_MC_REPLY_VALUE || (mc == SLIM_USR_MC_ADDR_REPLY &&
 616             mt == SLIM_MSG_MT_SRC_REFERRED_USER) ||
 617                 (mc == SLIM_USR_MC_GENERIC_ACK &&
 618                  mt == SLIM_MSG_MT_SRC_REFERRED_USER)) {
 619                 slim_msg_response(&ctrl->ctrl, &buf[4], buf[3], len - 4);
 620                 pm_runtime_mark_last_busy(ctrl->ctrl.dev);
 621         }
 622 }
 623
 624 static void qcom_slim_ngd_rx_msgq_cb(void *args)
 625 {
 626         struct qcom_slim_ngd_dma_desc *desc = args;
 627         struct qcom_slim_ngd_ctrl *ctrl = desc->ctrl;
 628
 629         qcom_slim_ngd_rx(ctrl, (u8 *)desc->base);
 630         /* Add descriptor back to the queue */
 631         desc->desc = dmaengine_prep_slave_single(ctrl->dma_rx_channel,
 632                                         desc->phys, SLIM_MSGQ_BUF_LEN,
 633                                         DMA_DEV_TO_MEM,
 634                                         DMA_PREP_INTERRUPT);
 635         if (!desc->desc) {
 636                 dev_err(ctrl->dev, "Unable to prepare rx channel\n");
 637                 return;
 638         }
 639
 640         desc->desc->callback = qcom_slim_ngd_rx_msgq_cb;
 641         desc->desc->callback_param = desc;
 642         desc->desc->cookie = dmaengine_submit(desc->desc);
 643         dma_async_issue_pending(ctrl->dma_rx_channel);
 644 }
 645
 646 static int qcom_slim_ngd_post_rx_msgq(struct qcom_slim_ngd_ctrl *ctrl)
 647 {
 648         struct qcom_slim_ngd_dma_desc *desc;
 649         int i;
 650
 651         for (i = 0; i < QCOM_SLIM_NGD_DESC_NUM; i++) {
 652                 desc = &ctrl->rx_desc[i];
 653                 desc->phys = ctrl->rx_phys_base + i * SLIM_MSGQ_BUF_LEN;
 654                 desc->ctrl = ctrl;
 655                 desc->base = ctrl->rx_base + i * SLIM_MSGQ_BUF_LEN;
 656                 desc->desc = dmaengine_prep_slave_single(ctrl->dma_rx_channel,
 657                                                 desc->phys, SLIM_MSGQ_BUF_LEN,
 658                                                 DMA_DEV_TO_MEM,
 659                                                 DMA_PREP_INTERRUPT);
 660                 if (!desc->desc) {
 661                         dev_err(ctrl->dev, "Unable to prepare rx channel\n");
 662                         return -EINVAL;
 663                 }
 664
 665                 desc->desc->callback = qcom_slim_ngd_rx_msgq_cb;
 666                 desc->desc->callback_param = desc;
 667                 desc->desc->cookie = dmaengine_submit(desc->desc);
 668         }
 669         dma_async_issue_pending(ctrl->dma_rx_channel);
 670
 671         return 0;
 672 }
 673
 674 static int qcom_slim_ngd_init_rx_msgq(struct qcom_slim_ngd_ctrl *ctrl)
 675 {
 676         struct device *dev = ctrl->dev;
 677         int ret, size;
 678
 679         ctrl->dma_rx_channel = dma_request_chan(dev, "rx");
 680         if (IS_ERR(ctrl->dma_rx_channel)) {
 681                 dev_err(dev, "Failed to request RX dma channel");
 682                 ret = PTR_ERR(ctrl->dma_rx_channel);
 683                 ctrl->dma_rx_channel = NULL;
 684                 return ret;
 685         }
 686
 687         size = QCOM_SLIM_NGD_DESC_NUM * SLIM_MSGQ_BUF_LEN;
 688         ctrl->rx_base = dma_alloc_coherent(dev, size, &ctrl->rx_phys_base,
 689                                            GFP_KERNEL);
 690         if (!ctrl->rx_base) {
 691                 ret = -ENOMEM;
 692                 goto rel_rx;
 693         }
 694
 695         ret = qcom_slim_ngd_post_rx_msgq(ctrl);
 696         if (ret) {
 697                 dev_err(dev, "post_rx_msgq() failed 0x%x\n", ret);
 698                 goto rx_post_err;
 699         }
 700
 701         return 0;
 702
 703 rx_post_err:
 704         dma_free_coherent(dev, size, ctrl->rx_base, ctrl->rx_phys_base);
 705 rel_rx:
 706         dma_release_channel(ctrl->dma_rx_channel);
 707         return ret;
 708 }
 709
 710 static int qcom_slim_ngd_init_tx_msgq(struct qcom_slim_ngd_ctrl *ctrl)
 711 {
 712         struct device *dev = ctrl->dev;
 713         unsigned long flags;
 714         int ret = 0;
 715         int size;
 716
 717         ctrl->dma_tx_channel = dma_request_chan(dev, "tx");
 718         if (IS_ERR(ctrl->dma_tx_channel)) {
 719                 dev_err(dev, "Failed to request TX dma channel");
 720                 ret = PTR_ERR(ctrl->dma_tx_channel);
 721                 ctrl->dma_tx_channel = NULL;
 722                 return ret;
 723         }
 724
 725         size = ((QCOM_SLIM_NGD_DESC_NUM + 1) * SLIM_MSGQ_BUF_LEN);
 726         ctrl->tx_base = dma_alloc_coherent(dev, size, &ctrl->tx_phys_base,
 727                                            GFP_KERNEL);
 728         if (!ctrl->tx_base) {
 729                 ret = -EINVAL;
 730                 goto rel_tx;
 731         }
 732
 733         spin_lock_irqsave(&ctrl->tx_buf_lock, flags);
 734         ctrl->tx_tail = 0;
 735         ctrl->tx_head = 0;
 736         spin_unlock_irqrestore(&ctrl->tx_buf_lock, flags);
 737
 738         return 0;
 739 rel_tx:
 740         dma_release_channel(ctrl->dma_tx_channel);
 741         return ret;
 742 }
 743
 744 static int qcom_slim_ngd_init_dma(struct qcom_slim_ngd_ctrl *ctrl)
 745 {
 746         int ret = 0;
 747
 748         ret = qcom_slim_ngd_init_rx_msgq(ctrl);
 749         if (ret) {
 750                 dev_err(ctrl->dev, "rx dma init failed\n");
 751                 return ret;
 752         }
 753
 754         ret = qcom_slim_ngd_init_tx_msgq(ctrl);
 755         if (ret)
 756                 dev_err(ctrl->dev, "tx dma init failed\n");
 757
 758         return ret;
 759 }
 760
 761 static irqreturn_t qcom_slim_ngd_interrupt(int irq, void *d)
 762 {
 763         struct qcom_slim_ngd_ctrl *ctrl = d;
 764         void __iomem *base = ctrl->ngd->base;
 765         u32 stat;
 766
 767         if (pm_runtime_suspended(ctrl->ctrl.dev)) {
 768                 dev_warn_once(ctrl->dev, "Interrupt received while suspended\n");
 769                 return IRQ_NONE;
 770         }
 771
 772         stat = readl(base + NGD_INT_STAT);
 773
 774         if ((stat & NGD_INT_MSG_BUF_CONTE) ||
 775                 (stat & NGD_INT_MSG_TX_INVAL) || (stat & NGD_INT_DEV_ERR) ||
 776                 (stat & NGD_INT_TX_NACKED_2)) {
 777                 dev_err(ctrl->dev, "Error Interrupt received 0x%x\n", stat);
 778         }
 779
 780         writel(stat, base + NGD_INT_CLR);
 781
 782         return IRQ_HANDLED;
 783 }
 784
 785 static int qcom_slim_ngd_xfer_msg(struct slim_controller *sctrl,
 786                                   struct slim_msg_txn *txn)
 787 {
 788         struct qcom_slim_ngd_ctrl *ctrl = dev_get_drvdata(sctrl->dev);
 789         DECLARE_COMPLETION_ONSTACK(tx_sent);
 790         DECLARE_COMPLETION_ONSTACK(done);
 791         int ret, timeout, i;
 792         u8 wbuf[SLIM_MSGQ_BUF_LEN];
 793         u8 rbuf[SLIM_MSGQ_BUF_LEN];
 794         u32 *pbuf;
 795         u8 *puc;
 796         u8 la = txn->la;
 797         bool usr_msg = false;
 798
 799         if (txn->mt == SLIM_MSG_MT_CORE &&
 800                 (txn->mc >= SLIM_MSG_MC_BEGIN_RECONFIGURATION &&
 801                  txn->mc <= SLIM_MSG_MC_RECONFIGURE_NOW))
 802                 return 0;
 803
 804         if (txn->dt == SLIM_MSG_DEST_ENUMADDR)
 805                 return -EPROTONOSUPPORT;
 806
 807         if (txn->msg->num_bytes > SLIM_MSGQ_BUF_LEN ||
 808                         txn->rl > SLIM_MSGQ_BUF_LEN) {
 809                 dev_err(ctrl->dev, "msg exceeds HW limit\n");
 810                 return -EINVAL;
 811         }
 812
 813         pbuf = qcom_slim_ngd_tx_msg_get(ctrl, txn->rl, &tx_sent);
 814         if (!pbuf) {
 815                 dev_err(ctrl->dev, "Message buffer unavailable\n");
 816                 return -ENOMEM;
 817         }
 818
 819         if (txn->mt == SLIM_MSG_MT_CORE &&
 820                 (txn->mc == SLIM_MSG_MC_CONNECT_SOURCE ||
 821                 txn->mc == SLIM_MSG_MC_CONNECT_SINK ||
 822                 txn->mc == SLIM_MSG_MC_DISCONNECT_PORT)) {
 823                 txn->mt = SLIM_MSG_MT_DEST_REFERRED_USER;
 824                 switch (txn->mc) {
 825                 case SLIM_MSG_MC_CONNECT_SOURCE:
 826                         txn->mc = SLIM_USR_MC_CONNECT_SRC;
 827                         break;
 828                 case SLIM_MSG_MC_CONNECT_SINK:
 829                         txn->mc = SLIM_USR_MC_CONNECT_SINK;
 830                         break;
 831                 case SLIM_MSG_MC_DISCONNECT_PORT:
 832                         txn->mc = SLIM_USR_MC_DISCONNECT_PORT;
 833                         break;
 834                 default:
 835                         return -EINVAL;
 836                 }
 837
 838                 usr_msg = true;
 839                 i = 0;
 840                 wbuf[i++] = txn->la;
 841                 la = SLIM_LA_MGR;
 842                 wbuf[i++] = txn->msg->wbuf[0];
 843                 if (txn->mc != SLIM_USR_MC_DISCONNECT_PORT)
 844                         wbuf[i++] = txn->msg->wbuf[1];
 845
 846                 txn->comp = &done;
 847                 ret = slim_alloc_txn_tid(sctrl, txn);
 848                 if (ret) {
 849                         dev_err(ctrl->dev, "Unable to allocate TID\n");
 850                         return ret;
 851                 }
 852
 853                 wbuf[i++] = txn->tid;
 854
 855                 txn->msg->num_bytes = i;
 856                 txn->msg->wbuf = wbuf;
 857                 txn->msg->rbuf = rbuf;
 858                 txn->rl = txn->msg->num_bytes + 4;
 859         }
 860
 861         /* HW expects length field to be excluded */
 862         txn->rl--;
 863         puc = (u8 *)pbuf;
 864         *pbuf = 0;
 865         if (txn->dt == SLIM_MSG_DEST_LOGICALADDR) {
 866                 *pbuf = SLIM_MSG_ASM_FIRST_WORD(txn->rl, txn->mt, txn->mc, 0,
 867                                 la);
 868                 puc += 3;
 869         } else {
 870                 *pbuf = SLIM_MSG_ASM_FIRST_WORD(txn->rl, txn->mt, txn->mc, 1,
 871                                 la);
 872                 puc += 2;
 873         }
 874
 875         if (slim_tid_txn(txn->mt, txn->mc))
 876                 *(puc++) = txn->tid;
 877
 878         if (slim_ec_txn(txn->mt, txn->mc)) {
 879                 *(puc++) = (txn->ec & 0xFF);
 880                 *(puc++) = (txn->ec >> 8) & 0xFF;
 881         }
 882
 883         if (txn->msg && txn->msg->wbuf)
 884                 memcpy(puc, txn->msg->wbuf, txn->msg->num_bytes);
 885
 886         mutex_lock(&ctrl->tx_lock);
 887         ret = qcom_slim_ngd_tx_msg_post(ctrl, pbuf, txn->rl);
 888         if (ret) {
 889                 mutex_unlock(&ctrl->tx_lock);
 890                 return ret;
 891         }
 892
 893         timeout = wait_for_completion_timeout(&tx_sent, HZ);
 894         if (!timeout) {
 895                 dev_err(sctrl->dev, "TX timed out:MC:0x%x,mt:0x%x", txn->mc,
 896                                         txn->mt);
 897                 mutex_unlock(&ctrl->tx_lock);
 898                 return -ETIMEDOUT;
 899         }
 900
 901         if (usr_msg) {
 902                 timeout = wait_for_completion_timeout(&done, HZ);
 903                 if (!timeout) {
 904                         dev_err(sctrl->dev, "TX timed out:MC:0x%x,mt:0x%x",
 905                                 txn->mc, txn->mt);
 906                         mutex_unlock(&ctrl->tx_lock);
 907                         return -ETIMEDOUT;
 908                 }
 909         }
 910
 911         mutex_unlock(&ctrl->tx_lock);
 912         return 0;
 913 }
 914
 915 static int qcom_slim_ngd_xfer_msg_sync(struct slim_controller *ctrl,
 916                                        struct slim_msg_txn *txn)
 917 {
 918         DECLARE_COMPLETION_ONSTACK(done);
 919         int ret, timeout;
 920
 921         ret = pm_runtime_get_sync(ctrl->dev);
 922         if (ret < 0)
 923                 goto pm_put;
 924
 925         txn->comp = &done;
 926
 927         ret = qcom_slim_ngd_xfer_msg(ctrl, txn);
 928         if (ret)
 929                 goto pm_put;
 930
 931         timeout = wait_for_completion_timeout(&done, HZ);
 932         if (!timeout) {
 933                 dev_err(ctrl->dev, "TX timed out:MC:0x%x,mt:0x%x", txn->mc,
 934                                 txn->mt);
 935                 ret = -ETIMEDOUT;
 936                 goto pm_put;
 937         }
 938         return 0;
 939
 940 pm_put:
 941         pm_runtime_put(ctrl->dev);
 942
 943         return ret;
 944 }
 945
 946 static int qcom_slim_calc_coef(struct slim_stream_runtime *rt, int *exp)
 947 {
 948         struct slim_controller *ctrl = rt->dev->ctrl;
 949         int coef;
 950
 951         if (rt->ratem * ctrl->a_framer->superfreq < rt->rate)
 952                 rt->ratem++;
 953
 954         coef = rt->ratem;
 955         *exp = 0;
 956
 957         /*
 958          * CRM = Cx(2^E) is the formula we are using.
 959          * Here C is the coffecient and E is the exponent.
 960          * CRM is the Channel Rate Multiplier.
 961          * Coefficeint should be either 1 or 3 and exponenet
 962          * should be an integer between 0 to 9, inclusive.
 963          */
 964         while (1) {
 965                 while ((coef & 0x1) != 0x1) {
 966                         coef >>= 1;
 967                         *exp = *exp + 1;
 968                 }
 969
 970                 if (coef <= 3)
 971                         break;
 972
 973                 coef++;
 974         }
 975
 976         /*
 977          * we rely on the coef value (1 or 3) to set a bit
 978          * in the slimbus message packet. This bit is
 979          * BIT(5) which is the segment rate coefficient.
 980          */
 981         if (coef == 1) {
 982                 if (*exp > 9)
 983                         return -EIO;
 984                 coef = 0;
 985         } else {
 986                 if (*exp > 8)
 987                         return -EIO;
 988                 coef = 1;
 989         }
 990
 991         return coef;
 992 }
 993
 994 static int qcom_slim_ngd_enable_stream(struct slim_stream_runtime *rt)
 995 {
 996         struct slim_device *sdev = rt->dev;
 997         struct slim_controller *ctrl = sdev->ctrl;
 998         struct slim_val_inf msg =  {0};
 999         u8 wbuf[SLIM_MSGQ_BUF_LEN];
1000         u8 rbuf[SLIM_MSGQ_BUF_LEN];
1001         struct slim_msg_txn txn = {0,};
1002         int i, ret;
1003
1004         txn.mt = SLIM_MSG_MT_DEST_REFERRED_USER;
1005         txn.dt = SLIM_MSG_DEST_LOGICALADDR;
1006         txn.la = SLIM_LA_MGR;
1007         txn.ec = 0;
1008         txn.msg = &msg;
1009         txn.msg->num_bytes = 0;
1010         txn.msg->wbuf = wbuf;
1011         txn.msg->rbuf = rbuf;
1012
1013         for (i = 0; i < rt->num_ports; i++) {
1014                 struct slim_port *port = &rt->ports[i];
1015
1016                 if (txn.msg->num_bytes == 0) {
1017                         int exp = 0, coef = 0;
1018
1019                         wbuf[txn.msg->num_bytes++] = sdev->laddr;
1020                         wbuf[txn.msg->num_bytes] = rt->bps >> 2 |
1021                                                    (port->ch.aux_fmt << 6);
1022
1023                         /* calculate coef dynamically */
1024                         coef = qcom_slim_calc_coef(rt, &exp);
1025                         if (coef < 0) {
1026                                 dev_err(&sdev->dev,
1027                                 "%s: error calculating coef %d\n", __func__,
1028                                                                         coef);
1029                                 return -EIO;
1030                         }
1031
1032                         if (coef)
1033                                 wbuf[txn.msg->num_bytes] |= BIT(5);
1034
1035                         txn.msg->num_bytes++;
1036                         wbuf[txn.msg->num_bytes++] = exp << 4 | rt->prot;
1037
1038                         if (rt->prot == SLIM_PROTO_ISO)
1039                                 wbuf[txn.msg->num_bytes++] =
1040                                                 port->ch.prrate |
1041                                                 SLIM_CHANNEL_CONTENT_FL;
1042                         else
1043                                 wbuf[txn.msg->num_bytes++] =  port->ch.prrate;
1044
1045                         ret = slim_alloc_txn_tid(ctrl, &txn);
1046                         if (ret) {
1047                                 dev_err(&sdev->dev, "Fail to allocate TID\n");
1048                                 return -ENXIO;
1049                         }
1050                         wbuf[txn.msg->num_bytes++] = txn.tid;
1051                 }
1052                 wbuf[txn.msg->num_bytes++] = port->ch.id;
1053         }
1054
1055         txn.mc = SLIM_USR_MC_DEF_ACT_CHAN;
1056         txn.rl = txn.msg->num_bytes + 4;
1057         ret = qcom_slim_ngd_xfer_msg_sync(ctrl, &txn);
1058         if (ret) {
1059                 slim_free_txn_tid(ctrl, &txn);
1060                 dev_err(&sdev->dev, "TX timed out:MC:0x%x,mt:0x%x", txn.mc,
1061                                 txn.mt);
1062                 return ret;
1063         }
1064
1065         txn.mc = SLIM_USR_MC_RECONFIG_NOW;
1066         txn.msg->num_bytes = 2;
1067         wbuf[1] = sdev->laddr;
1068         txn.rl = txn.msg->num_bytes + 4;
1069
1070         ret = slim_alloc_txn_tid(ctrl, &txn);
1071         if (ret) {
1072                 dev_err(ctrl->dev, "Fail to allocate TID\n");
1073                 return ret;
1074         }
1075
1076         wbuf[0] = txn.tid;
1077         ret = qcom_slim_ngd_xfer_msg_sync(ctrl, &txn);
1078         if (ret) {
1079                 slim_free_txn_tid(ctrl, &txn);
1080                 dev_err(&sdev->dev, "TX timed out:MC:0x%x,mt:0x%x", txn.mc,
1081                                 txn.mt);
1082         }
1083
1084         return ret;
1085 }
1086
1087 static int qcom_slim_ngd_get_laddr(struct slim_controller *ctrl,
1088                                    struct slim_eaddr *ea, u8 *laddr)
1089 {
1090         struct slim_val_inf msg =  {0};
1091         u8 failed_ea[6] = {0, 0, 0, 0, 0, 0};
1092         struct slim_msg_txn txn;
1093         u8 wbuf[10] = {0};
1094         u8 rbuf[10] = {0};
1095         int ret;
1096
1097         txn.mt = SLIM_MSG_MT_DEST_REFERRED_USER;
1098         txn.dt = SLIM_MSG_DEST_LOGICALADDR;
1099         txn.la = SLIM_LA_MGR;
1100         txn.ec = 0;
1101
1102         txn.mc = SLIM_USR_MC_ADDR_QUERY;
1103         txn.rl = 11;
1104         txn.msg = &msg;
1105         txn.msg->num_bytes = 7;
1106         txn.msg->wbuf = wbuf;
1107         txn.msg->rbuf = rbuf;
1108
1109         ret = slim_alloc_txn_tid(ctrl, &txn);
1110         if (ret < 0)
1111                 return ret;
1112
1113         wbuf[0] = (u8)txn.tid;
1114         memcpy(&wbuf[1], ea, sizeof(*ea));
1115
1116         ret = qcom_slim_ngd_xfer_msg_sync(ctrl, &txn);
1117         if (ret) {
1118                 slim_free_txn_tid(ctrl, &txn);
1119                 return ret;
1120         }
1121
1122         if (!memcmp(rbuf, failed_ea, 6))
1123                 return -ENXIO;
1124
1125         *laddr = rbuf[6];
1126
1127         return ret;
1128 }
1129
1130 static int qcom_slim_ngd_exit_dma(struct qcom_slim_ngd_ctrl *ctrl)
1131 {
1132         if (ctrl->dma_rx_channel) {
1133                 dmaengine_terminate_sync(ctrl->dma_rx_channel);
1134                 dma_release_channel(ctrl->dma_rx_channel);
1135         }
1136
1137         if (ctrl->dma_tx_channel) {
1138                 dmaengine_terminate_sync(ctrl->dma_tx_channel);
1139                 dma_release_channel(ctrl->dma_tx_channel);
1140         }
1141
1142         ctrl->dma_tx_channel = ctrl->dma_rx_channel = NULL;
1143
1144         return 0;
1145 }
1146
1147 static void qcom_slim_ngd_setup(struct qcom_slim_ngd_ctrl *ctrl)
1148 {
1149         u32 cfg = readl_relaxed(ctrl->ngd->base);
1150
1151         if (ctrl->state == QCOM_SLIM_NGD_CTRL_DOWN ||
1152                 ctrl->state == QCOM_SLIM_NGD_CTRL_ASLEEP)
1153                 qcom_slim_ngd_init_dma(ctrl);
1154
1155         /* By default enable message queues */
1156         cfg |= NGD_CFG_RX_MSGQ_EN;
1157         cfg |= NGD_CFG_TX_MSGQ_EN;
1158
1159         /* Enable NGD if it's not already enabled*/
1160         if (!(cfg & NGD_CFG_ENABLE))
1161                 cfg |= NGD_CFG_ENABLE;
1162
1163         writel_relaxed(cfg, ctrl->ngd->base);
1164 }
1165
1166 static int qcom_slim_ngd_power_up(struct qcom_slim_ngd_ctrl *ctrl)
1167 {
1168         enum qcom_slim_ngd_state cur_state = ctrl->state;
1169         struct qcom_slim_ngd *ngd = ctrl->ngd;
1170         u32 laddr, rx_msgq;
1171         int timeout, ret = 0;
1172
1173         if (ctrl->state == QCOM_SLIM_NGD_CTRL_DOWN) {
1174                 timeout = wait_for_completion_timeout(&ctrl->qmi.qmi_comp, HZ);
1175                 if (!timeout)
1176                         return -EREMOTEIO;
1177         }
1178
1179         if (ctrl->state == QCOM_SLIM_NGD_CTRL_ASLEEP ||
1180                 ctrl->state == QCOM_SLIM_NGD_CTRL_DOWN) {
1181                 ret = qcom_slim_qmi_power_request(ctrl, true);
1182                 if (ret) {
1183                         dev_err(ctrl->dev, "SLIM QMI power request failed:%d\n",
1184                                         ret);
1185                         return ret;
1186                 }
1187         }
1188
1189         ctrl->ver = readl_relaxed(ctrl->base);
1190         /* Version info in 16 MSbits */
1191         ctrl->ver >>= 16;
1192
1193         laddr = readl_relaxed(ngd->base + NGD_STATUS);
1194         if (laddr & NGD_LADDR) {
1195                 /*
1196                  * external MDM restart case where ADSP itself was active framer
1197                  * For example, modem restarted when playback was active
1198                  */
1199                 if (cur_state == QCOM_SLIM_NGD_CTRL_AWAKE) {
1200                         dev_info(ctrl->dev, "Subsys restart: ADSP active framer\n");
1201                         return 0;
1202                 }
1203                 qcom_slim_ngd_setup(ctrl);
1204                 return 0;
1205         }
1206
1207         /*
1208          * Reinitialize only when registers are not retained or when enumeration
1209          * is lost for ngd.
1210          */
1211         reinit_completion(&ctrl->reconf);
1212
1213         writel_relaxed(DEF_NGD_INT_MASK, ngd->base + NGD_INT_EN);
1214         rx_msgq = readl_relaxed(ngd->base + NGD_RX_MSGQ_CFG);
1215
1216         writel_relaxed(rx_msgq|SLIM_RX_MSGQ_TIMEOUT_VAL,
1217                                 ngd->base + NGD_RX_MSGQ_CFG);
1218         qcom_slim_ngd_setup(ctrl);
1219
1220         timeout = wait_for_completion_timeout(&ctrl->reconf, HZ);
1221         if (!timeout) {
1222                 dev_err(ctrl->dev, "capability exchange timed-out\n");
1223                 return -ETIMEDOUT;
1224         }
1225
1226         return 0;
1227 }
1228
1229 static void qcom_slim_ngd_notify_slaves(struct qcom_slim_ngd_ctrl *ctrl)
1230 {
1231         struct slim_device *sbdev;
1232         struct device_node *node;
1233
1234         for_each_child_of_node(ctrl->ngd->pdev->dev.of_node, node) {
1235                 sbdev = of_slim_get_device(&ctrl->ctrl, node);
1236                 if (!sbdev)
1237                         continue;
1238
1239                 if (slim_get_logical_addr(sbdev))
1240                         dev_err(ctrl->dev, "Failed to get logical address\n");
1241         }
1242 }
1243
1244 static void qcom_slim_ngd_master_worker(struct work_struct *work)
1245 {
1246         struct qcom_slim_ngd_ctrl *ctrl;
1247         struct slim_msg_txn txn;
1248         struct slim_val_inf msg = {0};
1249         int retries = 0;
1250         u8 wbuf[8];
1251         int ret = 0;
1252
1253         ctrl = container_of(work, struct qcom_slim_ngd_ctrl, m_work);
1254         txn.dt = SLIM_MSG_DEST_LOGICALADDR;
1255         txn.ec = 0;
1256         txn.mc = SLIM_USR_MC_REPORT_SATELLITE;
1257         txn.mt = SLIM_MSG_MT_SRC_REFERRED_USER;
1258         txn.la = SLIM_LA_MGR;
1259         wbuf[0] = SAT_MAGIC_LSB;
1260         wbuf[1] = SAT_MAGIC_MSB;
1261         wbuf[2] = SAT_MSG_VER;
1262         wbuf[3] = SAT_MSG_PROT;
1263         txn.msg = &msg;
1264         txn.msg->wbuf = wbuf;
1265         txn.msg->num_bytes = 4;
1266         txn.rl = 8;
1267
1268         dev_info(ctrl->dev, "SLIM SAT: Rcvd master capability\n");
1269
1270 capability_retry:
1271         ret = qcom_slim_ngd_xfer_msg(&ctrl->ctrl, &txn);
1272         if (!ret) {
1273                 if (ctrl->state >= QCOM_SLIM_NGD_CTRL_ASLEEP)
1274                         complete(&ctrl->reconf);
1275                 else
1276                         dev_err(ctrl->dev, "unexpected state:%d\n",
1277                                                 ctrl->state);
1278
1279                 if (ctrl->state == QCOM_SLIM_NGD_CTRL_DOWN)
1280                         qcom_slim_ngd_notify_slaves(ctrl);
1281
1282         } else if (ret == -EIO) {
1283                 dev_err(ctrl->dev, "capability message NACKed, retrying\n");
1284                 if (retries < INIT_MX_RETRIES) {
1285                         msleep(DEF_RETRY_MS);
1286                         retries++;
1287                         goto capability_retry;
1288                 }
1289         } else {
1290                 dev_err(ctrl->dev, "SLIM: capability TX failed:%d\n", ret);
1291         }
1292 }
1293
1294 static int qcom_slim_ngd_update_device_status(struct device *dev, void *null)
1295 {
1296         slim_report_absent(to_slim_device(dev));
1297
1298         return 0;
1299 }
1300
1301 static int qcom_slim_ngd_runtime_resume(struct device *dev)
1302 {
1303         struct qcom_slim_ngd_ctrl *ctrl = dev_get_drvdata(dev);
1304         int ret = 0;
1305
1306         if (!ctrl->qmi.handle)
1307                 return 0;
1308
1309         if (ctrl->state >= QCOM_SLIM_NGD_CTRL_ASLEEP)
1310                 ret = qcom_slim_ngd_power_up(ctrl);
1311         if (ret) {
1312                 /* Did SSR cause this power up failure */
1313                 if (ctrl->state != QCOM_SLIM_NGD_CTRL_DOWN)
1314                         ctrl->state = QCOM_SLIM_NGD_CTRL_ASLEEP;
1315                 else
1316                         dev_err(ctrl->dev, "HW wakeup attempt during SSR\n");
1317         } else {
1318                 ctrl->state = QCOM_SLIM_NGD_CTRL_AWAKE;
1319         }
1320
1321         return 0;
1322 }
1323
1324 static int qcom_slim_ngd_enable(struct qcom_slim_ngd_ctrl *ctrl, bool enable)
1325 {
1326         if (enable) {
1327                 int ret = qcom_slim_qmi_init(ctrl, false);
1328
1329                 if (ret) {
1330                         dev_err(ctrl->dev, "qmi init fail, ret:%d, state:%d\n",
1331                                 ret, ctrl->state);
1332                         return ret;
1333                 }
1334                 /* controller state should be in sync with framework state */
1335                 complete(&ctrl->qmi.qmi_comp);
1336                 if (!pm_runtime_enabled(ctrl->ctrl.dev) ||
1337                          !pm_runtime_suspended(ctrl->ctrl.dev))
1338                         qcom_slim_ngd_runtime_resume(ctrl->ctrl.dev);
1339                 else
1340                         pm_runtime_resume(ctrl->ctrl.dev);
1341
1342                 pm_runtime_mark_last_busy(ctrl->ctrl.dev);
1343                 pm_runtime_put(ctrl->ctrl.dev);
1344
1345                 ret = slim_register_controller(&ctrl->ctrl);
1346                 if (ret) {
1347                         dev_err(ctrl->dev, "error adding slim controller\n");
1348                         return ret;
1349                 }
1350
1351                 dev_info(ctrl->dev, "SLIM controller Registered\n");
1352         } else {
1353                 qcom_slim_qmi_exit(ctrl);
1354                 slim_unregister_controller(&ctrl->ctrl);
1355         }
1356
1357         return 0;
1358 }
1359
1360 static int qcom_slim_ngd_qmi_new_server(struct qmi_handle *hdl,
1361                                         struct qmi_service *service)
1362 {
1363         struct qcom_slim_ngd_qmi *qmi =
1364                 container_of(hdl, struct qcom_slim_ngd_qmi, svc_event_hdl);
1365         struct qcom_slim_ngd_ctrl *ctrl =
1366                 container_of(qmi, struct qcom_slim_ngd_ctrl, qmi);
1367
1368         qmi->svc_info.sq_family = AF_QIPCRTR;
1369         qmi->svc_info.sq_node = service->node;
1370         qmi->svc_info.sq_port = service->port;
1371
1372         complete(&ctrl->qmi_up);
1373
1374         return 0;
1375 }
1376
1377 static void qcom_slim_ngd_qmi_del_server(struct qmi_handle *hdl,
1378                                          struct qmi_service *service)
1379 {
1380         struct qcom_slim_ngd_qmi *qmi =
1381                 container_of(hdl, struct qcom_slim_ngd_qmi, svc_event_hdl);
1382         struct qcom_slim_ngd_ctrl *ctrl =
1383                 container_of(qmi, struct qcom_slim_ngd_ctrl, qmi);
1384
1385         reinit_completion(&ctrl->qmi_up);
1386         qmi->svc_info.sq_node = 0;
1387         qmi->svc_info.sq_port = 0;
1388 }
1389
1390 static const struct qmi_ops qcom_slim_ngd_qmi_svc_event_ops = {
1391         .new_server = qcom_slim_ngd_qmi_new_server,
1392         .del_server = qcom_slim_ngd_qmi_del_server,
1393 };
1394
1395 static int qcom_slim_ngd_qmi_svc_event_init(struct qcom_slim_ngd_ctrl *ctrl)
1396 {
1397         struct qcom_slim_ngd_qmi *qmi = &ctrl->qmi;
1398         int ret;
1399
1400         ret = qmi_handle_init(&qmi->svc_event_hdl, 0,
1401                                 &qcom_slim_ngd_qmi_svc_event_ops, NULL);
1402         if (ret < 0) {
1403                 dev_err(ctrl->dev, "qmi_handle_init failed: %d\n", ret);
1404                 return ret;
1405         }
1406
1407         ret = qmi_add_lookup(&qmi->svc_event_hdl, SLIMBUS_QMI_SVC_ID,
1408                         SLIMBUS_QMI_SVC_V1, SLIMBUS_QMI_INS_ID);
1409         if (ret < 0) {
1410                 dev_err(ctrl->dev, "qmi_add_lookup failed: %d\n", ret);
1411                 qmi_handle_release(&qmi->svc_event_hdl);
1412         }
1413         return ret;
1414 }
1415
1416 static void qcom_slim_ngd_qmi_svc_event_deinit(struct qcom_slim_ngd_qmi *qmi)
1417 {
1418         qmi_handle_release(&qmi->svc_event_hdl);
1419 }
1420
1421 static struct platform_driver qcom_slim_ngd_driver;
1422 #define QCOM_SLIM_NGD_DRV_NAME  "qcom,slim-ngd"
1423
1424 static const struct of_device_id qcom_slim_ngd_dt_match[] = {
1425         {
1426                 .compatible = "qcom,slim-ngd-v1.5.0",
1427                 .data = &ngd_v1_5_offset_info,
1428         },{
1429                 .compatible = "qcom,slim-ngd-v2.1.0",
1430                 .data = &ngd_v1_5_offset_info,
1431         },
1432         {}
1433 };
1434
1435 MODULE_DEVICE_TABLE(of, qcom_slim_ngd_dt_match);
1436
1437 static void qcom_slim_ngd_down(struct qcom_slim_ngd_ctrl *ctrl)
1438 {
1439         mutex_lock(&ctrl->ssr_lock);
1440         device_for_each_child(ctrl->ctrl.dev, NULL,
1441                               qcom_slim_ngd_update_device_status);
1442         qcom_slim_ngd_enable(ctrl, false);
1443         mutex_unlock(&ctrl->ssr_lock);
1444 }
1445
1446 static void qcom_slim_ngd_up_worker(struct work_struct *work)
1447 {
1448         struct qcom_slim_ngd_ctrl *ctrl;
1449
1450         ctrl = container_of(work, struct qcom_slim_ngd_ctrl, ngd_up_work);
1451
1452         /* Make sure qmi service is up before continuing */
1453         if (!wait_for_completion_interruptible_timeout(&ctrl->qmi_up,
1454                                                        msecs_to_jiffies(MSEC_PER_SEC))) {
1455                 dev_err(ctrl->dev, "QMI wait timeout\n");
1456                 return;
1457         }
1458
1459         mutex_lock(&ctrl->ssr_lock);
1460         qcom_slim_ngd_enable(ctrl, true);
1461         mutex_unlock(&ctrl->ssr_lock);
1462 }
1463
1464 static int qcom_slim_ngd_ssr_pdr_notify(struct qcom_slim_ngd_ctrl *ctrl,
1465                                         unsigned long action)
1466 {
1467         switch (action) {
1468         case QCOM_SSR_BEFORE_SHUTDOWN:
1469         case SERVREG_SERVICE_STATE_DOWN:
1470                 /* Make sure the last dma xfer is finished */
1471                 mutex_lock(&ctrl->tx_lock);
1472                 if (ctrl->state != QCOM_SLIM_NGD_CTRL_DOWN) {
1473                         pm_runtime_get_noresume(ctrl->ctrl.dev);
1474                         ctrl->state = QCOM_SLIM_NGD_CTRL_DOWN;
1475                         qcom_slim_ngd_down(ctrl);
1476                         qcom_slim_ngd_exit_dma(ctrl);
1477                 }
1478                 mutex_unlock(&ctrl->tx_lock);
1479                 break;
1480         case QCOM_SSR_AFTER_POWERUP:
1481         case SERVREG_SERVICE_STATE_UP:
1482                 schedule_work(&ctrl->ngd_up_work);
1483                 break;
1484         default:
1485                 break;
1486         }
1487
1488         return NOTIFY_OK;
1489 }
1490
1491 static int qcom_slim_ngd_ssr_notify(struct notifier_block *nb,
1492                                     unsigned long action,
1493                                     void *data)
1494 {
1495         struct qcom_slim_ngd_ctrl *ctrl = container_of(nb,
1496                                                struct qcom_slim_ngd_ctrl, nb);
1497
1498         return qcom_slim_ngd_ssr_pdr_notify(ctrl, action);
1499 }
1500
1501 static void slim_pd_status(int state, char *svc_path, void *priv)
1502 {
1503         struct qcom_slim_ngd_ctrl *ctrl = (struct qcom_slim_ngd_ctrl *)priv;
1504
1505         qcom_slim_ngd_ssr_pdr_notify(ctrl, state);
1506 }
1507 static int of_qcom_slim_ngd_register(struct device *parent,
1508                                      struct qcom_slim_ngd_ctrl *ctrl)
1509 {
1510         const struct ngd_reg_offset_data *data;
1511         struct qcom_slim_ngd *ngd;
1512         const struct of_device_id *match;
1513         struct device_node *node;
1514         u32 id;
1515         int ret;
1516
1517         match = of_match_node(qcom_slim_ngd_dt_match, parent->of_node);
1518         data = match->data;
1519         for_each_available_child_of_node(parent->of_node, node) {
1520                 if (of_property_read_u32(node, "reg", &id))
1521                         continue;
1522
1523                 ngd = kzalloc(sizeof(*ngd), GFP_KERNEL);
1524                 if (!ngd) {
1525                         of_node_put(node);
1526                         return -ENOMEM;
1527                 }
1528
1529                 ngd->pdev = platform_device_alloc(QCOM_SLIM_NGD_DRV_NAME, id);
1530                 if (!ngd->pdev) {
1531                         kfree(ngd);
1532                         of_node_put(node);
1533                         return -ENOMEM;
1534                 }
1535                 ngd->id = id;
1536                 ngd->pdev->dev.parent = parent;
1537
1538                 ret = driver_set_override(&ngd->pdev->dev,
1539                                           &ngd->pdev->driver_override,
1540                                           QCOM_SLIM_NGD_DRV_NAME,
1541                                           strlen(QCOM_SLIM_NGD_DRV_NAME));
1542                 if (ret) {
1543                         platform_device_put(ngd->pdev);
1544                         kfree(ngd);
1545                         of_node_put(node);
1546                         return ret;
1547                 }
1548                 ngd->pdev->dev.of_node = node;
1549                 ctrl->ngd = ngd;
1550
1551                 ret = platform_device_add(ngd->pdev);
1552                 if (ret) {
1553                         platform_device_put(ngd->pdev);
1554                         kfree(ngd);
1555                         of_node_put(node);
1556                         return ret;
1557                 }
1558                 ngd->base = ctrl->base + ngd->id * data->offset +
1559                                         (ngd->id - 1) * data->size;
1560
1561                 return 0;
1562         }
1563
1564         return -ENODEV;
1565 }
1566
1567 static int qcom_slim_ngd_probe(struct platform_device *pdev)
1568 {
1569         struct device *dev = &pdev->dev;
1570         struct qcom_slim_ngd_ctrl *ctrl = dev_get_drvdata(dev->parent);
1571         int ret;
1572
1573         ctrl->ctrl.dev = dev;
1574
1575         platform_set_drvdata(pdev, ctrl);
1576         pm_runtime_use_autosuspend(dev);
1577         pm_runtime_set_autosuspend_delay(dev, 100);
1578         pm_runtime_set_suspended(dev);
1579         pm_runtime_enable(dev);
1580         pm_runtime_get_noresume(dev);
1581         ret = qcom_slim_ngd_qmi_svc_event_init(ctrl);
1582         if (ret) {
1583                 dev_err(&pdev->dev, "QMI service registration failed:%d", ret);
1584                 return ret;
1585         }
1586
1587         INIT_WORK(&ctrl->m_work, qcom_slim_ngd_master_worker);
1588         INIT_WORK(&ctrl->ngd_up_work, qcom_slim_ngd_up_worker);
1589         ctrl->mwq = create_singlethread_workqueue("ngd_master");
1590         if (!ctrl->mwq) {
1591                 dev_err(&pdev->dev, "Failed to start master worker\n");
1592                 ret = -ENOMEM;
1593                 goto wq_err;
1594         }
1595
1596         return 0;
1597 wq_err:
1598         qcom_slim_ngd_qmi_svc_event_deinit(&ctrl->qmi);
1599         if (ctrl->mwq)
1600                 destroy_workqueue(ctrl->mwq);
1601
1602         return ret;
1603 }
1604
1605 static int qcom_slim_ngd_ctrl_probe(struct platform_device *pdev)
1606 {
1607         struct device *dev = &pdev->dev;
1608         struct qcom_slim_ngd_ctrl *ctrl;
1609         int ret;
1610         struct pdr_service *pds;
1611
1612         ctrl = devm_kzalloc(dev, sizeof(*ctrl), GFP_KERNEL);
1613         if (!ctrl)
1614                 return -ENOMEM;
1615
1616         dev_set_drvdata(dev, ctrl);
1617
1618         ctrl->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
1619         if (IS_ERR(ctrl->base))
1620                 return PTR_ERR(ctrl->base);
1621
1622         ret = platform_get_irq(pdev, 0);
1623         if (ret < 0)
1624                 return ret;
1625
1626         ret = devm_request_irq(dev, ret, qcom_slim_ngd_interrupt,
1627                                IRQF_TRIGGER_HIGH, "slim-ngd", ctrl);
1628         if (ret)
1629                 return dev_err_probe(&pdev->dev, ret, "request IRQ failed\n");
1630
1631         ctrl->nb.notifier_call = qcom_slim_ngd_ssr_notify;
1632         ctrl->notifier = qcom_register_ssr_notifier("lpass", &ctrl->nb);
1633         if (IS_ERR(ctrl->notifier))
1634                 return PTR_ERR(ctrl->notifier);
1635
1636         ctrl->dev = dev;
1637         ctrl->framer.rootfreq = SLIM_ROOT_FREQ >> 3;
1638         ctrl->framer.superfreq =
1639                 ctrl->framer.rootfreq / SLIM_CL_PER_SUPERFRAME_DIV8;
1640
1641         ctrl->ctrl.a_framer = &ctrl->framer;
1642         ctrl->ctrl.clkgear = SLIM_MAX_CLK_GEAR;
1643         ctrl->ctrl.get_laddr = qcom_slim_ngd_get_laddr;
1644         ctrl->ctrl.enable_stream = qcom_slim_ngd_enable_stream;
1645         ctrl->ctrl.xfer_msg = qcom_slim_ngd_xfer_msg;
1646         ctrl->ctrl.wakeup = NULL;
1647         ctrl->state = QCOM_SLIM_NGD_CTRL_DOWN;
1648
1649         mutex_init(&ctrl->tx_lock);
1650         mutex_init(&ctrl->ssr_lock);
1651         spin_lock_init(&ctrl->tx_buf_lock);
1652         init_completion(&ctrl->reconf);
1653         init_completion(&ctrl->qmi.qmi_comp);
1654         init_completion(&ctrl->qmi_up);
1655
1656         ctrl->pdr = pdr_handle_alloc(slim_pd_status, ctrl);
1657         if (IS_ERR(ctrl->pdr)) {
1658                 ret = dev_err_probe(dev, PTR_ERR(ctrl->pdr),
1659                                     "Failed to init PDR handle\n");
1660                 goto err_pdr_alloc;
1661         }
1662
1663         pds = pdr_add_lookup(ctrl->pdr, "avs/audio", "msm/adsp/audio_pd");
1664         if (IS_ERR(pds) && PTR_ERR(pds) != -EALREADY) {
1665                 ret = dev_err_probe(dev, PTR_ERR(pds), "pdr add lookup failed\n");
1666                 goto err_pdr_lookup;
1667         }
1668
1669         platform_driver_register(&qcom_slim_ngd_driver);
1670         return of_qcom_slim_ngd_register(dev, ctrl);
1671
1672 err_pdr_alloc:
1673         qcom_unregister_ssr_notifier(ctrl->notifier, &ctrl->nb);
1674
1675 err_pdr_lookup:
1676         pdr_handle_release(ctrl->pdr);
1677
1678         return ret;
1679 }
1680
1681 static void qcom_slim_ngd_ctrl_remove(struct platform_device *pdev)
1682 {
1683         platform_driver_unregister(&qcom_slim_ngd_driver);
1684 }
1685
1686 static void qcom_slim_ngd_remove(struct platform_device *pdev)
1687 {
1688         struct qcom_slim_ngd_ctrl *ctrl = platform_get_drvdata(pdev);
1689
1690         pm_runtime_disable(&pdev->dev);
1691         pdr_handle_release(ctrl->pdr);
1692         qcom_unregister_ssr_notifier(ctrl->notifier, &ctrl->nb);
1693         qcom_slim_ngd_enable(ctrl, false);
1694         qcom_slim_ngd_exit_dma(ctrl);
1695         qcom_slim_ngd_qmi_svc_event_deinit(&ctrl->qmi);
1696         if (ctrl->mwq)
1697                 destroy_workqueue(ctrl->mwq);
1698
1699         kfree(ctrl->ngd);
1700         ctrl->ngd = NULL;
1701 }
1702
1703 static int __maybe_unused qcom_slim_ngd_runtime_idle(struct device *dev)
1704 {
1705         struct qcom_slim_ngd_ctrl *ctrl = dev_get_drvdata(dev);
1706
1707         if (ctrl->state == QCOM_SLIM_NGD_CTRL_AWAKE)
1708                 ctrl->state = QCOM_SLIM_NGD_CTRL_IDLE;
1709         pm_request_autosuspend(dev);
1710         return -EAGAIN;
1711 }
1712
1713 static int __maybe_unused qcom_slim_ngd_runtime_suspend(struct device *dev)
1714 {
1715         struct qcom_slim_ngd_ctrl *ctrl = dev_get_drvdata(dev);
1716         int ret = 0;
1717
1718         qcom_slim_ngd_exit_dma(ctrl);
1719         if (!ctrl->qmi.handle)
1720                 return 0;
1721
1722         ret = qcom_slim_qmi_power_request(ctrl, false);
1723         if (ret && ret != -EBUSY)
1724                 dev_info(ctrl->dev, "slim resource not idle:%d\n", ret);
1725         if (!ret || ret == -ETIMEDOUT)
1726                 ctrl->state = QCOM_SLIM_NGD_CTRL_ASLEEP;
1727
1728         return ret;
1729 }
1730
1731 static const struct dev_pm_ops qcom_slim_ngd_dev_pm_ops = {
1732         SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1733                                 pm_runtime_force_resume)
1734         SET_RUNTIME_PM_OPS(
1735                 qcom_slim_ngd_runtime_suspend,
1736                 qcom_slim_ngd_runtime_resume,
1737                 qcom_slim_ngd_runtime_idle
1738         )
1739 };
1740
1741 static struct platform_driver qcom_slim_ngd_ctrl_driver = {
1742         .probe = qcom_slim_ngd_ctrl_probe,
1743         .remove_new = qcom_slim_ngd_ctrl_remove,
1744         .driver = {
1745                 .name = "qcom,slim-ngd-ctrl",
1746                 .of_match_table = qcom_slim_ngd_dt_match,
1747         },
1748 };
1749
1750 static struct platform_driver qcom_slim_ngd_driver = {
1751         .probe = qcom_slim_ngd_probe,
1752         .remove_new = qcom_slim_ngd_remove,
1753         .driver = {
1754                 .name = QCOM_SLIM_NGD_DRV_NAME,
1755                 .pm = &qcom_slim_ngd_dev_pm_ops,
1756         },
1757 };
1758
1759 module_platform_driver(qcom_slim_ngd_ctrl_driver);
1760 MODULE_LICENSE("GPL v2");
1761 MODULE_DESCRIPTION("Qualcomm SLIMBus NGD controller");