drivers/scsi/hisi_sas/hisi_sas_v3_hw.c

   1 /*
   2  * Copyright (c) 2017 Hisilicon Limited.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  */
  10
  11 #include "hisi_sas.h"
  12 #define DRV_NAME "hisi_sas_v3_hw"
  13
  14 /* global registers need init*/
  15 #define DLVRY_QUEUE_ENABLE              0x0
  16 #define IOST_BASE_ADDR_LO               0x8
  17 #define IOST_BASE_ADDR_HI               0xc
  18 #define ITCT_BASE_ADDR_LO               0x10
  19 #define ITCT_BASE_ADDR_HI               0x14
  20 #define IO_BROKEN_MSG_ADDR_LO           0x18
  21 #define IO_BROKEN_MSG_ADDR_HI           0x1c
  22 #define PHY_CONTEXT                     0x20
  23 #define PHY_STATE                       0x24
  24 #define PHY_PORT_NUM_MA                 0x28
  25 #define PHY_CONN_RATE                   0x30
  26 #define ITCT_CLR                        0x44
  27 #define ITCT_CLR_EN_OFF                 16
  28 #define ITCT_CLR_EN_MSK                 (0x1 << ITCT_CLR_EN_OFF)
  29 #define ITCT_DEV_OFF                    0
  30 #define ITCT_DEV_MSK                    (0x7ff << ITCT_DEV_OFF)
  31 #define IO_SATA_BROKEN_MSG_ADDR_LO      0x58
  32 #define IO_SATA_BROKEN_MSG_ADDR_HI      0x5c
  33 #define SATA_INITI_D2H_STORE_ADDR_LO    0x60
  34 #define SATA_INITI_D2H_STORE_ADDR_HI    0x64
  35 #define CFG_MAX_TAG                     0x68
  36 #define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL 0x84
  37 #define HGC_SAS_TXFAIL_RETRY_CTRL       0x88
  38 #define HGC_GET_ITV_TIME                0x90
  39 #define DEVICE_MSG_WORK_MODE            0x94
  40 #define OPENA_WT_CONTI_TIME             0x9c
  41 #define I_T_NEXUS_LOSS_TIME             0xa0
  42 #define MAX_CON_TIME_LIMIT_TIME         0xa4
  43 #define BUS_INACTIVE_LIMIT_TIME         0xa8
  44 #define REJECT_TO_OPEN_LIMIT_TIME       0xac
  45 #define CFG_AGING_TIME                  0xbc
  46 #define HGC_DFX_CFG2                    0xc0
  47 #define CFG_ABT_SET_QUERY_IPTT  0xd4
  48 #define CFG_SET_ABORTED_IPTT_OFF        0
  49 #define CFG_SET_ABORTED_IPTT_MSK        (0xfff << CFG_SET_ABORTED_IPTT_OFF)
  50 #define CFG_SET_ABORTED_EN_OFF  12
  51 #define CFG_ABT_SET_IPTT_DONE   0xd8
  52 #define CFG_ABT_SET_IPTT_DONE_OFF       0
  53 #define HGC_IOMB_PROC1_STATUS   0x104
  54 #define CHNL_INT_STATUS                 0x148
  55 #define HGC_AXI_FIFO_ERR_INFO  0x154
  56 #define AXI_ERR_INFO_OFF               0
  57 #define AXI_ERR_INFO_MSK               (0xff << AXI_ERR_INFO_OFF)
  58 #define FIFO_ERR_INFO_OFF              8
  59 #define FIFO_ERR_INFO_MSK              (0xff << FIFO_ERR_INFO_OFF)
  60 #define INT_COAL_EN                     0x19c
  61 #define OQ_INT_COAL_TIME                0x1a0
  62 #define OQ_INT_COAL_CNT                 0x1a4
  63 #define ENT_INT_COAL_TIME               0x1a8
  64 #define ENT_INT_COAL_CNT                0x1ac
  65 #define OQ_INT_SRC                      0x1b0
  66 #define OQ_INT_SRC_MSK                  0x1b4
  67 #define ENT_INT_SRC1                    0x1b8
  68 #define ENT_INT_SRC1_D2H_FIS_CH0_OFF    0
  69 #define ENT_INT_SRC1_D2H_FIS_CH0_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH0_OFF)
  70 #define ENT_INT_SRC1_D2H_FIS_CH1_OFF    8
  71 #define ENT_INT_SRC1_D2H_FIS_CH1_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF)
  72 #define ENT_INT_SRC2                    0x1bc
  73 #define ENT_INT_SRC3                    0x1c0
  74 #define ENT_INT_SRC3_WP_DEPTH_OFF               8
  75 #define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF      9
  76 #define ENT_INT_SRC3_RP_DEPTH_OFF               10
  77 #define ENT_INT_SRC3_AXI_OFF                    11
  78 #define ENT_INT_SRC3_FIFO_OFF                   12
  79 #define ENT_INT_SRC3_LM_OFF                             14
  80 #define ENT_INT_SRC3_ITC_INT_OFF        15
  81 #define ENT_INT_SRC3_ITC_INT_MSK        (0x1 << ENT_INT_SRC3_ITC_INT_OFF)
  82 #define ENT_INT_SRC3_ABT_OFF            16
  83 #define ENT_INT_SRC_MSK1                0x1c4
  84 #define ENT_INT_SRC_MSK2                0x1c8
  85 #define ENT_INT_SRC_MSK3                0x1cc
  86 #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF  31
  87 #define CHNL_PHYUPDOWN_INT_MSK          0x1d0
  88 #define CHNL_ENT_INT_MSK                        0x1d4
  89 #define HGC_COM_INT_MSK                         0x1d8
  90 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK  (0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
  91 #define SAS_ECC_INTR                    0x1e8
  92 #define SAS_ECC_INTR_MSK                0x1ec
  93 #define HGC_ERR_STAT_EN                 0x238
  94 #define CQE_SEND_CNT                    0x248
  95 #define DLVRY_Q_0_BASE_ADDR_LO          0x260
  96 #define DLVRY_Q_0_BASE_ADDR_HI          0x264
  97 #define DLVRY_Q_0_DEPTH                 0x268
  98 #define DLVRY_Q_0_WR_PTR                0x26c
  99 #define DLVRY_Q_0_RD_PTR                0x270
 100 #define HYPER_STREAM_ID_EN_CFG          0xc80
 101 #define OQ0_INT_SRC_MSK                 0xc90
 102 #define COMPL_Q_0_BASE_ADDR_LO          0x4e0
 103 #define COMPL_Q_0_BASE_ADDR_HI          0x4e4
 104 #define COMPL_Q_0_DEPTH                 0x4e8
 105 #define COMPL_Q_0_WR_PTR                0x4ec
 106 #define COMPL_Q_0_RD_PTR                0x4f0
 107 #define AWQOS_AWCACHE_CFG       0xc84
 108 #define ARQOS_ARCACHE_CFG       0xc88
 109 #define HILINK_ERR_DFX          0xe04
 110 #define SAS_GPIO_CFG_0          0x1000
 111 #define SAS_GPIO_CFG_1          0x1004
 112 #define SAS_GPIO_TX_0_1 0x1040
 113 #define SAS_CFG_DRIVE_VLD       0x1070
 114
 115 /* phy registers requiring init */
 116 #define PORT_BASE                       (0x2000)
 117 #define PHY_CFG                         (PORT_BASE + 0x0)
 118 #define HARD_PHY_LINKRATE               (PORT_BASE + 0x4)
 119 #define PHY_CFG_ENA_OFF                 0
 120 #define PHY_CFG_ENA_MSK                 (0x1 << PHY_CFG_ENA_OFF)
 121 #define PHY_CFG_DC_OPT_OFF              2
 122 #define PHY_CFG_DC_OPT_MSK              (0x1 << PHY_CFG_DC_OPT_OFF)
 123 #define PHY_CFG_PHY_RST_OFF             3
 124 #define PHY_CFG_PHY_RST_MSK             (0x1 << PHY_CFG_PHY_RST_OFF)
 125 #define PROG_PHY_LINK_RATE              (PORT_BASE + 0x8)
 126 #define PHY_CTRL                        (PORT_BASE + 0x14)
 127 #define PHY_CTRL_RESET_OFF              0
 128 #define PHY_CTRL_RESET_MSK              (0x1 << PHY_CTRL_RESET_OFF)
 129 #define SL_CFG                          (PORT_BASE + 0x84)
 130 #define AIP_LIMIT                       (PORT_BASE + 0x90)
 131 #define SL_CONTROL                      (PORT_BASE + 0x94)
 132 #define SL_CONTROL_NOTIFY_EN_OFF        0
 133 #define SL_CONTROL_NOTIFY_EN_MSK        (0x1 << SL_CONTROL_NOTIFY_EN_OFF)
 134 #define SL_CTA_OFF              17
 135 #define SL_CTA_MSK              (0x1 << SL_CTA_OFF)
 136 #define RX_PRIMS_STATUS                 (PORT_BASE + 0x98)
 137 #define RX_BCAST_CHG_OFF                1
 138 #define RX_BCAST_CHG_MSK                (0x1 << RX_BCAST_CHG_OFF)
 139 #define TX_ID_DWORD0                    (PORT_BASE + 0x9c)
 140 #define TX_ID_DWORD1                    (PORT_BASE + 0xa0)
 141 #define TX_ID_DWORD2                    (PORT_BASE + 0xa4)
 142 #define TX_ID_DWORD3                    (PORT_BASE + 0xa8)
 143 #define TX_ID_DWORD4                    (PORT_BASE + 0xaC)
 144 #define TX_ID_DWORD5                    (PORT_BASE + 0xb0)
 145 #define TX_ID_DWORD6                    (PORT_BASE + 0xb4)
 146 #define TXID_AUTO                               (PORT_BASE + 0xb8)
 147 #define CT3_OFF         1
 148 #define CT3_MSK         (0x1 << CT3_OFF)
 149 #define TX_HARDRST_OFF          2
 150 #define TX_HARDRST_MSK          (0x1 << TX_HARDRST_OFF)
 151 #define RX_IDAF_DWORD0                  (PORT_BASE + 0xc4)
 152 #define RXOP_CHECK_CFG_H                (PORT_BASE + 0xfc)
 153 #define STP_LINK_TIMER                  (PORT_BASE + 0x120)
 154 #define STP_LINK_TIMEOUT_STATE          (PORT_BASE + 0x124)
 155 #define CON_CFG_DRIVER                  (PORT_BASE + 0x130)
 156 #define SAS_SSP_CON_TIMER_CFG           (PORT_BASE + 0x134)
 157 #define SAS_SMP_CON_TIMER_CFG           (PORT_BASE + 0x138)
 158 #define SAS_STP_CON_TIMER_CFG           (PORT_BASE + 0x13c)
 159 #define CHL_INT0                        (PORT_BASE + 0x1b4)
 160 #define CHL_INT0_HOTPLUG_TOUT_OFF       0
 161 #define CHL_INT0_HOTPLUG_TOUT_MSK       (0x1 << CHL_INT0_HOTPLUG_TOUT_OFF)
 162 #define CHL_INT0_SL_RX_BCST_ACK_OFF     1
 163 #define CHL_INT0_SL_RX_BCST_ACK_MSK     (0x1 << CHL_INT0_SL_RX_BCST_ACK_OFF)
 164 #define CHL_INT0_SL_PHY_ENABLE_OFF      2
 165 #define CHL_INT0_SL_PHY_ENABLE_MSK      (0x1 << CHL_INT0_SL_PHY_ENABLE_OFF)
 166 #define CHL_INT0_NOT_RDY_OFF            4
 167 #define CHL_INT0_NOT_RDY_MSK            (0x1 << CHL_INT0_NOT_RDY_OFF)
 168 #define CHL_INT0_PHY_RDY_OFF            5
 169 #define CHL_INT0_PHY_RDY_MSK            (0x1 << CHL_INT0_PHY_RDY_OFF)
 170 #define CHL_INT1                        (PORT_BASE + 0x1b8)
 171 #define CHL_INT1_DMAC_TX_ECC_ERR_OFF    15
 172 #define CHL_INT1_DMAC_TX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_TX_ECC_ERR_OFF)
 173 #define CHL_INT1_DMAC_RX_ECC_ERR_OFF    17
 174 #define CHL_INT1_DMAC_RX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_RX_ECC_ERR_OFF)
 175 #define CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF 19
 176 #define CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF 20
 177 #define CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF 21
 178 #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF 22
 179 #define CHL_INT2                        (PORT_BASE + 0x1bc)
 180 #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF  0
 181 #define CHL_INT2_RX_INVLD_DW_OFF        30
 182 #define CHL_INT2_STP_LINK_TIMEOUT_OFF   31
 183 #define CHL_INT0_MSK                    (PORT_BASE + 0x1c0)
 184 #define CHL_INT1_MSK                    (PORT_BASE + 0x1c4)
 185 #define CHL_INT2_MSK                    (PORT_BASE + 0x1c8)
 186 #define CHL_INT_COAL_EN                 (PORT_BASE + 0x1d0)
 187 #define SAS_RX_TRAIN_TIMER              (PORT_BASE + 0x2a4)
 188 #define PHY_CTRL_RDY_MSK                (PORT_BASE + 0x2b0)
 189 #define PHYCTRL_NOT_RDY_MSK             (PORT_BASE + 0x2b4)
 190 #define PHYCTRL_DWS_RESET_MSK           (PORT_BASE + 0x2b8)
 191 #define PHYCTRL_PHY_ENA_MSK             (PORT_BASE + 0x2bc)
 192 #define SL_RX_BCAST_CHK_MSK             (PORT_BASE + 0x2c0)
 193 #define PHYCTRL_OOB_RESTART_MSK         (PORT_BASE + 0x2c4)
 194 #define DMA_TX_STATUS                   (PORT_BASE + 0x2d0)
 195 #define DMA_TX_STATUS_BUSY_OFF          0
 196 #define DMA_TX_STATUS_BUSY_MSK          (0x1 << DMA_TX_STATUS_BUSY_OFF)
 197 #define DMA_RX_STATUS                   (PORT_BASE + 0x2e8)
 198 #define DMA_RX_STATUS_BUSY_OFF          0
 199 #define DMA_RX_STATUS_BUSY_MSK          (0x1 << DMA_RX_STATUS_BUSY_OFF)
 200
 201 #define COARSETUNE_TIME                 (PORT_BASE + 0x304)
 202 #define ERR_CNT_DWS_LOST                (PORT_BASE + 0x380)
 203 #define ERR_CNT_RESET_PROB              (PORT_BASE + 0x384)
 204 #define ERR_CNT_INVLD_DW                (PORT_BASE + 0x390)
 205 #define ERR_CNT_DISP_ERR                (PORT_BASE + 0x398)
 206
 207 #define DEFAULT_ITCT_HW         2048 /* reset value, not reprogrammed */
 208 #if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW)
 209 #error Max ITCT exceeded
 210 #endif
 211
 212 #define AXI_MASTER_CFG_BASE             (0x5000)
 213 #define AM_CTRL_GLOBAL                  (0x0)
 214 #define AM_CTRL_SHUTDOWN_REQ_OFF        0
 215 #define AM_CTRL_SHUTDOWN_REQ_MSK        (0x1 << AM_CTRL_SHUTDOWN_REQ_OFF)
 216 #define AM_CURR_TRANS_RETURN    (0x150)
 217
 218 #define AM_CFG_MAX_TRANS                (0x5010)
 219 #define AM_CFG_SINGLE_PORT_MAX_TRANS    (0x5014)
 220 #define AXI_CFG                                 (0x5100)
 221 #define AM_ROB_ECC_ERR_ADDR             (0x510c)
 222 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF  0
 223 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF)
 224 #define AM_ROB_ECC_MULBIT_ERR_ADDR_OFF  8
 225 #define AM_ROB_ECC_MULBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_MULBIT_ERR_ADDR_OFF)
 226
 227 /* RAS registers need init */
 228 #define RAS_BASE                (0x6000)
 229 #define SAS_RAS_INTR0                   (RAS_BASE)
 230 #define SAS_RAS_INTR1                   (RAS_BASE + 0x04)
 231 #define SAS_RAS_INTR0_MASK              (RAS_BASE + 0x08)
 232 #define SAS_RAS_INTR1_MASK              (RAS_BASE + 0x0c)
 233 #define CFG_SAS_RAS_INTR_MASK           (RAS_BASE + 0x1c)
 234 #define SAS_RAS_INTR2                   (RAS_BASE + 0x20)
 235 #define SAS_RAS_INTR2_MASK              (RAS_BASE + 0x24)
 236
 237 /* HW dma structures */
 238 /* Delivery queue header */
 239 /* dw0 */
 240 #define CMD_HDR_ABORT_FLAG_OFF          0
 241 #define CMD_HDR_ABORT_FLAG_MSK          (0x3 << CMD_HDR_ABORT_FLAG_OFF)
 242 #define CMD_HDR_ABORT_DEVICE_TYPE_OFF   2
 243 #define CMD_HDR_ABORT_DEVICE_TYPE_MSK   (0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF)
 244 #define CMD_HDR_RESP_REPORT_OFF         5
 245 #define CMD_HDR_RESP_REPORT_MSK         (0x1 << CMD_HDR_RESP_REPORT_OFF)
 246 #define CMD_HDR_TLR_CTRL_OFF            6
 247 #define CMD_HDR_TLR_CTRL_MSK            (0x3 << CMD_HDR_TLR_CTRL_OFF)
 248 #define CMD_HDR_PORT_OFF                18
 249 #define CMD_HDR_PORT_MSK                (0xf << CMD_HDR_PORT_OFF)
 250 #define CMD_HDR_PRIORITY_OFF            27
 251 #define CMD_HDR_PRIORITY_MSK            (0x1 << CMD_HDR_PRIORITY_OFF)
 252 #define CMD_HDR_CMD_OFF                 29
 253 #define CMD_HDR_CMD_MSK                 (0x7 << CMD_HDR_CMD_OFF)
 254 /* dw1 */
 255 #define CMD_HDR_UNCON_CMD_OFF   3
 256 #define CMD_HDR_DIR_OFF                 5
 257 #define CMD_HDR_DIR_MSK                 (0x3 << CMD_HDR_DIR_OFF)
 258 #define CMD_HDR_RESET_OFF               7
 259 #define CMD_HDR_RESET_MSK               (0x1 << CMD_HDR_RESET_OFF)
 260 #define CMD_HDR_VDTL_OFF                10
 261 #define CMD_HDR_VDTL_MSK                (0x1 << CMD_HDR_VDTL_OFF)
 262 #define CMD_HDR_FRAME_TYPE_OFF          11
 263 #define CMD_HDR_FRAME_TYPE_MSK          (0x1f << CMD_HDR_FRAME_TYPE_OFF)
 264 #define CMD_HDR_DEV_ID_OFF              16
 265 #define CMD_HDR_DEV_ID_MSK              (0xffff << CMD_HDR_DEV_ID_OFF)
 266 /* dw2 */
 267 #define CMD_HDR_CFL_OFF                 0
 268 #define CMD_HDR_CFL_MSK                 (0x1ff << CMD_HDR_CFL_OFF)
 269 #define CMD_HDR_NCQ_TAG_OFF             10
 270 #define CMD_HDR_NCQ_TAG_MSK             (0x1f << CMD_HDR_NCQ_TAG_OFF)
 271 #define CMD_HDR_MRFL_OFF                15
 272 #define CMD_HDR_MRFL_MSK                (0x1ff << CMD_HDR_MRFL_OFF)
 273 #define CMD_HDR_SG_MOD_OFF              24
 274 #define CMD_HDR_SG_MOD_MSK              (0x3 << CMD_HDR_SG_MOD_OFF)
 275 /* dw3 */
 276 #define CMD_HDR_IPTT_OFF                0
 277 #define CMD_HDR_IPTT_MSK                (0xffff << CMD_HDR_IPTT_OFF)
 278 /* dw6 */
 279 #define CMD_HDR_DIF_SGL_LEN_OFF         0
 280 #define CMD_HDR_DIF_SGL_LEN_MSK         (0xffff << CMD_HDR_DIF_SGL_LEN_OFF)
 281 #define CMD_HDR_DATA_SGL_LEN_OFF        16
 282 #define CMD_HDR_DATA_SGL_LEN_MSK        (0xffff << CMD_HDR_DATA_SGL_LEN_OFF)
 283 /* dw7 */
 284 #define CMD_HDR_ADDR_MODE_SEL_OFF               15
 285 #define CMD_HDR_ADDR_MODE_SEL_MSK               (1 << CMD_HDR_ADDR_MODE_SEL_OFF)
 286 #define CMD_HDR_ABORT_IPTT_OFF          16
 287 #define CMD_HDR_ABORT_IPTT_MSK          (0xffff << CMD_HDR_ABORT_IPTT_OFF)
 288
 289 /* Completion header */
 290 /* dw0 */
 291 #define CMPLT_HDR_CMPLT_OFF             0
 292 #define CMPLT_HDR_CMPLT_MSK             (0x3 << CMPLT_HDR_CMPLT_OFF)
 293 #define CMPLT_HDR_ERROR_PHASE_OFF   2
 294 #define CMPLT_HDR_ERROR_PHASE_MSK   (0xff << CMPLT_HDR_ERROR_PHASE_OFF)
 295 #define CMPLT_HDR_RSPNS_XFRD_OFF        10
 296 #define CMPLT_HDR_RSPNS_XFRD_MSK        (0x1 << CMPLT_HDR_RSPNS_XFRD_OFF)
 297 #define CMPLT_HDR_ERX_OFF               12
 298 #define CMPLT_HDR_ERX_MSK               (0x1 << CMPLT_HDR_ERX_OFF)
 299 #define CMPLT_HDR_ABORT_STAT_OFF        13
 300 #define CMPLT_HDR_ABORT_STAT_MSK        (0x7 << CMPLT_HDR_ABORT_STAT_OFF)
 301 /* abort_stat */
 302 #define STAT_IO_NOT_VALID               0x1
 303 #define STAT_IO_NO_DEVICE               0x2
 304 #define STAT_IO_COMPLETE                0x3
 305 #define STAT_IO_ABORTED                 0x4
 306 /* dw1 */
 307 #define CMPLT_HDR_IPTT_OFF              0
 308 #define CMPLT_HDR_IPTT_MSK              (0xffff << CMPLT_HDR_IPTT_OFF)
 309 #define CMPLT_HDR_DEV_ID_OFF            16
 310 #define CMPLT_HDR_DEV_ID_MSK            (0xffff << CMPLT_HDR_DEV_ID_OFF)
 311 /* dw3 */
 312 #define CMPLT_HDR_IO_IN_TARGET_OFF      17
 313 #define CMPLT_HDR_IO_IN_TARGET_MSK      (0x1 << CMPLT_HDR_IO_IN_TARGET_OFF)
 314
 315 /* ITCT header */
 316 /* qw0 */
 317 #define ITCT_HDR_DEV_TYPE_OFF           0
 318 #define ITCT_HDR_DEV_TYPE_MSK           (0x3 << ITCT_HDR_DEV_TYPE_OFF)
 319 #define ITCT_HDR_VALID_OFF              2
 320 #define ITCT_HDR_VALID_MSK              (0x1 << ITCT_HDR_VALID_OFF)
 321 #define ITCT_HDR_MCR_OFF                5
 322 #define ITCT_HDR_MCR_MSK                (0xf << ITCT_HDR_MCR_OFF)
 323 #define ITCT_HDR_VLN_OFF                9
 324 #define ITCT_HDR_VLN_MSK                (0xf << ITCT_HDR_VLN_OFF)
 325 #define ITCT_HDR_SMP_TIMEOUT_OFF        16
 326 #define ITCT_HDR_AWT_CONTINUE_OFF       25
 327 #define ITCT_HDR_PORT_ID_OFF            28
 328 #define ITCT_HDR_PORT_ID_MSK            (0xf << ITCT_HDR_PORT_ID_OFF)
 329 /* qw2 */
 330 #define ITCT_HDR_INLT_OFF               0
 331 #define ITCT_HDR_INLT_MSK               (0xffffULL << ITCT_HDR_INLT_OFF)
 332 #define ITCT_HDR_RTOLT_OFF              48
 333 #define ITCT_HDR_RTOLT_MSK              (0xffffULL << ITCT_HDR_RTOLT_OFF)
 334
 335 struct hisi_sas_complete_v3_hdr {
 336         __le32 dw0;
 337         __le32 dw1;
 338         __le32 act;
 339         __le32 dw3;
 340 };
 341
 342 struct hisi_sas_err_record_v3 {
 343         /* dw0 */
 344         __le32 trans_tx_fail_type;
 345
 346         /* dw1 */
 347         __le32 trans_rx_fail_type;
 348
 349         /* dw2 */
 350         __le16 dma_tx_err_type;
 351         __le16 sipc_rx_err_type;
 352
 353         /* dw3 */
 354         __le32 dma_rx_err_type;
 355 };
 356
 357 #define RX_DATA_LEN_UNDERFLOW_OFF       6
 358 #define RX_DATA_LEN_UNDERFLOW_MSK       (1 << RX_DATA_LEN_UNDERFLOW_OFF)
 359
 360 #define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096
 361 #define HISI_SAS_MSI_COUNT_V3_HW 32
 362
 363 #define DIR_NO_DATA 0
 364 #define DIR_TO_INI 1
 365 #define DIR_TO_DEVICE 2
 366 #define DIR_RESERVED 3
 367
 368 #define FIS_CMD_IS_UNCONSTRAINED(fis) \
 369         ((fis.command == ATA_CMD_READ_LOG_EXT) || \
 370         (fis.command == ATA_CMD_READ_LOG_DMA_EXT) || \
 371         ((fis.command == ATA_CMD_DEV_RESET) && \
 372         ((fis.control & ATA_SRST) != 0)))
 373
 374 static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
 375 {
 376         void __iomem *regs = hisi_hba->regs + off;
 377
 378         return readl(regs);
 379 }
 380
 381 static u32 hisi_sas_read32_relaxed(struct hisi_hba *hisi_hba, u32 off)
 382 {
 383         void __iomem *regs = hisi_hba->regs + off;
 384
 385         return readl_relaxed(regs);
 386 }
 387
 388 static void hisi_sas_write32(struct hisi_hba *hisi_hba, u32 off, u32 val)
 389 {
 390         void __iomem *regs = hisi_hba->regs + off;
 391
 392         writel(val, regs);
 393 }
 394
 395 static void hisi_sas_phy_write32(struct hisi_hba *hisi_hba, int phy_no,
 396                                  u32 off, u32 val)
 397 {
 398         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 399
 400         writel(val, regs);
 401 }
 402
 403 static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
 404                                       int phy_no, u32 off)
 405 {
 406         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 407
 408         return readl(regs);
 409 }
 410
 411 #define hisi_sas_read32_poll_timeout(off, val, cond, delay_us,          \
 412                                      timeout_us)                        \
 413 ({                                                                      \
 414         void __iomem *regs = hisi_hba->regs + off;                      \
 415         readl_poll_timeout(regs, val, cond, delay_us, timeout_us);      \
 416 })
 417
 418 #define hisi_sas_read32_poll_timeout_atomic(off, val, cond, delay_us,   \
 419                                             timeout_us)                 \
 420 ({                                                                      \
 421         void __iomem *regs = hisi_hba->regs + off;                      \
 422         readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\
 423 })
 424
 425 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 426 {
 427         struct pci_dev *pdev = hisi_hba->pci_dev;
 428         int i;
 429
 430         /* Global registers init */
 431         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
 432                          (u32)((1ULL << hisi_hba->queue_count) - 1));
 433         hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff0400);
 434         hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108);
 435         hisi_sas_write32(hisi_hba, CFG_AGING_TIME, 0x1);
 436         hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
 437         hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1);
 438         hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1);
 439         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 0xffff);
 440         hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff);
 441         hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff);
 442         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
 443         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
 444         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
 445         if (pdev->revision >= 0x21)
 446                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffff7aff);
 447         else
 448                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
 449         hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 450         hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 451         hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
 452         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x0);
 453         hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
 454         hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
 455         for (i = 0; i < hisi_hba->queue_count; i++)
 456                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0);
 457
 458         hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
 459
 460         for (i = 0; i < hisi_hba->n_phy; i++) {
 461                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 462                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 463                 u32 prog_phy_link_rate = 0x800;
 464
 465                 if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
 466                                 SAS_LINK_RATE_1_5_GBPS)) {
 467                         prog_phy_link_rate = 0x855;
 468                 } else {
 469                         enum sas_linkrate max = sas_phy->phy->maximum_linkrate;
 470
 471                         prog_phy_link_rate =
 472                                 hisi_sas_get_prog_phy_linkrate_mask(max) |
 473                                 0x800;
 474                 }
 475                 hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
 476                         prog_phy_link_rate);
 477                 hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80);
 478                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
 479                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
 480                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
 481                 hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
 482                 if (pdev->revision >= 0x21)
 483                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 484                                         0xffffffff);
 485                 else
 486                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 487                                         0xff87ffff);
 488                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
 489                 hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
 490                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
 491                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0);
 492                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0);
 493                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
 494                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x1);
 495                 hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120);
 496                 hisi_sas_phy_write32(hisi_hba, i, CON_CFG_DRIVER, 0x2a0a01);
 497
 498                 /* used for 12G negotiate */
 499                 hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e);
 500                 hisi_sas_phy_write32(hisi_hba, i, AIP_LIMIT, 0x2ffff);
 501         }
 502
 503         for (i = 0; i < hisi_hba->queue_count; i++) {
 504                 /* Delivery queue */
 505                 hisi_sas_write32(hisi_hba,
 506                                  DLVRY_Q_0_BASE_ADDR_HI + (i * 0x14),
 507                                  upper_32_bits(hisi_hba->cmd_hdr_dma[i]));
 508
 509                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_BASE_ADDR_LO + (i * 0x14),
 510                                  lower_32_bits(hisi_hba->cmd_hdr_dma[i]));
 511
 512                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_DEPTH + (i * 0x14),
 513                                  HISI_SAS_QUEUE_SLOTS);
 514
 515                 /* Completion queue */
 516                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_HI + (i * 0x14),
 517                                  upper_32_bits(hisi_hba->complete_hdr_dma[i]));
 518
 519                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_LO + (i * 0x14),
 520                                  lower_32_bits(hisi_hba->complete_hdr_dma[i]));
 521
 522                 hisi_sas_write32(hisi_hba, COMPL_Q_0_DEPTH + (i * 0x14),
 523                                  HISI_SAS_QUEUE_SLOTS);
 524         }
 525
 526         /* itct */
 527         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_LO,
 528                          lower_32_bits(hisi_hba->itct_dma));
 529
 530         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_HI,
 531                          upper_32_bits(hisi_hba->itct_dma));
 532
 533         /* iost */
 534         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_LO,
 535                          lower_32_bits(hisi_hba->iost_dma));
 536
 537         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_HI,
 538                          upper_32_bits(hisi_hba->iost_dma));
 539
 540         /* breakpoint */
 541         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_LO,
 542                          lower_32_bits(hisi_hba->breakpoint_dma));
 543
 544         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_HI,
 545                          upper_32_bits(hisi_hba->breakpoint_dma));
 546
 547         /* SATA broken msg */
 548         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_LO,
 549                          lower_32_bits(hisi_hba->sata_breakpoint_dma));
 550
 551         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_HI,
 552                          upper_32_bits(hisi_hba->sata_breakpoint_dma));
 553
 554         /* SATA initial fis */
 555         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_LO,
 556                          lower_32_bits(hisi_hba->initial_fis_dma));
 557
 558         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_HI,
 559                          upper_32_bits(hisi_hba->initial_fis_dma));
 560
 561         /* RAS registers init */
 562         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, 0x0);
 563         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, 0x0);
 564         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, 0x0);
 565         hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, 0x0);
 566
 567         /* LED registers init */
 568         hisi_sas_write32(hisi_hba, SAS_CFG_DRIVE_VLD, 0x80000ff);
 569         hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1, 0x80808080);
 570         hisi_sas_write32(hisi_hba, SAS_GPIO_TX_0_1 + 0x4, 0x80808080);
 571         /* Configure blink generator rate A to 1Hz and B to 4Hz */
 572         hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_1, 0x121700);
 573         hisi_sas_write32(hisi_hba, SAS_GPIO_CFG_0, 0x800000);
 574 }
 575
 576 static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 577 {
 578         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 579
 580         cfg &= ~PHY_CFG_DC_OPT_MSK;
 581         cfg |= 1 << PHY_CFG_DC_OPT_OFF;
 582         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 583 }
 584
 585 static void config_id_frame_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 586 {
 587         struct sas_identify_frame identify_frame;
 588         u32 *identify_buffer;
 589
 590         memset(&identify_frame, 0, sizeof(identify_frame));
 591         identify_frame.dev_type = SAS_END_DEVICE;
 592         identify_frame.frame_type = 0;
 593         identify_frame._un1 = 1;
 594         identify_frame.initiator_bits = SAS_PROTOCOL_ALL;
 595         identify_frame.target_bits = SAS_PROTOCOL_NONE;
 596         memcpy(&identify_frame._un4_11[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 597         memcpy(&identify_frame.sas_addr[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 598         identify_frame.phy_id = phy_no;
 599         identify_buffer = (u32 *)(&identify_frame);
 600
 601         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
 602                         __swab32(identify_buffer[0]));
 603         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
 604                         __swab32(identify_buffer[1]));
 605         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
 606                         __swab32(identify_buffer[2]));
 607         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
 608                         __swab32(identify_buffer[3]));
 609         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
 610                         __swab32(identify_buffer[4]));
 611         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
 612                         __swab32(identify_buffer[5]));
 613 }
 614
 615 static void setup_itct_v3_hw(struct hisi_hba *hisi_hba,
 616                              struct hisi_sas_device *sas_dev)
 617 {
 618         struct domain_device *device = sas_dev->sas_device;
 619         struct device *dev = hisi_hba->dev;
 620         u64 qw0, device_id = sas_dev->device_id;
 621         struct hisi_sas_itct *itct = &hisi_hba->itct[device_id];
 622         struct domain_device *parent_dev = device->parent;
 623         struct asd_sas_port *sas_port = device->port;
 624         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
 625
 626         memset(itct, 0, sizeof(*itct));
 627
 628         /* qw0 */
 629         qw0 = 0;
 630         switch (sas_dev->dev_type) {
 631         case SAS_END_DEVICE:
 632         case SAS_EDGE_EXPANDER_DEVICE:
 633         case SAS_FANOUT_EXPANDER_DEVICE:
 634                 qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF;
 635                 break;
 636         case SAS_SATA_DEV:
 637         case SAS_SATA_PENDING:
 638                 if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
 639                         qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF;
 640                 else
 641                         qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF;
 642                 break;
 643         default:
 644                 dev_warn(dev, "setup itct: unsupported dev type (%d)\n",
 645                          sas_dev->dev_type);
 646         }
 647
 648         qw0 |= ((1 << ITCT_HDR_VALID_OFF) |
 649                 (device->linkrate << ITCT_HDR_MCR_OFF) |
 650                 (1 << ITCT_HDR_VLN_OFF) |
 651                 (0xfa << ITCT_HDR_SMP_TIMEOUT_OFF) |
 652                 (1 << ITCT_HDR_AWT_CONTINUE_OFF) |
 653                 (port->id << ITCT_HDR_PORT_ID_OFF));
 654         itct->qw0 = cpu_to_le64(qw0);
 655
 656         /* qw1 */
 657         memcpy(&itct->sas_addr, device->sas_addr, SAS_ADDR_SIZE);
 658         itct->sas_addr = __swab64(itct->sas_addr);
 659
 660         /* qw2 */
 661         if (!dev_is_sata(device))
 662                 itct->qw2 = cpu_to_le64((5000ULL << ITCT_HDR_INLT_OFF) |
 663                                         (0x1ULL << ITCT_HDR_RTOLT_OFF));
 664 }
 665
 666 static void clear_itct_v3_hw(struct hisi_hba *hisi_hba,
 667                               struct hisi_sas_device *sas_dev)
 668 {
 669         DECLARE_COMPLETION_ONSTACK(completion);
 670         u64 dev_id = sas_dev->device_id;
 671         struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
 672         u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
 673
 674         sas_dev->completion = &completion;
 675
 676         /* clear the itct interrupt state */
 677         if (ENT_INT_SRC3_ITC_INT_MSK & reg_val)
 678                 hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
 679                                  ENT_INT_SRC3_ITC_INT_MSK);
 680
 681         /* clear the itct table*/
 682         reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
 683         hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val);
 684
 685         wait_for_completion(sas_dev->completion);
 686         memset(itct, 0, sizeof(struct hisi_sas_itct));
 687 }
 688
 689 static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
 690                                 struct domain_device *device)
 691 {
 692         struct hisi_sas_slot *slot, *slot2;
 693         struct hisi_sas_device *sas_dev = device->lldd_dev;
 694         u32 cfg_abt_set_query_iptt;
 695
 696         cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba,
 697                 CFG_ABT_SET_QUERY_IPTT);
 698         list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) {
 699                 cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK;
 700                 cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) |
 701                         (slot->idx << CFG_SET_ABORTED_IPTT_OFF);
 702                 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 703                         cfg_abt_set_query_iptt);
 704         }
 705         cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF);
 706         hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 707                 cfg_abt_set_query_iptt);
 708         hisi_sas_write32(hisi_hba, CFG_ABT_SET_IPTT_DONE,
 709                                         1 << CFG_ABT_SET_IPTT_DONE_OFF);
 710 }
 711
 712 static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
 713 {
 714         struct device *dev = hisi_hba->dev;
 715         int ret;
 716         u32 val;
 717
 718         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
 719
 720         /* Disable all of the PHYs */
 721         hisi_sas_stop_phys(hisi_hba);
 722         udelay(50);
 723
 724         /* Ensure axi bus idle */
 725         ret = hisi_sas_read32_poll_timeout(AXI_CFG, val, !val,
 726                                            20000, 1000000);
 727         if (ret) {
 728                 dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
 729                 return -EIO;
 730         }
 731
 732         if (ACPI_HANDLE(dev)) {
 733                 acpi_status s;
 734
 735                 s = acpi_evaluate_object(ACPI_HANDLE(dev), "_RST", NULL, NULL);
 736                 if (ACPI_FAILURE(s)) {
 737                         dev_err(dev, "Reset failed\n");
 738                         return -EIO;
 739                 }
 740         } else {
 741                 dev_err(dev, "no reset method!\n");
 742                 return -EINVAL;
 743         }
 744
 745         return 0;
 746 }
 747
 748 static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 749 {
 750         struct device *dev = hisi_hba->dev;
 751         int rc;
 752
 753         rc = reset_hw_v3_hw(hisi_hba);
 754         if (rc) {
 755                 dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc);
 756                 return rc;
 757         }
 758
 759         msleep(100);
 760         init_reg_v3_hw(hisi_hba);
 761
 762         return 0;
 763 }
 764
 765 static void enable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 766 {
 767         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 768
 769         cfg |= PHY_CFG_ENA_MSK;
 770         cfg &= ~PHY_CFG_PHY_RST_MSK;
 771         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 772 }
 773
 774 static void disable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 775 {
 776         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 777         u32 state;
 778
 779         cfg &= ~PHY_CFG_ENA_MSK;
 780         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 781
 782         mdelay(50);
 783
 784         state = hisi_sas_read32(hisi_hba, PHY_STATE);
 785         if (state & BIT(phy_no)) {
 786                 cfg |= PHY_CFG_PHY_RST_MSK;
 787                 hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 788         }
 789 }
 790
 791 static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 792 {
 793         config_id_frame_v3_hw(hisi_hba, phy_no);
 794         config_phy_opt_mode_v3_hw(hisi_hba, phy_no);
 795         enable_phy_v3_hw(hisi_hba, phy_no);
 796 }
 797
 798 static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 799 {
 800         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 801         u32 txid_auto;
 802
 803         disable_phy_v3_hw(hisi_hba, phy_no);
 804         if (phy->identify.device_type == SAS_END_DEVICE) {
 805                 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 806                 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
 807                                         txid_auto | TX_HARDRST_MSK);
 808         }
 809         msleep(100);
 810         start_phy_v3_hw(hisi_hba, phy_no);
 811 }
 812
 813 static enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
 814 {
 815         return SAS_LINK_RATE_12_0_GBPS;
 816 }
 817
 818 static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
 819 {
 820         int i;
 821
 822         for (i = 0; i < hisi_hba->n_phy; i++) {
 823                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 824                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 825
 826                 if (!sas_phy->phy->enabled)
 827                         continue;
 828
 829                 start_phy_v3_hw(hisi_hba, i);
 830         }
 831 }
 832
 833 static void sl_notify_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 834 {
 835         u32 sl_control;
 836
 837         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 838         sl_control |= SL_CONTROL_NOTIFY_EN_MSK;
 839         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 840         msleep(1);
 841         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 842         sl_control &= ~SL_CONTROL_NOTIFY_EN_MSK;
 843         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 844 }
 845
 846 static int get_wideport_bitmap_v3_hw(struct hisi_hba *hisi_hba, int port_id)
 847 {
 848         int i, bitmap = 0;
 849         u32 phy_port_num_ma = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
 850         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
 851
 852         for (i = 0; i < hisi_hba->n_phy; i++)
 853                 if (phy_state & BIT(i))
 854                         if (((phy_port_num_ma >> (i * 4)) & 0xf) == port_id)
 855                                 bitmap |= BIT(i);
 856
 857         return bitmap;
 858 }
 859
 860 /**
 861  * The callpath to this function and upto writing the write
 862  * queue pointer should be safe from interruption.
 863  */
 864 static int
 865 get_free_slot_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
 866 {
 867         struct device *dev = hisi_hba->dev;
 868         int queue = dq->id;
 869         u32 r, w;
 870
 871         w = dq->wr_point;
 872         r = hisi_sas_read32_relaxed(hisi_hba,
 873                                 DLVRY_Q_0_RD_PTR + (queue * 0x14));
 874         if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
 875                 dev_warn(dev, "full queue=%d r=%d w=%d\n",
 876                                 queue, r, w);
 877                 return -EAGAIN;
 878         }
 879
 880         dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
 881
 882         return w;
 883 }
 884
 885 static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
 886 {
 887         struct hisi_hba *hisi_hba = dq->hisi_hba;
 888         struct hisi_sas_slot *s, *s1, *s2 = NULL;
 889         int dlvry_queue = dq->id;
 890         int wp;
 891
 892         list_for_each_entry_safe(s, s1, &dq->list, delivery) {
 893                 if (!s->ready)
 894                         break;
 895                 s2 = s;
 896                 list_del(&s->delivery);
 897         }
 898
 899         if (!s2)
 900                 return;
 901
 902         /*
 903          * Ensure that memories for slots built on other CPUs is observed.
 904          */
 905         smp_rmb();
 906         wp = (s2->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
 907
 908         hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
 909 }
 910
 911 static void prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
 912                               struct hisi_sas_slot *slot,
 913                               struct hisi_sas_cmd_hdr *hdr,
 914                               struct scatterlist *scatter,
 915                               int n_elem)
 916 {
 917         struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
 918         struct scatterlist *sg;
 919         int i;
 920
 921         for_each_sg(scatter, sg, n_elem, i) {
 922                 struct hisi_sas_sge *entry = &sge_page->sge[i];
 923
 924                 entry->addr = cpu_to_le64(sg_dma_address(sg));
 925                 entry->page_ctrl_0 = entry->page_ctrl_1 = 0;
 926                 entry->data_len = cpu_to_le32(sg_dma_len(sg));
 927                 entry->data_off = 0;
 928         }
 929
 930         hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 931
 932         hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
 933 }
 934
 935 static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 936                           struct hisi_sas_slot *slot)
 937 {
 938         struct sas_task *task = slot->task;
 939         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 940         struct domain_device *device = task->dev;
 941         struct hisi_sas_device *sas_dev = device->lldd_dev;
 942         struct hisi_sas_port *port = slot->port;
 943         struct sas_ssp_task *ssp_task = &task->ssp_task;
 944         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
 945         struct hisi_sas_tmf_task *tmf = slot->tmf;
 946         int has_data = 0, priority = !!tmf;
 947         u8 *buf_cmd;
 948         u32 dw1 = 0, dw2 = 0;
 949
 950         hdr->dw0 = cpu_to_le32((1 << CMD_HDR_RESP_REPORT_OFF) |
 951                                (2 << CMD_HDR_TLR_CTRL_OFF) |
 952                                (port->id << CMD_HDR_PORT_OFF) |
 953                                (priority << CMD_HDR_PRIORITY_OFF) |
 954                                (1 << CMD_HDR_CMD_OFF)); /* ssp */
 955
 956         dw1 = 1 << CMD_HDR_VDTL_OFF;
 957         if (tmf) {
 958                 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
 959                 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
 960         } else {
 961                 dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF;
 962                 switch (scsi_cmnd->sc_data_direction) {
 963                 case DMA_TO_DEVICE:
 964                         has_data = 1;
 965                         dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
 966                         break;
 967                 case DMA_FROM_DEVICE:
 968                         has_data = 1;
 969                         dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
 970                         break;
 971                 default:
 972                         dw1 &= ~CMD_HDR_DIR_MSK;
 973                 }
 974         }
 975
 976         /* map itct entry */
 977         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
 978         hdr->dw1 = cpu_to_le32(dw1);
 979
 980         dw2 = (((sizeof(struct ssp_command_iu) + sizeof(struct ssp_frame_hdr)
 981               + 3) / 4) << CMD_HDR_CFL_OFF) |
 982               ((HISI_SAS_MAX_SSP_RESP_SZ / 4) << CMD_HDR_MRFL_OFF) |
 983               (2 << CMD_HDR_SG_MOD_OFF);
 984         hdr->dw2 = cpu_to_le32(dw2);
 985         hdr->transfer_tags = cpu_to_le32(slot->idx);
 986
 987         if (has_data)
 988                 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
 989                                         slot->n_elem);
 990
 991         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 992         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
 993         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
 994
 995         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot) +
 996                 sizeof(struct ssp_frame_hdr);
 997
 998         memcpy(buf_cmd, &task->ssp_task.LUN, 8);
 999         if (!tmf) {
1000                 buf_cmd[9] = ssp_task->task_attr | (ssp_task->task_prio << 3);
1001                 memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
1002         } else {
1003                 buf_cmd[10] = tmf->tmf;
1004                 switch (tmf->tmf) {
1005                 case TMF_ABORT_TASK:
1006                 case TMF_QUERY_TASK:
1007                         buf_cmd[12] =
1008                                 (tmf->tag_of_task_to_be_managed >> 8) & 0xff;
1009                         buf_cmd[13] =
1010                                 tmf->tag_of_task_to_be_managed & 0xff;
1011                         break;
1012                 default:
1013                         break;
1014                 }
1015         }
1016 }
1017
1018 static void prep_smp_v3_hw(struct hisi_hba *hisi_hba,
1019                           struct hisi_sas_slot *slot)
1020 {
1021         struct sas_task *task = slot->task;
1022         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1023         struct domain_device *device = task->dev;
1024         struct hisi_sas_port *port = slot->port;
1025         struct scatterlist *sg_req;
1026         struct hisi_sas_device *sas_dev = device->lldd_dev;
1027         dma_addr_t req_dma_addr;
1028         unsigned int req_len;
1029
1030         /* req */
1031         sg_req = &task->smp_task.smp_req;
1032         req_len = sg_dma_len(sg_req);
1033         req_dma_addr = sg_dma_address(sg_req);
1034
1035         /* create header */
1036         /* dw0 */
1037         hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
1038                                (1 << CMD_HDR_PRIORITY_OFF) | /* high pri */
1039                                (2 << CMD_HDR_CMD_OFF)); /* smp */
1040
1041         /* map itct entry */
1042         hdr->dw1 = cpu_to_le32((sas_dev->device_id << CMD_HDR_DEV_ID_OFF) |
1043                                (1 << CMD_HDR_FRAME_TYPE_OFF) |
1044                                (DIR_NO_DATA << CMD_HDR_DIR_OFF));
1045
1046         /* dw2 */
1047         hdr->dw2 = cpu_to_le32((((req_len - 4) / 4) << CMD_HDR_CFL_OFF) |
1048                                (HISI_SAS_MAX_SMP_RESP_SZ / 4 <<
1049                                CMD_HDR_MRFL_OFF));
1050
1051         hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
1052
1053         hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
1054         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1055
1056 }
1057
1058 static void prep_ata_v3_hw(struct hisi_hba *hisi_hba,
1059                           struct hisi_sas_slot *slot)
1060 {
1061         struct sas_task *task = slot->task;
1062         struct domain_device *device = task->dev;
1063         struct domain_device *parent_dev = device->parent;
1064         struct hisi_sas_device *sas_dev = device->lldd_dev;
1065         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1066         struct asd_sas_port *sas_port = device->port;
1067         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
1068         u8 *buf_cmd;
1069         int has_data = 0, hdr_tag = 0;
1070         u32 dw1 = 0, dw2 = 0;
1071
1072         hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
1073         if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
1074                 hdr->dw0 |= cpu_to_le32(3 << CMD_HDR_CMD_OFF);
1075         else
1076                 hdr->dw0 |= cpu_to_le32(4 << CMD_HDR_CMD_OFF);
1077
1078         switch (task->data_dir) {
1079         case DMA_TO_DEVICE:
1080                 has_data = 1;
1081                 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1082                 break;
1083         case DMA_FROM_DEVICE:
1084                 has_data = 1;
1085                 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1086                 break;
1087         default:
1088                 dw1 &= ~CMD_HDR_DIR_MSK;
1089         }
1090
1091         if ((task->ata_task.fis.command == ATA_CMD_DEV_RESET) &&
1092                         (task->ata_task.fis.control & ATA_SRST))
1093                 dw1 |= 1 << CMD_HDR_RESET_OFF;
1094
1095         dw1 |= (hisi_sas_get_ata_protocol(
1096                 &task->ata_task.fis, task->data_dir))
1097                 << CMD_HDR_FRAME_TYPE_OFF;
1098         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1099
1100         if (FIS_CMD_IS_UNCONSTRAINED(task->ata_task.fis))
1101                 dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
1102
1103         hdr->dw1 = cpu_to_le32(dw1);
1104
1105         /* dw2 */
1106         if (task->ata_task.use_ncq && hisi_sas_get_ncq_tag(task, &hdr_tag)) {
1107                 task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
1108                 dw2 |= hdr_tag << CMD_HDR_NCQ_TAG_OFF;
1109         }
1110
1111         dw2 |= (HISI_SAS_MAX_STP_RESP_SZ / 4) << CMD_HDR_CFL_OFF |
1112                         2 << CMD_HDR_SG_MOD_OFF;
1113         hdr->dw2 = cpu_to_le32(dw2);
1114
1115         /* dw3 */
1116         hdr->transfer_tags = cpu_to_le32(slot->idx);
1117
1118         if (has_data)
1119                 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
1120                                         slot->n_elem);
1121
1122         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
1123         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1124         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1125
1126         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot);
1127
1128         if (likely(!task->ata_task.device_control_reg_update))
1129                 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
1130         /* fill in command FIS */
1131         memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
1132 }
1133
1134 static void prep_abort_v3_hw(struct hisi_hba *hisi_hba,
1135                 struct hisi_sas_slot *slot,
1136                 int device_id, int abort_flag, int tag_to_abort)
1137 {
1138         struct sas_task *task = slot->task;
1139         struct domain_device *dev = task->dev;
1140         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1141         struct hisi_sas_port *port = slot->port;
1142
1143         /* dw0 */
1144         hdr->dw0 = cpu_to_le32((5 << CMD_HDR_CMD_OFF) | /*abort*/
1145                                (port->id << CMD_HDR_PORT_OFF) |
1146                                    (dev_is_sata(dev)
1147                                         << CMD_HDR_ABORT_DEVICE_TYPE_OFF) |
1148                                         (abort_flag
1149                                          << CMD_HDR_ABORT_FLAG_OFF));
1150
1151         /* dw1 */
1152         hdr->dw1 = cpu_to_le32(device_id
1153                         << CMD_HDR_DEV_ID_OFF);
1154
1155         /* dw7 */
1156         hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
1157         hdr->transfer_tags = cpu_to_le32(slot->idx);
1158
1159 }
1160
1161 static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1162 {
1163         int i, res;
1164         u32 context, port_id, link_rate;
1165         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1166         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1167         struct device *dev = hisi_hba->dev;
1168         unsigned long flags;
1169
1170         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
1171
1172         port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1173         port_id = (port_id >> (4 * phy_no)) & 0xf;
1174         link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE);
1175         link_rate = (link_rate >> (phy_no * 4)) & 0xf;
1176
1177         if (port_id == 0xf) {
1178                 dev_err(dev, "phyup: phy%d invalid portid\n", phy_no);
1179                 res = IRQ_NONE;
1180                 goto end;
1181         }
1182         sas_phy->linkrate = link_rate;
1183         phy->phy_type &= ~(PORT_TYPE_SAS | PORT_TYPE_SATA);
1184
1185         /* Check for SATA dev */
1186         context = hisi_sas_read32(hisi_hba, PHY_CONTEXT);
1187         if (context & (1 << phy_no)) {
1188                 struct hisi_sas_initial_fis *initial_fis;
1189                 struct dev_to_host_fis *fis;
1190                 u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
1191
1192                 dev_info(dev, "phyup: phy%d link_rate=%d(sata)\n", phy_no, link_rate);
1193                 initial_fis = &hisi_hba->initial_fis[phy_no];
1194                 fis = &initial_fis->fis;
1195
1196                 /* check ERR bit of Status Register */
1197                 if (fis->status & ATA_ERR) {
1198                         dev_warn(dev, "sata int: phy%d FIS status: 0x%x\n",
1199                                  phy_no, fis->status);
1200                         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1201                         res = IRQ_NONE;
1202                         goto end;
1203                 }
1204
1205                 sas_phy->oob_mode = SATA_OOB_MODE;
1206                 attached_sas_addr[0] = 0x50;
1207                 attached_sas_addr[7] = phy_no;
1208                 memcpy(sas_phy->attached_sas_addr,
1209                        attached_sas_addr,
1210                        SAS_ADDR_SIZE);
1211                 memcpy(sas_phy->frame_rcvd, fis,
1212                        sizeof(struct dev_to_host_fis));
1213                 phy->phy_type |= PORT_TYPE_SATA;
1214                 phy->identify.device_type = SAS_SATA_DEV;
1215                 phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
1216                 phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
1217         } else {
1218                 u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
1219                 struct sas_identify_frame *id =
1220                         (struct sas_identify_frame *)frame_rcvd;
1221
1222                 dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate);
1223                 for (i = 0; i < 6; i++) {
1224                         u32 idaf = hisi_sas_phy_read32(hisi_hba, phy_no,
1225                                                RX_IDAF_DWORD0 + (i * 4));
1226                         frame_rcvd[i] = __swab32(idaf);
1227                 }
1228                 sas_phy->oob_mode = SAS_OOB_MODE;
1229                 memcpy(sas_phy->attached_sas_addr,
1230                        &id->sas_addr,
1231                        SAS_ADDR_SIZE);
1232                 phy->phy_type |= PORT_TYPE_SAS;
1233                 phy->identify.device_type = id->dev_type;
1234                 phy->frame_rcvd_size = sizeof(struct sas_identify_frame);
1235                 if (phy->identify.device_type == SAS_END_DEVICE)
1236                         phy->identify.target_port_protocols =
1237                                 SAS_PROTOCOL_SSP;
1238                 else if (phy->identify.device_type != SAS_PHY_UNUSED)
1239                         phy->identify.target_port_protocols =
1240                                 SAS_PROTOCOL_SMP;
1241         }
1242
1243         phy->port_id = port_id;
1244         phy->phy_attached = 1;
1245         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
1246         res = IRQ_HANDLED;
1247         spin_lock_irqsave(&phy->lock, flags);
1248         if (phy->reset_completion) {
1249                 phy->in_reset = 0;
1250                 complete(phy->reset_completion);
1251         }
1252         spin_unlock_irqrestore(&phy->lock, flags);
1253 end:
1254         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1255                              CHL_INT0_SL_PHY_ENABLE_MSK);
1256         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0);
1257
1258         return res;
1259 }
1260
1261 static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1262 {
1263         u32 phy_state, sl_ctrl, txid_auto;
1264         struct device *dev = hisi_hba->dev;
1265
1266         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1);
1267
1268         phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1269         dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state);
1270         hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0);
1271
1272         sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1273         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL,
1274                                                 sl_ctrl&(~SL_CTA_MSK));
1275
1276         txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1277         hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1278                                                 txid_auto | CT3_MSK);
1279
1280         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
1281         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0);
1282
1283         return IRQ_HANDLED;
1284 }
1285
1286 static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1287 {
1288         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1289         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1290         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
1291         u32 bcast_status;
1292
1293         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1);
1294         bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS);
1295         if ((bcast_status & RX_BCAST_CHG_MSK) &&
1296             !test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags))
1297                 sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD);
1298         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1299                              CHL_INT0_SL_RX_BCST_ACK_MSK);
1300         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0);
1301
1302         return IRQ_HANDLED;
1303 }
1304
1305 static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p)
1306 {
1307         struct hisi_hba *hisi_hba = p;
1308         u32 irq_msk;
1309         int phy_no = 0;
1310         irqreturn_t res = IRQ_NONE;
1311
1312         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1313                                 & 0x11111111;
1314         while (irq_msk) {
1315                 if (irq_msk  & 1) {
1316                         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1317                                                             CHL_INT0);
1318                         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1319                         int rdy = phy_state & (1 << phy_no);
1320
1321                         if (rdy) {
1322                                 if (irq_value & CHL_INT0_SL_PHY_ENABLE_MSK)
1323                                         /* phy up */
1324                                         if (phy_up_v3_hw(phy_no, hisi_hba)
1325                                                         == IRQ_HANDLED)
1326                                                 res = IRQ_HANDLED;
1327                                 if (irq_value & CHL_INT0_SL_RX_BCST_ACK_MSK)
1328                                         /* phy bcast */
1329                                         if (phy_bcast_v3_hw(phy_no, hisi_hba)
1330                                                         == IRQ_HANDLED)
1331                                                 res = IRQ_HANDLED;
1332                         } else {
1333                                 if (irq_value & CHL_INT0_NOT_RDY_MSK)
1334                                         /* phy down */
1335                                         if (phy_down_v3_hw(phy_no, hisi_hba)
1336                                                         == IRQ_HANDLED)
1337                                                 res = IRQ_HANDLED;
1338                         }
1339                 }
1340                 irq_msk >>= 4;
1341                 phy_no++;
1342         }
1343
1344         return res;
1345 }
1346
1347 static const struct hisi_sas_hw_error port_axi_error[] = {
1348         {
1349                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF),
1350                 .msg = "dma_tx_axi_wr_err",
1351         },
1352         {
1353                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF),
1354                 .msg = "dma_tx_axi_rd_err",
1355         },
1356         {
1357                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF),
1358                 .msg = "dma_rx_axi_wr_err",
1359         },
1360         {
1361                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF),
1362                 .msg = "dma_rx_axi_rd_err",
1363         },
1364 };
1365
1366 static void handle_chl_int1_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1367 {
1368         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1);
1369         u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT1_MSK);
1370         struct device *dev = hisi_hba->dev;
1371         int i;
1372
1373         irq_value &= ~irq_msk;
1374         if (!irq_value)
1375                 return;
1376
1377         for (i = 0; i < ARRAY_SIZE(port_axi_error); i++) {
1378                 const struct hisi_sas_hw_error *error = &port_axi_error[i];
1379
1380                 if (!(irq_value & error->irq_msk))
1381                         continue;
1382
1383                 dev_err(dev, "%s error (phy%d 0x%x) found!\n",
1384                         error->msg, phy_no, irq_value);
1385                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1386         }
1387
1388         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT1, irq_value);
1389 }
1390
1391 static void handle_chl_int2_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1392 {
1393         u32 irq_msk = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2_MSK);
1394         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no, CHL_INT2);
1395         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1396         struct pci_dev *pci_dev = hisi_hba->pci_dev;
1397         struct device *dev = hisi_hba->dev;
1398
1399         irq_value &= ~irq_msk;
1400         if (!irq_value)
1401                 return;
1402
1403         if (irq_value & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) {
1404                 dev_warn(dev, "phy%d identify timeout\n", phy_no);
1405                 hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1406         }
1407
1408         if (irq_value & BIT(CHL_INT2_STP_LINK_TIMEOUT_OFF)) {
1409                 u32 reg_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1410                                 STP_LINK_TIMEOUT_STATE);
1411
1412                 dev_warn(dev, "phy%d stp link timeout (0x%x)\n",
1413                          phy_no, reg_value);
1414                 if (reg_value & BIT(4))
1415                         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1416         }
1417
1418         if ((irq_value & BIT(CHL_INT2_RX_INVLD_DW_OFF)) &&
1419             (pci_dev->revision == 0x20)) {
1420                 u32 reg_value;
1421                 int rc;
1422
1423                 rc = hisi_sas_read32_poll_timeout_atomic(
1424                                 HILINK_ERR_DFX, reg_value,
1425                                 !((reg_value >> 8) & BIT(phy_no)),
1426                                 1000, 10000);
1427                 if (rc)
1428                         hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
1429         }
1430
1431         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, irq_value);
1432 }
1433
1434 static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
1435 {
1436         struct hisi_hba *hisi_hba = p;
1437         u32 irq_msk;
1438         int phy_no = 0;
1439
1440         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1441                                 & 0xeeeeeeee;
1442
1443         while (irq_msk) {
1444                 u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no,
1445                                                      CHL_INT0);
1446
1447                 if (irq_msk & (4 << (phy_no * 4)))
1448                         handle_chl_int1_v3_hw(hisi_hba, phy_no);
1449
1450                 if (irq_msk & (8 << (phy_no * 4)))
1451                         handle_chl_int2_v3_hw(hisi_hba, phy_no);
1452
1453                 if (irq_msk & (2 << (phy_no * 4)) && irq_value0) {
1454                         hisi_sas_phy_write32(hisi_hba, phy_no,
1455                                         CHL_INT0, irq_value0
1456                                         & (~CHL_INT0_SL_RX_BCST_ACK_MSK)
1457                                         & (~CHL_INT0_SL_PHY_ENABLE_MSK)
1458                                         & (~CHL_INT0_NOT_RDY_MSK));
1459                 }
1460                 irq_msk &= ~(0xe << (phy_no * 4));
1461                 phy_no++;
1462         }
1463
1464         return IRQ_HANDLED;
1465 }
1466
1467 static const struct hisi_sas_hw_error axi_error[] = {
1468         { .msk = BIT(0), .msg = "IOST_AXI_W_ERR" },
1469         { .msk = BIT(1), .msg = "IOST_AXI_R_ERR" },
1470         { .msk = BIT(2), .msg = "ITCT_AXI_W_ERR" },
1471         { .msk = BIT(3), .msg = "ITCT_AXI_R_ERR" },
1472         { .msk = BIT(4), .msg = "SATA_AXI_W_ERR" },
1473         { .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
1474         { .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
1475         { .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
1476         {},
1477 };
1478
1479 static const struct hisi_sas_hw_error fifo_error[] = {
1480         { .msk = BIT(8),  .msg = "CQE_WINFO_FIFO" },
1481         { .msk = BIT(9),  .msg = "CQE_MSG_FIFIO" },
1482         { .msk = BIT(10), .msg = "GETDQE_FIFO" },
1483         { .msk = BIT(11), .msg = "CMDP_FIFO" },
1484         { .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
1485         {},
1486 };
1487
1488 static const struct hisi_sas_hw_error fatal_axi_error[] = {
1489         {
1490                 .irq_msk = BIT(ENT_INT_SRC3_WP_DEPTH_OFF),
1491                 .msg = "write pointer and depth",
1492         },
1493         {
1494                 .irq_msk = BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF),
1495                 .msg = "iptt no match slot",
1496         },
1497         {
1498                 .irq_msk = BIT(ENT_INT_SRC3_RP_DEPTH_OFF),
1499                 .msg = "read pointer and depth",
1500         },
1501         {
1502                 .irq_msk = BIT(ENT_INT_SRC3_AXI_OFF),
1503                 .reg = HGC_AXI_FIFO_ERR_INFO,
1504                 .sub = axi_error,
1505         },
1506         {
1507                 .irq_msk = BIT(ENT_INT_SRC3_FIFO_OFF),
1508                 .reg = HGC_AXI_FIFO_ERR_INFO,
1509                 .sub = fifo_error,
1510         },
1511         {
1512                 .irq_msk = BIT(ENT_INT_SRC3_LM_OFF),
1513                 .msg = "LM add/fetch list",
1514         },
1515         {
1516                 .irq_msk = BIT(ENT_INT_SRC3_ABT_OFF),
1517                 .msg = "SAS_HGC_ABT fetch LM list",
1518         },
1519 };
1520
1521 static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
1522 {
1523         u32 irq_value, irq_msk;
1524         struct hisi_hba *hisi_hba = p;
1525         struct device *dev = hisi_hba->dev;
1526         int i;
1527
1528         irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
1529         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0x1df00);
1530
1531         irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
1532         irq_value &= ~irq_msk;
1533
1534         for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
1535                 const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
1536
1537                 if (!(irq_value & error->irq_msk))
1538                         continue;
1539
1540                 if (error->sub) {
1541                         const struct hisi_sas_hw_error *sub = error->sub;
1542                         u32 err_value = hisi_sas_read32(hisi_hba, error->reg);
1543
1544                         for (; sub->msk || sub->msg; sub++) {
1545                                 if (!(err_value & sub->msk))
1546                                         continue;
1547
1548                                 dev_err(dev, "%s error (0x%x) found!\n",
1549                                         sub->msg, irq_value);
1550                                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1551                         }
1552                 } else {
1553                         dev_err(dev, "%s error (0x%x) found!\n",
1554                                 error->msg, irq_value);
1555                         queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1556                 }
1557         }
1558
1559         if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
1560                 u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
1561                 u32 dev_id = reg_val & ITCT_DEV_MSK;
1562                 struct hisi_sas_device *sas_dev =
1563                                 &hisi_hba->devices[dev_id];
1564
1565                 hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
1566                 dev_dbg(dev, "clear ITCT ok\n");
1567                 complete(sas_dev->completion);
1568         }
1569
1570         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, irq_value & 0x1df00);
1571         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk);
1572
1573         return IRQ_HANDLED;
1574 }
1575
1576 static void
1577 slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task,
1578                struct hisi_sas_slot *slot)
1579 {
1580         struct task_status_struct *ts = &task->task_status;
1581         struct hisi_sas_complete_v3_hdr *complete_queue =
1582                         hisi_hba->complete_hdr[slot->cmplt_queue];
1583         struct hisi_sas_complete_v3_hdr *complete_hdr =
1584                         &complete_queue[slot->cmplt_queue_slot];
1585         struct hisi_sas_err_record_v3 *record =
1586                         hisi_sas_status_buf_addr_mem(slot);
1587         u32 dma_rx_err_type = record->dma_rx_err_type;
1588         u32 trans_tx_fail_type = record->trans_tx_fail_type;
1589
1590         switch (task->task_proto) {
1591         case SAS_PROTOCOL_SSP:
1592                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1593                         ts->residual = trans_tx_fail_type;
1594                         ts->stat = SAS_DATA_UNDERRUN;
1595                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1596                         ts->stat = SAS_QUEUE_FULL;
1597                         slot->abort = 1;
1598                 } else {
1599                         ts->stat = SAS_OPEN_REJECT;
1600                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1601                 }
1602                 break;
1603         case SAS_PROTOCOL_SATA:
1604         case SAS_PROTOCOL_STP:
1605         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1606                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1607                         ts->residual = trans_tx_fail_type;
1608                         ts->stat = SAS_DATA_UNDERRUN;
1609                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1610                         ts->stat = SAS_PHY_DOWN;
1611                         slot->abort = 1;
1612                 } else {
1613                         ts->stat = SAS_OPEN_REJECT;
1614                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1615                 }
1616                 hisi_sas_sata_done(task, slot);
1617                 break;
1618         case SAS_PROTOCOL_SMP:
1619                 ts->stat = SAM_STAT_CHECK_CONDITION;
1620                 break;
1621         default:
1622                 break;
1623         }
1624 }
1625
1626 static int
1627 slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
1628 {
1629         struct sas_task *task = slot->task;
1630         struct hisi_sas_device *sas_dev;
1631         struct device *dev = hisi_hba->dev;
1632         struct task_status_struct *ts;
1633         struct domain_device *device;
1634         struct sas_ha_struct *ha;
1635         enum exec_status sts;
1636         struct hisi_sas_complete_v3_hdr *complete_queue =
1637                         hisi_hba->complete_hdr[slot->cmplt_queue];
1638         struct hisi_sas_complete_v3_hdr *complete_hdr =
1639                         &complete_queue[slot->cmplt_queue_slot];
1640         unsigned long flags;
1641         bool is_internal = slot->is_internal;
1642
1643         if (unlikely(!task || !task->lldd_task || !task->dev))
1644                 return -EINVAL;
1645
1646         ts = &task->task_status;
1647         device = task->dev;
1648         ha = device->port->ha;
1649         sas_dev = device->lldd_dev;
1650
1651         spin_lock_irqsave(&task->task_state_lock, flags);
1652         task->task_state_flags &=
1653                 ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
1654         spin_unlock_irqrestore(&task->task_state_lock, flags);
1655
1656         memset(ts, 0, sizeof(*ts));
1657         ts->resp = SAS_TASK_COMPLETE;
1658
1659         if (unlikely(!sas_dev)) {
1660                 dev_dbg(dev, "slot complete: port has not device\n");
1661                 ts->stat = SAS_PHY_DOWN;
1662                 goto out;
1663         }
1664
1665         /*
1666          * Use SAS+TMF status codes
1667          */
1668         switch ((complete_hdr->dw0 & CMPLT_HDR_ABORT_STAT_MSK)
1669                         >> CMPLT_HDR_ABORT_STAT_OFF) {
1670         case STAT_IO_ABORTED:
1671                 /* this IO has been aborted by abort command */
1672                 ts->stat = SAS_ABORTED_TASK;
1673                 goto out;
1674         case STAT_IO_COMPLETE:
1675                 /* internal abort command complete */
1676                 ts->stat = TMF_RESP_FUNC_SUCC;
1677                 goto out;
1678         case STAT_IO_NO_DEVICE:
1679                 ts->stat = TMF_RESP_FUNC_COMPLETE;
1680                 goto out;
1681         case STAT_IO_NOT_VALID:
1682                 /*
1683                  * abort single IO, the controller can't find the IO
1684                  */
1685                 ts->stat = TMF_RESP_FUNC_FAILED;
1686                 goto out;
1687         default:
1688                 break;
1689         }
1690
1691         /* check for erroneous completion */
1692         if ((complete_hdr->dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) {
1693                 u32 *error_info = hisi_sas_status_buf_addr_mem(slot);
1694
1695                 slot_err_v3_hw(hisi_hba, task, slot);
1696                 if (ts->stat != SAS_DATA_UNDERRUN)
1697                         dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
1698                                 "CQ hdr: 0x%x 0x%x 0x%x 0x%x "
1699                                 "Error info: 0x%x 0x%x 0x%x 0x%x\n",
1700                                 slot->idx, task, sas_dev->device_id,
1701                                 complete_hdr->dw0, complete_hdr->dw1,
1702                                 complete_hdr->act, complete_hdr->dw3,
1703                                 error_info[0], error_info[1],
1704                                 error_info[2], error_info[3]);
1705                 if (unlikely(slot->abort))
1706                         return ts->stat;
1707                 goto out;
1708         }
1709
1710         switch (task->task_proto) {
1711         case SAS_PROTOCOL_SSP: {
1712                 struct ssp_response_iu *iu =
1713                         hisi_sas_status_buf_addr_mem(slot) +
1714                         sizeof(struct hisi_sas_err_record);
1715
1716                 sas_ssp_task_response(dev, task, iu);
1717                 break;
1718         }
1719         case SAS_PROTOCOL_SMP: {
1720                 struct scatterlist *sg_resp = &task->smp_task.smp_resp;
1721                 void *to;
1722
1723                 ts->stat = SAM_STAT_GOOD;
1724                 to = kmap_atomic(sg_page(sg_resp));
1725
1726                 dma_unmap_sg(dev, &task->smp_task.smp_resp, 1,
1727                              DMA_FROM_DEVICE);
1728                 dma_unmap_sg(dev, &task->smp_task.smp_req, 1,
1729                              DMA_TO_DEVICE);
1730                 memcpy(to + sg_resp->offset,
1731                         hisi_sas_status_buf_addr_mem(slot) +
1732                        sizeof(struct hisi_sas_err_record),
1733                        sg_dma_len(sg_resp));
1734                 kunmap_atomic(to);
1735                 break;
1736         }
1737         case SAS_PROTOCOL_SATA:
1738         case SAS_PROTOCOL_STP:
1739         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1740                 ts->stat = SAM_STAT_GOOD;
1741                 hisi_sas_sata_done(task, slot);
1742                 break;
1743         default:
1744                 ts->stat = SAM_STAT_CHECK_CONDITION;
1745                 break;
1746         }
1747
1748         if (!slot->port->port_attached) {
1749                 dev_warn(dev, "slot complete: port %d has removed\n",
1750                         slot->port->sas_port.id);
1751                 ts->stat = SAS_PHY_DOWN;
1752         }
1753
1754 out:
1755         sts = ts->stat;
1756         spin_lock_irqsave(&task->task_state_lock, flags);
1757         if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
1758                 spin_unlock_irqrestore(&task->task_state_lock, flags);
1759                 dev_info(dev, "slot complete: task(%p) aborted\n", task);
1760                 return SAS_ABORTED_TASK;
1761         }
1762         task->task_state_flags |= SAS_TASK_STATE_DONE;
1763         spin_unlock_irqrestore(&task->task_state_lock, flags);
1764         hisi_sas_slot_task_free(hisi_hba, task, slot);
1765
1766         if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
1767                 spin_lock_irqsave(&device->done_lock, flags);
1768                 if (test_bit(SAS_HA_FROZEN, &ha->state)) {
1769                         spin_unlock_irqrestore(&device->done_lock, flags);
1770                         dev_info(dev, "slot complete: task(%p) ignored\n ",
1771                                  task);
1772                         return sts;
1773                 }
1774                 spin_unlock_irqrestore(&device->done_lock, flags);
1775         }
1776
1777         if (task->task_done)
1778                 task->task_done(task);
1779
1780         return sts;
1781 }
1782
1783 static void cq_tasklet_v3_hw(unsigned long val)
1784 {
1785         struct hisi_sas_cq *cq = (struct hisi_sas_cq *)val;
1786         struct hisi_hba *hisi_hba = cq->hisi_hba;
1787         struct hisi_sas_slot *slot;
1788         struct hisi_sas_complete_v3_hdr *complete_queue;
1789         u32 rd_point = cq->rd_point, wr_point;
1790         int queue = cq->id;
1791
1792         complete_queue = hisi_hba->complete_hdr[queue];
1793
1794         wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
1795                                    (0x14 * queue));
1796
1797         while (rd_point != wr_point) {
1798                 struct hisi_sas_complete_v3_hdr *complete_hdr;
1799                 struct device *dev = hisi_hba->dev;
1800                 int iptt;
1801
1802                 complete_hdr = &complete_queue[rd_point];
1803
1804                 iptt = (complete_hdr->dw1) & CMPLT_HDR_IPTT_MSK;
1805                 if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
1806                         slot = &hisi_hba->slot_info[iptt];
1807                         slot->cmplt_queue_slot = rd_point;
1808                         slot->cmplt_queue = queue;
1809                         slot_complete_v3_hw(hisi_hba, slot);
1810                 } else
1811                         dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
1812
1813                 if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
1814                         rd_point = 0;
1815         }
1816
1817         /* update rd_point */
1818         cq->rd_point = rd_point;
1819         hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
1820 }
1821
1822 static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
1823 {
1824         struct hisi_sas_cq *cq = p;
1825         struct hisi_hba *hisi_hba = cq->hisi_hba;
1826         int queue = cq->id;
1827
1828         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue);
1829
1830         tasklet_schedule(&cq->tasklet);
1831
1832         return IRQ_HANDLED;
1833 }
1834
1835 static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
1836 {
1837         struct device *dev = hisi_hba->dev;
1838         struct pci_dev *pdev = hisi_hba->pci_dev;
1839         int vectors, rc;
1840         int i, k;
1841         int max_msi = HISI_SAS_MSI_COUNT_V3_HW;
1842
1843         vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, 1,
1844                                         max_msi, PCI_IRQ_MSI);
1845         if (vectors < max_msi) {
1846                 dev_err(dev, "could not allocate all msi (%d)\n", vectors);
1847                 return -ENOENT;
1848         }
1849
1850         rc = devm_request_irq(dev, pci_irq_vector(pdev, 1),
1851                               int_phy_up_down_bcast_v3_hw, 0,
1852                               DRV_NAME " phy", hisi_hba);
1853         if (rc) {
1854                 dev_err(dev, "could not request phy interrupt, rc=%d\n", rc);
1855                 rc = -ENOENT;
1856                 goto free_irq_vectors;
1857         }
1858
1859         rc = devm_request_irq(dev, pci_irq_vector(pdev, 2),
1860                               int_chnl_int_v3_hw, 0,
1861                               DRV_NAME " channel", hisi_hba);
1862         if (rc) {
1863                 dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc);
1864                 rc = -ENOENT;
1865                 goto free_phy_irq;
1866         }
1867
1868         rc = devm_request_irq(dev, pci_irq_vector(pdev, 11),
1869                               fatal_axi_int_v3_hw, 0,
1870                               DRV_NAME " fatal", hisi_hba);
1871         if (rc) {
1872                 dev_err(dev, "could not request fatal interrupt, rc=%d\n", rc);
1873                 rc = -ENOENT;
1874                 goto free_chnl_interrupt;
1875         }
1876
1877         /* Init tasklets for cq only */
1878         for (i = 0; i < hisi_hba->queue_count; i++) {
1879                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
1880                 struct tasklet_struct *t = &cq->tasklet;
1881
1882                 rc = devm_request_irq(dev, pci_irq_vector(pdev, i+16),
1883                                           cq_interrupt_v3_hw, 0,
1884                                           DRV_NAME " cq", cq);
1885                 if (rc) {
1886                         dev_err(dev,
1887                                 "could not request cq%d interrupt, rc=%d\n",
1888                                 i, rc);
1889                         rc = -ENOENT;
1890                         goto free_cq_irqs;
1891                 }
1892
1893                 tasklet_init(t, cq_tasklet_v3_hw, (unsigned long)cq);
1894         }
1895
1896         return 0;
1897
1898 free_cq_irqs:
1899         for (k = 0; k < i; k++) {
1900                 struct hisi_sas_cq *cq = &hisi_hba->cq[k];
1901
1902                 free_irq(pci_irq_vector(pdev, k+16), cq);
1903         }
1904         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
1905 free_chnl_interrupt:
1906         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
1907 free_phy_irq:
1908         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
1909 free_irq_vectors:
1910         pci_free_irq_vectors(pdev);
1911         return rc;
1912 }
1913
1914 static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
1915 {
1916         int rc;
1917
1918         rc = hw_init_v3_hw(hisi_hba);
1919         if (rc)
1920                 return rc;
1921
1922         rc = interrupt_init_v3_hw(hisi_hba);
1923         if (rc)
1924                 return rc;
1925
1926         return 0;
1927 }
1928
1929 static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
1930                 struct sas_phy_linkrates *r)
1931 {
1932         enum sas_linkrate max = r->maximum_linkrate;
1933         u32 prog_phy_link_rate = 0x800;
1934
1935         prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
1936         hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
1937                              prog_phy_link_rate);
1938 }
1939
1940 static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
1941 {
1942         struct pci_dev *pdev = hisi_hba->pci_dev;
1943         int i;
1944
1945         synchronize_irq(pci_irq_vector(pdev, 1));
1946         synchronize_irq(pci_irq_vector(pdev, 2));
1947         synchronize_irq(pci_irq_vector(pdev, 11));
1948         for (i = 0; i < hisi_hba->queue_count; i++) {
1949                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0x1);
1950                 synchronize_irq(pci_irq_vector(pdev, i + 16));
1951         }
1952
1953         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xffffffff);
1954         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xffffffff);
1955         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffffffff);
1956         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
1957
1958         for (i = 0; i < hisi_hba->n_phy; i++) {
1959                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
1960                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffffff);
1961                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x1);
1962                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x1);
1963                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x1);
1964         }
1965 }
1966
1967 static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba)
1968 {
1969         return hisi_sas_read32(hisi_hba, PHY_STATE);
1970 }
1971
1972 static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1973 {
1974         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1975         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1976         struct sas_phy *sphy = sas_phy->phy;
1977         u32 reg_value;
1978
1979         /* loss dword sync */
1980         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST);
1981         sphy->loss_of_dword_sync_count += reg_value;
1982
1983         /* phy reset problem */
1984         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB);
1985         sphy->phy_reset_problem_count += reg_value;
1986
1987         /* invalid dword */
1988         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1989         sphy->invalid_dword_count += reg_value;
1990
1991         /* disparity err */
1992         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
1993         sphy->running_disparity_error_count += reg_value;
1994
1995 }
1996
1997 static int disable_host_v3_hw(struct hisi_hba *hisi_hba)
1998 {
1999         struct device *dev = hisi_hba->dev;
2000         u32 status, reg_val;
2001         int rc;
2002
2003         interrupt_disable_v3_hw(hisi_hba);
2004         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
2005         hisi_sas_kill_tasklets(hisi_hba);
2006
2007         hisi_sas_stop_phys(hisi_hba);
2008
2009         mdelay(10);
2010
2011         reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
2012                                   AM_CTRL_GLOBAL);
2013         reg_val |= AM_CTRL_SHUTDOWN_REQ_MSK;
2014         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2015                          AM_CTRL_GLOBAL, reg_val);
2016
2017         /* wait until bus idle */
2018         rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
2019                                           AM_CURR_TRANS_RETURN, status,
2020                                           status == 0x3, 10, 100);
2021         if (rc) {
2022                 dev_err(dev, "axi bus is not idle, rc=%d\n", rc);
2023                 return rc;
2024         }
2025
2026         return 0;
2027 }
2028
2029 static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
2030 {
2031         struct device *dev = hisi_hba->dev;
2032         int rc;
2033
2034         rc = disable_host_v3_hw(hisi_hba);
2035         if (rc) {
2036                 dev_err(dev, "soft reset: disable host failed rc=%d\n", rc);
2037                 return rc;
2038         }
2039
2040         hisi_sas_init_mem(hisi_hba);
2041
2042         return hw_init_v3_hw(hisi_hba);
2043 }
2044
2045 static int write_gpio_v3_hw(struct hisi_hba *hisi_hba, u8 reg_type,
2046                         u8 reg_index, u8 reg_count, u8 *write_data)
2047 {
2048         struct device *dev = hisi_hba->dev;
2049         u32 *data = (u32 *)write_data;
2050         int i;
2051
2052         switch (reg_type) {
2053         case SAS_GPIO_REG_TX:
2054                 if ((reg_index + reg_count) > ((hisi_hba->n_phy + 3) / 4)) {
2055                         dev_err(dev, "write gpio: invalid reg range[%d, %d]\n",
2056                                 reg_index, reg_index + reg_count - 1);
2057                         return -EINVAL;
2058                 }
2059
2060                 for (i = 0; i < reg_count; i++)
2061                         hisi_sas_write32(hisi_hba,
2062                                          SAS_GPIO_TX_0_1 + (reg_index + i) * 4,
2063                                          data[i]);
2064                 break;
2065         default:
2066                 dev_err(dev, "write gpio: unsupported or bad reg type %d\n",
2067                                 reg_type);
2068                 return -EINVAL;
2069         }
2070
2071         return 0;
2072 }
2073
2074 static void wait_cmds_complete_timeout_v3_hw(struct hisi_hba *hisi_hba,
2075                                              int delay_ms, int timeout_ms)
2076 {
2077         struct device *dev = hisi_hba->dev;
2078         int entries, entries_old = 0, time;
2079
2080         for (time = 0; time < timeout_ms; time += delay_ms) {
2081                 entries = hisi_sas_read32(hisi_hba, CQE_SEND_CNT);
2082                 if (entries == entries_old)
2083                         break;
2084
2085                 entries_old = entries;
2086                 msleep(delay_ms);
2087         }
2088
2089         dev_dbg(dev, "wait commands complete %dms\n", time);
2090 }
2091
2092 static struct scsi_host_template sht_v3_hw = {
2093         .name                   = DRV_NAME,
2094         .module                 = THIS_MODULE,
2095         .queuecommand           = sas_queuecommand,
2096         .target_alloc           = sas_target_alloc,
2097         .slave_configure        = hisi_sas_slave_configure,
2098         .scan_finished          = hisi_sas_scan_finished,
2099         .scan_start             = hisi_sas_scan_start,
2100         .change_queue_depth     = sas_change_queue_depth,
2101         .bios_param             = sas_bios_param,
2102         .this_id                = -1,
2103         .sg_tablesize           = SG_ALL,
2104         .max_sectors            = SCSI_DEFAULT_MAX_SECTORS,
2105         .use_clustering         = ENABLE_CLUSTERING,
2106         .eh_device_reset_handler = sas_eh_device_reset_handler,
2107         .eh_target_reset_handler = sas_eh_target_reset_handler,
2108         .target_destroy         = sas_target_destroy,
2109         .ioctl                  = sas_ioctl,
2110         .shost_attrs            = host_attrs,
2111         .tag_alloc_policy       = BLK_TAG_ALLOC_RR,
2112 };
2113
2114 static const struct hisi_sas_hw hisi_sas_v3_hw = {
2115         .hw_init = hisi_sas_v3_init,
2116         .setup_itct = setup_itct_v3_hw,
2117         .max_command_entries = HISI_SAS_COMMAND_ENTRIES_V3_HW,
2118         .get_wideport_bitmap = get_wideport_bitmap_v3_hw,
2119         .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr),
2120         .clear_itct = clear_itct_v3_hw,
2121         .sl_notify = sl_notify_v3_hw,
2122         .prep_ssp = prep_ssp_v3_hw,
2123         .prep_smp = prep_smp_v3_hw,
2124         .prep_stp = prep_ata_v3_hw,
2125         .prep_abort = prep_abort_v3_hw,
2126         .get_free_slot = get_free_slot_v3_hw,
2127         .start_delivery = start_delivery_v3_hw,
2128         .slot_complete = slot_complete_v3_hw,
2129         .phys_init = phys_init_v3_hw,
2130         .phy_start = start_phy_v3_hw,
2131         .phy_disable = disable_phy_v3_hw,
2132         .phy_hard_reset = phy_hard_reset_v3_hw,
2133         .phy_get_max_linkrate = phy_get_max_linkrate_v3_hw,
2134         .phy_set_linkrate = phy_set_linkrate_v3_hw,
2135         .dereg_device = dereg_device_v3_hw,
2136         .soft_reset = soft_reset_v3_hw,
2137         .get_phys_state = get_phys_state_v3_hw,
2138         .get_events = phy_get_events_v3_hw,
2139         .write_gpio = write_gpio_v3_hw,
2140         .wait_cmds_complete_timeout = wait_cmds_complete_timeout_v3_hw,
2141 };
2142
2143 static struct Scsi_Host *
2144 hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
2145 {
2146         struct Scsi_Host *shost;
2147         struct hisi_hba *hisi_hba;
2148         struct device *dev = &pdev->dev;
2149
2150         shost = scsi_host_alloc(&sht_v3_hw, sizeof(*hisi_hba));
2151         if (!shost) {
2152                 dev_err(dev, "shost alloc failed\n");
2153                 return NULL;
2154         }
2155         hisi_hba = shost_priv(shost);
2156
2157         INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler);
2158         hisi_hba->hw = &hisi_sas_v3_hw;
2159         hisi_hba->pci_dev = pdev;
2160         hisi_hba->dev = dev;
2161         hisi_hba->shost = shost;
2162         SHOST_TO_SAS_HA(shost) = &hisi_hba->sha;
2163
2164         timer_setup(&hisi_hba->timer, NULL, 0);
2165
2166         if (hisi_sas_get_fw_info(hisi_hba) < 0)
2167                 goto err_out;
2168
2169         if (hisi_sas_alloc(hisi_hba, shost)) {
2170                 hisi_sas_free(hisi_hba);
2171                 goto err_out;
2172         }
2173
2174         return shost;
2175 err_out:
2176         scsi_host_put(shost);
2177         dev_err(dev, "shost alloc failed\n");
2178         return NULL;
2179 }
2180
2181 static int
2182 hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2183 {
2184         struct Scsi_Host *shost;
2185         struct hisi_hba *hisi_hba;
2186         struct device *dev = &pdev->dev;
2187         struct asd_sas_phy **arr_phy;
2188         struct asd_sas_port **arr_port;
2189         struct sas_ha_struct *sha;
2190         int rc, phy_nr, port_nr, i;
2191
2192         rc = pci_enable_device(pdev);
2193         if (rc)
2194                 goto err_out;
2195
2196         pci_set_master(pdev);
2197
2198         rc = pci_request_regions(pdev, DRV_NAME);
2199         if (rc)
2200                 goto err_out_disable_device;
2201
2202         if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) ||
2203             (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0)) {
2204                 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) ||
2205                    (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)) != 0)) {
2206                         dev_err(dev, "No usable DMA addressing method\n");
2207                         rc = -EIO;
2208                         goto err_out_regions;
2209                 }
2210         }
2211
2212         shost = hisi_sas_shost_alloc_pci(pdev);
2213         if (!shost) {
2214                 rc = -ENOMEM;
2215                 goto err_out_regions;
2216         }
2217
2218         sha = SHOST_TO_SAS_HA(shost);
2219         hisi_hba = shost_priv(shost);
2220         dev_set_drvdata(dev, sha);
2221
2222         hisi_hba->regs = pcim_iomap(pdev, 5, 0);
2223         if (!hisi_hba->regs) {
2224                 dev_err(dev, "cannot map register.\n");
2225                 rc = -ENOMEM;
2226                 goto err_out_ha;
2227         }
2228
2229         phy_nr = port_nr = hisi_hba->n_phy;
2230
2231         arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
2232         arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL);
2233         if (!arr_phy || !arr_port) {
2234                 rc = -ENOMEM;
2235                 goto err_out_ha;
2236         }
2237
2238         sha->sas_phy = arr_phy;
2239         sha->sas_port = arr_port;
2240         sha->core.shost = shost;
2241         sha->lldd_ha = hisi_hba;
2242
2243         shost->transportt = hisi_sas_stt;
2244         shost->max_id = HISI_SAS_MAX_DEVICES;
2245         shost->max_lun = ~0;
2246         shost->max_channel = 1;
2247         shost->max_cmd_len = 16;
2248         shost->sg_tablesize = min_t(u16, SG_ALL, HISI_SAS_SGE_PAGE_CNT);
2249         shost->can_queue = hisi_hba->hw->max_command_entries -
2250                 HISI_SAS_RESERVED_IPTT_CNT;
2251         shost->cmd_per_lun = hisi_hba->hw->max_command_entries -
2252                 HISI_SAS_RESERVED_IPTT_CNT;
2253
2254         sha->sas_ha_name = DRV_NAME;
2255         sha->dev = dev;
2256         sha->lldd_module = THIS_MODULE;
2257         sha->sas_addr = &hisi_hba->sas_addr[0];
2258         sha->num_phys = hisi_hba->n_phy;
2259         sha->core.shost = hisi_hba->shost;
2260
2261         for (i = 0; i < hisi_hba->n_phy; i++) {
2262                 sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy;
2263                 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
2264         }
2265
2266         rc = scsi_add_host(shost, dev);
2267         if (rc)
2268                 goto err_out_ha;
2269
2270         rc = sas_register_ha(sha);
2271         if (rc)
2272                 goto err_out_register_ha;
2273
2274         rc = hisi_hba->hw->hw_init(hisi_hba);
2275         if (rc)
2276                 goto err_out_register_ha;
2277
2278         scsi_scan_host(shost);
2279
2280         return 0;
2281
2282 err_out_register_ha:
2283         scsi_remove_host(shost);
2284 err_out_ha:
2285         scsi_host_put(shost);
2286 err_out_regions:
2287         pci_release_regions(pdev);
2288 err_out_disable_device:
2289         pci_disable_device(pdev);
2290 err_out:
2291         return rc;
2292 }
2293
2294 static void
2295 hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
2296 {
2297         int i;
2298
2299         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
2300         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
2301         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
2302         for (i = 0; i < hisi_hba->queue_count; i++) {
2303                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2304
2305                 free_irq(pci_irq_vector(pdev, i+16), cq);
2306         }
2307         pci_free_irq_vectors(pdev);
2308 }
2309
2310 static void hisi_sas_v3_remove(struct pci_dev *pdev)
2311 {
2312         struct device *dev = &pdev->dev;
2313         struct sas_ha_struct *sha = dev_get_drvdata(dev);
2314         struct hisi_hba *hisi_hba = sha->lldd_ha;
2315         struct Scsi_Host *shost = sha->core.shost;
2316
2317         if (timer_pending(&hisi_hba->timer))
2318                 del_timer(&hisi_hba->timer);
2319
2320         sas_unregister_ha(sha);
2321         sas_remove_host(sha->core.shost);
2322
2323         hisi_sas_v3_destroy_irqs(pdev, hisi_hba);
2324         hisi_sas_kill_tasklets(hisi_hba);
2325         pci_release_regions(pdev);
2326         pci_disable_device(pdev);
2327         hisi_sas_free(hisi_hba);
2328         scsi_host_put(shost);
2329 }
2330
2331 static const struct hisi_sas_hw_error sas_ras_intr0_nfe[] = {
2332         { .irq_msk = BIT(19), .msg = "HILINK_INT" },
2333         { .irq_msk = BIT(20), .msg = "HILINK_PLL0_OUT_OF_LOCK" },
2334         { .irq_msk = BIT(21), .msg = "HILINK_PLL1_OUT_OF_LOCK" },
2335         { .irq_msk = BIT(22), .msg = "HILINK_LOSS_OF_REFCLK0" },
2336         { .irq_msk = BIT(23), .msg = "HILINK_LOSS_OF_REFCLK1" },
2337         { .irq_msk = BIT(24), .msg = "DMAC0_TX_POISON" },
2338         { .irq_msk = BIT(25), .msg = "DMAC1_TX_POISON" },
2339         { .irq_msk = BIT(26), .msg = "DMAC2_TX_POISON" },
2340         { .irq_msk = BIT(27), .msg = "DMAC3_TX_POISON" },
2341         { .irq_msk = BIT(28), .msg = "DMAC4_TX_POISON" },
2342         { .irq_msk = BIT(29), .msg = "DMAC5_TX_POISON" },
2343         { .irq_msk = BIT(30), .msg = "DMAC6_TX_POISON" },
2344         { .irq_msk = BIT(31), .msg = "DMAC7_TX_POISON" },
2345 };
2346
2347 static const struct hisi_sas_hw_error sas_ras_intr1_nfe[] = {
2348         { .irq_msk = BIT(0), .msg = "RXM_CFG_MEM3_ECC2B_INTR" },
2349         { .irq_msk = BIT(1), .msg = "RXM_CFG_MEM2_ECC2B_INTR" },
2350         { .irq_msk = BIT(2), .msg = "RXM_CFG_MEM1_ECC2B_INTR" },
2351         { .irq_msk = BIT(3), .msg = "RXM_CFG_MEM0_ECC2B_INTR" },
2352         { .irq_msk = BIT(4), .msg = "HGC_CQE_ECC2B_INTR" },
2353         { .irq_msk = BIT(5), .msg = "LM_CFG_IOSTL_ECC2B_INTR" },
2354         { .irq_msk = BIT(6), .msg = "LM_CFG_ITCTL_ECC2B_INTR" },
2355         { .irq_msk = BIT(7), .msg = "HGC_ITCT_ECC2B_INTR" },
2356         { .irq_msk = BIT(8), .msg = "HGC_IOST_ECC2B_INTR" },
2357         { .irq_msk = BIT(9), .msg = "HGC_DQE_ECC2B_INTR" },
2358         { .irq_msk = BIT(10), .msg = "DMAC0_RAM_ECC2B_INTR" },
2359         { .irq_msk = BIT(11), .msg = "DMAC1_RAM_ECC2B_INTR" },
2360         { .irq_msk = BIT(12), .msg = "DMAC2_RAM_ECC2B_INTR" },
2361         { .irq_msk = BIT(13), .msg = "DMAC3_RAM_ECC2B_INTR" },
2362         { .irq_msk = BIT(14), .msg = "DMAC4_RAM_ECC2B_INTR" },
2363         { .irq_msk = BIT(15), .msg = "DMAC5_RAM_ECC2B_INTR" },
2364         { .irq_msk = BIT(16), .msg = "DMAC6_RAM_ECC2B_INTR" },
2365         { .irq_msk = BIT(17), .msg = "DMAC7_RAM_ECC2B_INTR" },
2366         { .irq_msk = BIT(18), .msg = "OOO_RAM_ECC2B_INTR" },
2367         { .irq_msk = BIT(20), .msg = "HGC_DQE_POISON_INTR" },
2368         { .irq_msk = BIT(21), .msg = "HGC_IOST_POISON_INTR" },
2369         { .irq_msk = BIT(22), .msg = "HGC_ITCT_POISON_INTR" },
2370         { .irq_msk = BIT(23), .msg = "HGC_ITCT_NCQ_POISON_INTR" },
2371         { .irq_msk = BIT(24), .msg = "DMAC0_RX_POISON" },
2372         { .irq_msk = BIT(25), .msg = "DMAC1_RX_POISON" },
2373         { .irq_msk = BIT(26), .msg = "DMAC2_RX_POISON" },
2374         { .irq_msk = BIT(27), .msg = "DMAC3_RX_POISON" },
2375         { .irq_msk = BIT(28), .msg = "DMAC4_RX_POISON" },
2376         { .irq_msk = BIT(29), .msg = "DMAC5_RX_POISON" },
2377         { .irq_msk = BIT(30), .msg = "DMAC6_RX_POISON" },
2378         { .irq_msk = BIT(31), .msg = "DMAC7_RX_POISON" },
2379 };
2380
2381 static const struct hisi_sas_hw_error sas_ras_intr2_nfe[] = {
2382         { .irq_msk = BIT(0), .msg = "DMAC0_AXI_BUS_ERR" },
2383         { .irq_msk = BIT(1), .msg = "DMAC1_AXI_BUS_ERR" },
2384         { .irq_msk = BIT(2), .msg = "DMAC2_AXI_BUS_ERR" },
2385         { .irq_msk = BIT(3), .msg = "DMAC3_AXI_BUS_ERR" },
2386         { .irq_msk = BIT(4), .msg = "DMAC4_AXI_BUS_ERR" },
2387         { .irq_msk = BIT(5), .msg = "DMAC5_AXI_BUS_ERR" },
2388         { .irq_msk = BIT(6), .msg = "DMAC6_AXI_BUS_ERR" },
2389         { .irq_msk = BIT(7), .msg = "DMAC7_AXI_BUS_ERR" },
2390         { .irq_msk = BIT(8), .msg = "DMAC0_FIFO_OMIT_ERR" },
2391         { .irq_msk = BIT(9), .msg = "DMAC1_FIFO_OMIT_ERR" },
2392         { .irq_msk = BIT(10), .msg = "DMAC2_FIFO_OMIT_ERR" },
2393         { .irq_msk = BIT(11), .msg = "DMAC3_FIFO_OMIT_ERR" },
2394         { .irq_msk = BIT(12), .msg = "DMAC4_FIFO_OMIT_ERR" },
2395         { .irq_msk = BIT(13), .msg = "DMAC5_FIFO_OMIT_ERR" },
2396         { .irq_msk = BIT(14), .msg = "DMAC6_FIFO_OMIT_ERR" },
2397         { .irq_msk = BIT(15), .msg = "DMAC7_FIFO_OMIT_ERR" },
2398         { .irq_msk = BIT(16), .msg = "HGC_RLSE_SLOT_UNMATCH" },
2399         { .irq_msk = BIT(17), .msg = "HGC_LM_ADD_FCH_LIST_ERR" },
2400         { .irq_msk = BIT(18), .msg = "HGC_AXI_BUS_ERR" },
2401         { .irq_msk = BIT(19), .msg = "HGC_FIFO_OMIT_ERR" },
2402 };
2403
2404 static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
2405 {
2406         struct device *dev = hisi_hba->dev;
2407         const struct hisi_sas_hw_error *ras_error;
2408         bool need_reset = false;
2409         u32 irq_value;
2410         int i;
2411
2412         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR0);
2413         for (i = 0; i < ARRAY_SIZE(sas_ras_intr0_nfe); i++) {
2414                 ras_error = &sas_ras_intr0_nfe[i];
2415                 if (ras_error->irq_msk & irq_value) {
2416                         dev_warn(dev, "SAS_RAS_INTR0: %s(irq_value=0x%x) found.\n",
2417                                         ras_error->msg, irq_value);
2418                         need_reset = true;
2419                 }
2420         }
2421         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0, irq_value);
2422
2423         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR1);
2424         for (i = 0; i < ARRAY_SIZE(sas_ras_intr1_nfe); i++) {
2425                 ras_error = &sas_ras_intr1_nfe[i];
2426                 if (ras_error->irq_msk & irq_value) {
2427                         dev_warn(dev, "SAS_RAS_INTR1: %s(irq_value=0x%x) found.\n",
2428                                         ras_error->msg, irq_value);
2429                         need_reset = true;
2430                 }
2431         }
2432         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1, irq_value);
2433
2434         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR2);
2435         for (i = 0; i < ARRAY_SIZE(sas_ras_intr2_nfe); i++) {
2436                 ras_error = &sas_ras_intr2_nfe[i];
2437                 if (ras_error->irq_msk & irq_value) {
2438                         dev_warn(dev, "SAS_RAS_INTR2: %s(irq_value=0x%x) found.\n",
2439                                         ras_error->msg, irq_value);
2440                         need_reset = true;
2441                 }
2442         }
2443         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2, irq_value);
2444
2445         return need_reset;
2446 }
2447
2448 static pci_ers_result_t hisi_sas_error_detected_v3_hw(struct pci_dev *pdev,
2449                 pci_channel_state_t state)
2450 {
2451         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2452         struct hisi_hba *hisi_hba = sha->lldd_ha;
2453         struct device *dev = hisi_hba->dev;
2454
2455         dev_info(dev, "PCI error: detected callback, state(%d)!!\n", state);
2456         if (state == pci_channel_io_perm_failure)
2457                 return PCI_ERS_RESULT_DISCONNECT;
2458
2459         if (process_non_fatal_error_v3_hw(hisi_hba))
2460                 return PCI_ERS_RESULT_NEED_RESET;
2461
2462         return PCI_ERS_RESULT_CAN_RECOVER;
2463 }
2464
2465 static pci_ers_result_t hisi_sas_mmio_enabled_v3_hw(struct pci_dev *pdev)
2466 {
2467         return PCI_ERS_RESULT_RECOVERED;
2468 }
2469
2470 static pci_ers_result_t hisi_sas_slot_reset_v3_hw(struct pci_dev *pdev)
2471 {
2472         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2473         struct hisi_hba *hisi_hba = sha->lldd_ha;
2474         struct device *dev = hisi_hba->dev;
2475         HISI_SAS_DECLARE_RST_WORK_ON_STACK(r);
2476
2477         dev_info(dev, "PCI error: slot reset callback!!\n");
2478         queue_work(hisi_hba->wq, &r.work);
2479         wait_for_completion(r.completion);
2480         if (r.done)
2481                 return PCI_ERS_RESULT_RECOVERED;
2482
2483         return PCI_ERS_RESULT_DISCONNECT;
2484 }
2485
2486 static void hisi_sas_reset_prepare_v3_hw(struct pci_dev *pdev)
2487 {
2488         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2489         struct hisi_hba *hisi_hba = sha->lldd_ha;
2490         struct device *dev = hisi_hba->dev;
2491         int rc;
2492
2493         dev_info(dev, "FLR prepare\n");
2494         set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2495         hisi_sas_controller_reset_prepare(hisi_hba);
2496
2497         rc = disable_host_v3_hw(hisi_hba);
2498         if (rc)
2499                 dev_err(dev, "FLR: disable host failed rc=%d\n", rc);
2500 }
2501
2502 static void hisi_sas_reset_done_v3_hw(struct pci_dev *pdev)
2503 {
2504         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2505         struct hisi_hba *hisi_hba = sha->lldd_ha;
2506         struct device *dev = hisi_hba->dev;
2507         int rc;
2508
2509         hisi_sas_init_mem(hisi_hba);
2510
2511         rc = hw_init_v3_hw(hisi_hba);
2512         if (rc) {
2513                 dev_err(dev, "FLR: hw init failed rc=%d\n", rc);
2514                 return;
2515         }
2516
2517         hisi_sas_controller_reset_done(hisi_hba);
2518         dev_info(dev, "FLR done\n");
2519 }
2520
2521 enum {
2522         /* instances of the controller */
2523         hip08,
2524 };
2525
2526 static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
2527 {
2528         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2529         struct hisi_hba *hisi_hba = sha->lldd_ha;
2530         struct device *dev = hisi_hba->dev;
2531         struct Scsi_Host *shost = hisi_hba->shost;
2532         u32 device_state;
2533         int rc;
2534
2535         if (!pdev->pm_cap) {
2536                 dev_err(dev, "PCI PM not supported\n");
2537                 return -ENODEV;
2538         }
2539
2540         if (test_and_set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags))
2541                 return -1;
2542
2543         scsi_block_requests(shost);
2544         set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2545         flush_workqueue(hisi_hba->wq);
2546
2547         rc = disable_host_v3_hw(hisi_hba);
2548         if (rc) {
2549                 dev_err(dev, "PM suspend: disable host failed rc=%d\n", rc);
2550                 clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2551                 clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2552                 scsi_unblock_requests(shost);
2553                 return rc;
2554         }
2555
2556         hisi_sas_init_mem(hisi_hba);
2557
2558         device_state = pci_choose_state(pdev, state);
2559         dev_warn(dev, "entering operating state [D%d]\n",
2560                         device_state);
2561         pci_save_state(pdev);
2562         pci_disable_device(pdev);
2563         pci_set_power_state(pdev, device_state);
2564
2565         hisi_sas_release_tasks(hisi_hba);
2566
2567         sas_suspend_ha(sha);
2568         return 0;
2569 }
2570
2571 static int hisi_sas_v3_resume(struct pci_dev *pdev)
2572 {
2573         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2574         struct hisi_hba *hisi_hba = sha->lldd_ha;
2575         struct Scsi_Host *shost = hisi_hba->shost;
2576         struct device *dev = hisi_hba->dev;
2577         unsigned int rc;
2578         u32 device_state = pdev->current_state;
2579
2580         dev_warn(dev, "resuming from operating state [D%d]\n",
2581                         device_state);
2582         pci_set_power_state(pdev, PCI_D0);
2583         pci_enable_wake(pdev, PCI_D0, 0);
2584         pci_restore_state(pdev);
2585         rc = pci_enable_device(pdev);
2586         if (rc)
2587                 dev_err(dev, "enable device failed during resume (%d)\n", rc);
2588
2589         pci_set_master(pdev);
2590         scsi_unblock_requests(shost);
2591         clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2592
2593         sas_prep_resume_ha(sha);
2594         init_reg_v3_hw(hisi_hba);
2595         hisi_hba->hw->phys_init(hisi_hba);
2596         sas_resume_ha(sha);
2597         clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2598
2599         return 0;
2600 }
2601
2602 static const struct pci_device_id sas_v3_pci_table[] = {
2603         { PCI_VDEVICE(HUAWEI, 0xa230), hip08 },
2604         {}
2605 };
2606 MODULE_DEVICE_TABLE(pci, sas_v3_pci_table);
2607
2608 static const struct pci_error_handlers hisi_sas_err_handler = {
2609         .error_detected = hisi_sas_error_detected_v3_hw,
2610         .mmio_enabled   = hisi_sas_mmio_enabled_v3_hw,
2611         .slot_reset     = hisi_sas_slot_reset_v3_hw,
2612         .reset_prepare  = hisi_sas_reset_prepare_v3_hw,
2613         .reset_done     = hisi_sas_reset_done_v3_hw,
2614 };
2615
2616 static struct pci_driver sas_v3_pci_driver = {
2617         .name           = DRV_NAME,
2618         .id_table       = sas_v3_pci_table,
2619         .probe          = hisi_sas_v3_probe,
2620         .remove         = hisi_sas_v3_remove,
2621         .suspend        = hisi_sas_v3_suspend,
2622         .resume         = hisi_sas_v3_resume,
2623         .err_handler    = &hisi_sas_err_handler,
2624 };
2625
2626 module_pci_driver(sas_v3_pci_driver);
2627
2628 MODULE_LICENSE("GPL");
2629 MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
2630 MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device");
2631 MODULE_ALIAS("pci:" DRV_NAME);