drivers/scsi/hisi_sas/hisi_sas_v3_hw.c

   1 /*
   2  * Copyright (c) 2017 Hisilicon Limited.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  */
  10
  11 #include "hisi_sas.h"
  12 #define DRV_NAME "hisi_sas_v3_hw"
  13
  14 /* global registers need init*/
  15 #define DLVRY_QUEUE_ENABLE              0x0
  16 #define IOST_BASE_ADDR_LO               0x8
  17 #define IOST_BASE_ADDR_HI               0xc
  18 #define ITCT_BASE_ADDR_LO               0x10
  19 #define ITCT_BASE_ADDR_HI               0x14
  20 #define IO_BROKEN_MSG_ADDR_LO           0x18
  21 #define IO_BROKEN_MSG_ADDR_HI           0x1c
  22 #define PHY_CONTEXT                     0x20
  23 #define PHY_STATE                       0x24
  24 #define PHY_PORT_NUM_MA                 0x28
  25 #define PHY_CONN_RATE                   0x30
  26 #define ITCT_CLR                        0x44
  27 #define ITCT_CLR_EN_OFF                 16
  28 #define ITCT_CLR_EN_MSK                 (0x1 << ITCT_CLR_EN_OFF)
  29 #define ITCT_DEV_OFF                    0
  30 #define ITCT_DEV_MSK                    (0x7ff << ITCT_DEV_OFF)
  31 #define IO_SATA_BROKEN_MSG_ADDR_LO      0x58
  32 #define IO_SATA_BROKEN_MSG_ADDR_HI      0x5c
  33 #define SATA_INITI_D2H_STORE_ADDR_LO    0x60
  34 #define SATA_INITI_D2H_STORE_ADDR_HI    0x64
  35 #define CFG_MAX_TAG                     0x68
  36 #define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL 0x84
  37 #define HGC_SAS_TXFAIL_RETRY_CTRL       0x88
  38 #define HGC_GET_ITV_TIME                0x90
  39 #define DEVICE_MSG_WORK_MODE            0x94
  40 #define OPENA_WT_CONTI_TIME             0x9c
  41 #define I_T_NEXUS_LOSS_TIME             0xa0
  42 #define MAX_CON_TIME_LIMIT_TIME         0xa4
  43 #define BUS_INACTIVE_LIMIT_TIME         0xa8
  44 #define REJECT_TO_OPEN_LIMIT_TIME       0xac
  45 #define CFG_AGING_TIME                  0xbc
  46 #define HGC_DFX_CFG2                    0xc0
  47 #define CFG_ABT_SET_QUERY_IPTT  0xd4
  48 #define CFG_SET_ABORTED_IPTT_OFF        0
  49 #define CFG_SET_ABORTED_IPTT_MSK        (0xfff << CFG_SET_ABORTED_IPTT_OFF)
  50 #define CFG_SET_ABORTED_EN_OFF  12
  51 #define CFG_ABT_SET_IPTT_DONE   0xd8
  52 #define CFG_ABT_SET_IPTT_DONE_OFF       0
  53 #define HGC_IOMB_PROC1_STATUS   0x104
  54 #define CFG_1US_TIMER_TRSH              0xcc
  55 #define CHNL_INT_STATUS                 0x148
  56 #define HGC_AXI_FIFO_ERR_INFO  0x154
  57 #define AXI_ERR_INFO_OFF               0
  58 #define AXI_ERR_INFO_MSK               (0xff << AXI_ERR_INFO_OFF)
  59 #define FIFO_ERR_INFO_OFF              8
  60 #define FIFO_ERR_INFO_MSK              (0xff << FIFO_ERR_INFO_OFF)
  61 #define INT_COAL_EN                     0x19c
  62 #define OQ_INT_COAL_TIME                0x1a0
  63 #define OQ_INT_COAL_CNT                 0x1a4
  64 #define ENT_INT_COAL_TIME               0x1a8
  65 #define ENT_INT_COAL_CNT                0x1ac
  66 #define OQ_INT_SRC                      0x1b0
  67 #define OQ_INT_SRC_MSK                  0x1b4
  68 #define ENT_INT_SRC1                    0x1b8
  69 #define ENT_INT_SRC1_D2H_FIS_CH0_OFF    0
  70 #define ENT_INT_SRC1_D2H_FIS_CH0_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH0_OFF)
  71 #define ENT_INT_SRC1_D2H_FIS_CH1_OFF    8
  72 #define ENT_INT_SRC1_D2H_FIS_CH1_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF)
  73 #define ENT_INT_SRC2                    0x1bc
  74 #define ENT_INT_SRC3                    0x1c0
  75 #define ENT_INT_SRC3_WP_DEPTH_OFF               8
  76 #define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF      9
  77 #define ENT_INT_SRC3_RP_DEPTH_OFF               10
  78 #define ENT_INT_SRC3_AXI_OFF                    11
  79 #define ENT_INT_SRC3_FIFO_OFF                   12
  80 #define ENT_INT_SRC3_LM_OFF                             14
  81 #define ENT_INT_SRC3_ITC_INT_OFF        15
  82 #define ENT_INT_SRC3_ITC_INT_MSK        (0x1 << ENT_INT_SRC3_ITC_INT_OFF)
  83 #define ENT_INT_SRC3_ABT_OFF            16
  84 #define ENT_INT_SRC_MSK1                0x1c4
  85 #define ENT_INT_SRC_MSK2                0x1c8
  86 #define ENT_INT_SRC_MSK3                0x1cc
  87 #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF  31
  88 #define CHNL_PHYUPDOWN_INT_MSK          0x1d0
  89 #define CHNL_ENT_INT_MSK                        0x1d4
  90 #define HGC_COM_INT_MSK                         0x1d8
  91 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK  (0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
  92 #define SAS_ECC_INTR                    0x1e8
  93 #define SAS_ECC_INTR_MSK                0x1ec
  94 #define HGC_ERR_STAT_EN                 0x238
  95 #define DLVRY_Q_0_BASE_ADDR_LO          0x260
  96 #define DLVRY_Q_0_BASE_ADDR_HI          0x264
  97 #define DLVRY_Q_0_DEPTH                 0x268
  98 #define DLVRY_Q_0_WR_PTR                0x26c
  99 #define DLVRY_Q_0_RD_PTR                0x270
 100 #define HYPER_STREAM_ID_EN_CFG          0xc80
 101 #define OQ0_INT_SRC_MSK                 0xc90
 102 #define COMPL_Q_0_BASE_ADDR_LO          0x4e0
 103 #define COMPL_Q_0_BASE_ADDR_HI          0x4e4
 104 #define COMPL_Q_0_DEPTH                 0x4e8
 105 #define COMPL_Q_0_WR_PTR                0x4ec
 106 #define COMPL_Q_0_RD_PTR                0x4f0
 107 #define AWQOS_AWCACHE_CFG       0xc84
 108 #define ARQOS_ARCACHE_CFG       0xc88
 109 #define HILINK_ERR_DFX          0xe04
 110
 111 /* phy registers requiring init */
 112 #define PORT_BASE                       (0x2000)
 113 #define PHY_CFG                         (PORT_BASE + 0x0)
 114 #define HARD_PHY_LINKRATE               (PORT_BASE + 0x4)
 115 #define PHY_CFG_ENA_OFF                 0
 116 #define PHY_CFG_ENA_MSK                 (0x1 << PHY_CFG_ENA_OFF)
 117 #define PHY_CFG_DC_OPT_OFF              2
 118 #define PHY_CFG_DC_OPT_MSK              (0x1 << PHY_CFG_DC_OPT_OFF)
 119 #define PROG_PHY_LINK_RATE              (PORT_BASE + 0x8)
 120 #define PHY_CTRL                        (PORT_BASE + 0x14)
 121 #define PHY_CTRL_RESET_OFF              0
 122 #define PHY_CTRL_RESET_MSK              (0x1 << PHY_CTRL_RESET_OFF)
 123 #define SL_CFG                          (PORT_BASE + 0x84)
 124 #define SL_CONTROL                      (PORT_BASE + 0x94)
 125 #define SL_CONTROL_NOTIFY_EN_OFF        0
 126 #define SL_CONTROL_NOTIFY_EN_MSK        (0x1 << SL_CONTROL_NOTIFY_EN_OFF)
 127 #define SL_CTA_OFF              17
 128 #define SL_CTA_MSK              (0x1 << SL_CTA_OFF)
 129 #define TX_ID_DWORD0                    (PORT_BASE + 0x9c)
 130 #define TX_ID_DWORD1                    (PORT_BASE + 0xa0)
 131 #define TX_ID_DWORD2                    (PORT_BASE + 0xa4)
 132 #define TX_ID_DWORD3                    (PORT_BASE + 0xa8)
 133 #define TX_ID_DWORD4                    (PORT_BASE + 0xaC)
 134 #define TX_ID_DWORD5                    (PORT_BASE + 0xb0)
 135 #define TX_ID_DWORD6                    (PORT_BASE + 0xb4)
 136 #define TXID_AUTO                               (PORT_BASE + 0xb8)
 137 #define CT3_OFF         1
 138 #define CT3_MSK         (0x1 << CT3_OFF)
 139 #define TX_HARDRST_OFF          2
 140 #define TX_HARDRST_MSK          (0x1 << TX_HARDRST_OFF)
 141 #define RX_IDAF_DWORD0                  (PORT_BASE + 0xc4)
 142 #define RXOP_CHECK_CFG_H                (PORT_BASE + 0xfc)
 143 #define STP_LINK_TIMER                  (PORT_BASE + 0x120)
 144 #define STP_LINK_TIMEOUT_STATE          (PORT_BASE + 0x124)
 145 #define CON_CFG_DRIVER                  (PORT_BASE + 0x130)
 146 #define SAS_SSP_CON_TIMER_CFG           (PORT_BASE + 0x134)
 147 #define SAS_SMP_CON_TIMER_CFG           (PORT_BASE + 0x138)
 148 #define SAS_STP_CON_TIMER_CFG           (PORT_BASE + 0x13c)
 149 #define CHL_INT0                        (PORT_BASE + 0x1b4)
 150 #define CHL_INT0_HOTPLUG_TOUT_OFF       0
 151 #define CHL_INT0_HOTPLUG_TOUT_MSK       (0x1 << CHL_INT0_HOTPLUG_TOUT_OFF)
 152 #define CHL_INT0_SL_RX_BCST_ACK_OFF     1
 153 #define CHL_INT0_SL_RX_BCST_ACK_MSK     (0x1 << CHL_INT0_SL_RX_BCST_ACK_OFF)
 154 #define CHL_INT0_SL_PHY_ENABLE_OFF      2
 155 #define CHL_INT0_SL_PHY_ENABLE_MSK      (0x1 << CHL_INT0_SL_PHY_ENABLE_OFF)
 156 #define CHL_INT0_NOT_RDY_OFF            4
 157 #define CHL_INT0_NOT_RDY_MSK            (0x1 << CHL_INT0_NOT_RDY_OFF)
 158 #define CHL_INT0_PHY_RDY_OFF            5
 159 #define CHL_INT0_PHY_RDY_MSK            (0x1 << CHL_INT0_PHY_RDY_OFF)
 160 #define CHL_INT1                        (PORT_BASE + 0x1b8)
 161 #define CHL_INT1_DMAC_TX_ECC_ERR_OFF    15
 162 #define CHL_INT1_DMAC_TX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_TX_ECC_ERR_OFF)
 163 #define CHL_INT1_DMAC_RX_ECC_ERR_OFF    17
 164 #define CHL_INT1_DMAC_RX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_RX_ECC_ERR_OFF)
 165 #define CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF 19
 166 #define CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF 20
 167 #define CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF 21
 168 #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF 22
 169 #define CHL_INT2                        (PORT_BASE + 0x1bc)
 170 #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF  0
 171 #define CHL_INT2_RX_INVLD_DW_OFF        30
 172 #define CHL_INT2_STP_LINK_TIMEOUT_OFF   31
 173 #define CHL_INT0_MSK                    (PORT_BASE + 0x1c0)
 174 #define CHL_INT1_MSK                    (PORT_BASE + 0x1c4)
 175 #define CHL_INT2_MSK                    (PORT_BASE + 0x1c8)
 176 #define CHL_INT_COAL_EN                 (PORT_BASE + 0x1d0)
 177 #define SAS_RX_TRAIN_TIMER              (PORT_BASE + 0x2a4)
 178 #define PHY_CTRL_RDY_MSK                (PORT_BASE + 0x2b0)
 179 #define PHYCTRL_NOT_RDY_MSK             (PORT_BASE + 0x2b4)
 180 #define PHYCTRL_DWS_RESET_MSK           (PORT_BASE + 0x2b8)
 181 #define PHYCTRL_PHY_ENA_MSK             (PORT_BASE + 0x2bc)
 182 #define SL_RX_BCAST_CHK_MSK             (PORT_BASE + 0x2c0)
 183 #define PHYCTRL_OOB_RESTART_MSK         (PORT_BASE + 0x2c4)
 184 #define DMA_TX_STATUS                   (PORT_BASE + 0x2d0)
 185 #define DMA_TX_STATUS_BUSY_OFF          0
 186 #define DMA_TX_STATUS_BUSY_MSK          (0x1 << DMA_TX_STATUS_BUSY_OFF)
 187 #define DMA_RX_STATUS                   (PORT_BASE + 0x2e8)
 188 #define DMA_RX_STATUS_BUSY_OFF          0
 189 #define DMA_RX_STATUS_BUSY_MSK          (0x1 << DMA_RX_STATUS_BUSY_OFF)
 190
 191 #define COARSETUNE_TIME                 (PORT_BASE + 0x304)
 192 #define ERR_CNT_DWS_LOST                (PORT_BASE + 0x380)
 193 #define ERR_CNT_RESET_PROB              (PORT_BASE + 0x384)
 194 #define ERR_CNT_INVLD_DW                (PORT_BASE + 0x390)
 195 #define ERR_CNT_DISP_ERR                (PORT_BASE + 0x398)
 196
 197 #define DEFAULT_ITCT_HW         2048 /* reset value, not reprogrammed */
 198 #if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW)
 199 #error Max ITCT exceeded
 200 #endif
 201
 202 #define AXI_MASTER_CFG_BASE             (0x5000)
 203 #define AM_CTRL_GLOBAL                  (0x0)
 204 #define AM_CURR_TRANS_RETURN    (0x150)
 205
 206 #define AM_CFG_MAX_TRANS                (0x5010)
 207 #define AM_CFG_SINGLE_PORT_MAX_TRANS    (0x5014)
 208 #define AXI_CFG                                 (0x5100)
 209 #define AM_ROB_ECC_ERR_ADDR             (0x510c)
 210 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF  0
 211 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF)
 212 #define AM_ROB_ECC_MULBIT_ERR_ADDR_OFF  8
 213 #define AM_ROB_ECC_MULBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_MULBIT_ERR_ADDR_OFF)
 214
 215 /* RAS registers need init */
 216 #define RAS_BASE                (0x6000)
 217 #define SAS_RAS_INTR0                   (RAS_BASE)
 218 #define SAS_RAS_INTR1                   (RAS_BASE + 0x04)
 219 #define SAS_RAS_INTR0_MASK              (RAS_BASE + 0x08)
 220 #define SAS_RAS_INTR1_MASK              (RAS_BASE + 0x0c)
 221 #define CFG_SAS_RAS_INTR_MASK           (RAS_BASE + 0x1c)
 222 #define SAS_RAS_INTR2                   (RAS_BASE + 0x20)
 223 #define SAS_RAS_INTR2_MASK              (RAS_BASE + 0x24)
 224
 225 /* HW dma structures */
 226 /* Delivery queue header */
 227 /* dw0 */
 228 #define CMD_HDR_ABORT_FLAG_OFF          0
 229 #define CMD_HDR_ABORT_FLAG_MSK          (0x3 << CMD_HDR_ABORT_FLAG_OFF)
 230 #define CMD_HDR_ABORT_DEVICE_TYPE_OFF   2
 231 #define CMD_HDR_ABORT_DEVICE_TYPE_MSK   (0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF)
 232 #define CMD_HDR_RESP_REPORT_OFF         5
 233 #define CMD_HDR_RESP_REPORT_MSK         (0x1 << CMD_HDR_RESP_REPORT_OFF)
 234 #define CMD_HDR_TLR_CTRL_OFF            6
 235 #define CMD_HDR_TLR_CTRL_MSK            (0x3 << CMD_HDR_TLR_CTRL_OFF)
 236 #define CMD_HDR_PORT_OFF                18
 237 #define CMD_HDR_PORT_MSK                (0xf << CMD_HDR_PORT_OFF)
 238 #define CMD_HDR_PRIORITY_OFF            27
 239 #define CMD_HDR_PRIORITY_MSK            (0x1 << CMD_HDR_PRIORITY_OFF)
 240 #define CMD_HDR_CMD_OFF                 29
 241 #define CMD_HDR_CMD_MSK                 (0x7 << CMD_HDR_CMD_OFF)
 242 /* dw1 */
 243 #define CMD_HDR_UNCON_CMD_OFF   3
 244 #define CMD_HDR_DIR_OFF                 5
 245 #define CMD_HDR_DIR_MSK                 (0x3 << CMD_HDR_DIR_OFF)
 246 #define CMD_HDR_RESET_OFF               7
 247 #define CMD_HDR_RESET_MSK               (0x1 << CMD_HDR_RESET_OFF)
 248 #define CMD_HDR_VDTL_OFF                10
 249 #define CMD_HDR_VDTL_MSK                (0x1 << CMD_HDR_VDTL_OFF)
 250 #define CMD_HDR_FRAME_TYPE_OFF          11
 251 #define CMD_HDR_FRAME_TYPE_MSK          (0x1f << CMD_HDR_FRAME_TYPE_OFF)
 252 #define CMD_HDR_DEV_ID_OFF              16
 253 #define CMD_HDR_DEV_ID_MSK              (0xffff << CMD_HDR_DEV_ID_OFF)
 254 /* dw2 */
 255 #define CMD_HDR_CFL_OFF                 0
 256 #define CMD_HDR_CFL_MSK                 (0x1ff << CMD_HDR_CFL_OFF)
 257 #define CMD_HDR_NCQ_TAG_OFF             10
 258 #define CMD_HDR_NCQ_TAG_MSK             (0x1f << CMD_HDR_NCQ_TAG_OFF)
 259 #define CMD_HDR_MRFL_OFF                15
 260 #define CMD_HDR_MRFL_MSK                (0x1ff << CMD_HDR_MRFL_OFF)
 261 #define CMD_HDR_SG_MOD_OFF              24
 262 #define CMD_HDR_SG_MOD_MSK              (0x3 << CMD_HDR_SG_MOD_OFF)
 263 /* dw3 */
 264 #define CMD_HDR_IPTT_OFF                0
 265 #define CMD_HDR_IPTT_MSK                (0xffff << CMD_HDR_IPTT_OFF)
 266 /* dw6 */
 267 #define CMD_HDR_DIF_SGL_LEN_OFF         0
 268 #define CMD_HDR_DIF_SGL_LEN_MSK         (0xffff << CMD_HDR_DIF_SGL_LEN_OFF)
 269 #define CMD_HDR_DATA_SGL_LEN_OFF        16
 270 #define CMD_HDR_DATA_SGL_LEN_MSK        (0xffff << CMD_HDR_DATA_SGL_LEN_OFF)
 271 /* dw7 */
 272 #define CMD_HDR_ADDR_MODE_SEL_OFF               15
 273 #define CMD_HDR_ADDR_MODE_SEL_MSK               (1 << CMD_HDR_ADDR_MODE_SEL_OFF)
 274 #define CMD_HDR_ABORT_IPTT_OFF          16
 275 #define CMD_HDR_ABORT_IPTT_MSK          (0xffff << CMD_HDR_ABORT_IPTT_OFF)
 276
 277 /* Completion header */
 278 /* dw0 */
 279 #define CMPLT_HDR_CMPLT_OFF             0
 280 #define CMPLT_HDR_CMPLT_MSK             (0x3 << CMPLT_HDR_CMPLT_OFF)
 281 #define CMPLT_HDR_ERROR_PHASE_OFF   2
 282 #define CMPLT_HDR_ERROR_PHASE_MSK   (0xff << CMPLT_HDR_ERROR_PHASE_OFF)
 283 #define CMPLT_HDR_RSPNS_XFRD_OFF        10
 284 #define CMPLT_HDR_RSPNS_XFRD_MSK        (0x1 << CMPLT_HDR_RSPNS_XFRD_OFF)
 285 #define CMPLT_HDR_ERX_OFF               12
 286 #define CMPLT_HDR_ERX_MSK               (0x1 << CMPLT_HDR_ERX_OFF)
 287 #define CMPLT_HDR_ABORT_STAT_OFF        13
 288 #define CMPLT_HDR_ABORT_STAT_MSK        (0x7 << CMPLT_HDR_ABORT_STAT_OFF)
 289 /* abort_stat */
 290 #define STAT_IO_NOT_VALID               0x1
 291 #define STAT_IO_NO_DEVICE               0x2
 292 #define STAT_IO_COMPLETE                0x3
 293 #define STAT_IO_ABORTED                 0x4
 294 /* dw1 */
 295 #define CMPLT_HDR_IPTT_OFF              0
 296 #define CMPLT_HDR_IPTT_MSK              (0xffff << CMPLT_HDR_IPTT_OFF)
 297 #define CMPLT_HDR_DEV_ID_OFF            16
 298 #define CMPLT_HDR_DEV_ID_MSK            (0xffff << CMPLT_HDR_DEV_ID_OFF)
 299 /* dw3 */
 300 #define CMPLT_HDR_IO_IN_TARGET_OFF      17
 301 #define CMPLT_HDR_IO_IN_TARGET_MSK      (0x1 << CMPLT_HDR_IO_IN_TARGET_OFF)
 302
 303 /* ITCT header */
 304 /* qw0 */
 305 #define ITCT_HDR_DEV_TYPE_OFF           0
 306 #define ITCT_HDR_DEV_TYPE_MSK           (0x3 << ITCT_HDR_DEV_TYPE_OFF)
 307 #define ITCT_HDR_VALID_OFF              2
 308 #define ITCT_HDR_VALID_MSK              (0x1 << ITCT_HDR_VALID_OFF)
 309 #define ITCT_HDR_MCR_OFF                5
 310 #define ITCT_HDR_MCR_MSK                (0xf << ITCT_HDR_MCR_OFF)
 311 #define ITCT_HDR_VLN_OFF                9
 312 #define ITCT_HDR_VLN_MSK                (0xf << ITCT_HDR_VLN_OFF)
 313 #define ITCT_HDR_SMP_TIMEOUT_OFF        16
 314 #define ITCT_HDR_AWT_CONTINUE_OFF       25
 315 #define ITCT_HDR_PORT_ID_OFF            28
 316 #define ITCT_HDR_PORT_ID_MSK            (0xf << ITCT_HDR_PORT_ID_OFF)
 317 /* qw2 */
 318 #define ITCT_HDR_INLT_OFF               0
 319 #define ITCT_HDR_INLT_MSK               (0xffffULL << ITCT_HDR_INLT_OFF)
 320 #define ITCT_HDR_RTOLT_OFF              48
 321 #define ITCT_HDR_RTOLT_MSK              (0xffffULL << ITCT_HDR_RTOLT_OFF)
 322
 323 struct hisi_sas_complete_v3_hdr {
 324         __le32 dw0;
 325         __le32 dw1;
 326         __le32 act;
 327         __le32 dw3;
 328 };
 329
 330 struct hisi_sas_err_record_v3 {
 331         /* dw0 */
 332         __le32 trans_tx_fail_type;
 333
 334         /* dw1 */
 335         __le32 trans_rx_fail_type;
 336
 337         /* dw2 */
 338         __le16 dma_tx_err_type;
 339         __le16 sipc_rx_err_type;
 340
 341         /* dw3 */
 342         __le32 dma_rx_err_type;
 343 };
 344
 345 #define RX_DATA_LEN_UNDERFLOW_OFF       6
 346 #define RX_DATA_LEN_UNDERFLOW_MSK       (1 << RX_DATA_LEN_UNDERFLOW_OFF)
 347
 348 #define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096
 349 #define HISI_SAS_MSI_COUNT_V3_HW 32
 350
 351 #define DIR_NO_DATA 0
 352 #define DIR_TO_INI 1
 353 #define DIR_TO_DEVICE 2
 354 #define DIR_RESERVED 3
 355
 356 #define FIS_CMD_IS_UNCONSTRAINED(fis) \
 357         ((fis.command == ATA_CMD_READ_LOG_EXT) || \
 358         (fis.command == ATA_CMD_READ_LOG_DMA_EXT) || \
 359         ((fis.command == ATA_CMD_DEV_RESET) && \
 360         ((fis.control & ATA_SRST) != 0)))
 361
 362 static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
 363 {
 364         void __iomem *regs = hisi_hba->regs + off;
 365
 366         return readl(regs);
 367 }
 368
 369 static u32 hisi_sas_read32_relaxed(struct hisi_hba *hisi_hba, u32 off)
 370 {
 371         void __iomem *regs = hisi_hba->regs + off;
 372
 373         return readl_relaxed(regs);
 374 }
 375
 376 static void hisi_sas_write32(struct hisi_hba *hisi_hba, u32 off, u32 val)
 377 {
 378         void __iomem *regs = hisi_hba->regs + off;
 379
 380         writel(val, regs);
 381 }
 382
 383 static void hisi_sas_phy_write32(struct hisi_hba *hisi_hba, int phy_no,
 384                                  u32 off, u32 val)
 385 {
 386         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 387
 388         writel(val, regs);
 389 }
 390
 391 static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
 392                                       int phy_no, u32 off)
 393 {
 394         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 395
 396         return readl(regs);
 397 }
 398
 399 #define hisi_sas_read32_poll_timeout(off, val, cond, delay_us,          \
 400                                      timeout_us)                        \
 401 ({                                                                      \
 402         void __iomem *regs = hisi_hba->regs + off;                      \
 403         readl_poll_timeout(regs, val, cond, delay_us, timeout_us);      \
 404 })
 405
 406 #define hisi_sas_read32_poll_timeout_atomic(off, val, cond, delay_us,   \
 407                                             timeout_us)                 \
 408 ({                                                                      \
 409         void __iomem *regs = hisi_hba->regs + off;                      \
 410         readl_poll_timeout_atomic(regs, val, cond, delay_us, timeout_us);\
 411 })
 412
 413 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 414 {
 415         struct pci_dev *pdev = hisi_hba->pci_dev;
 416         int i;
 417
 418         /* Global registers init */
 419         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
 420                          (u32)((1ULL << hisi_hba->queue_count) - 1));
 421         hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff0400);
 422         hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108);
 423         hisi_sas_write32(hisi_hba, CFG_1US_TIMER_TRSH, 0xd);
 424         hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
 425         hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1);
 426         hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1);
 427         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 0xffff);
 428         hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff);
 429         hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff);
 430         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
 431         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
 432         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
 433         if (pdev->revision >= 0x21)
 434                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffff7fff);
 435         else
 436                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
 437         hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 438         hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 439         hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
 440         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x0);
 441         hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
 442         hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
 443         for (i = 0; i < hisi_hba->queue_count; i++)
 444                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0);
 445
 446         hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
 447
 448         for (i = 0; i < hisi_hba->n_phy; i++) {
 449                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 450                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 451                 u32 prog_phy_link_rate = 0x800;
 452
 453                 if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
 454                                 SAS_LINK_RATE_1_5_GBPS)) {
 455                         prog_phy_link_rate = 0x855;
 456                 } else {
 457                         enum sas_linkrate max = sas_phy->phy->maximum_linkrate;
 458
 459                         prog_phy_link_rate =
 460                                 hisi_sas_get_prog_phy_linkrate_mask(max) |
 461                                 0x800;
 462                 }
 463                 hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
 464                         prog_phy_link_rate);
 465                 hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80);
 466                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
 467                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
 468                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
 469                 hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
 470                 if (pdev->revision >= 0x21)
 471                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 472                                         0xffffffff);
 473                 else
 474                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 475                                         0xff87ffff);
 476                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
 477                 hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
 478                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
 479                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0);
 480                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0);
 481                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
 482                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x1);
 483                 hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120);
 484
 485                 /* used for 12G negotiate */
 486                 hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e);
 487         }
 488
 489         for (i = 0; i < hisi_hba->queue_count; i++) {
 490                 /* Delivery queue */
 491                 hisi_sas_write32(hisi_hba,
 492                                  DLVRY_Q_0_BASE_ADDR_HI + (i * 0x14),
 493                                  upper_32_bits(hisi_hba->cmd_hdr_dma[i]));
 494
 495                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_BASE_ADDR_LO + (i * 0x14),
 496                                  lower_32_bits(hisi_hba->cmd_hdr_dma[i]));
 497
 498                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_DEPTH + (i * 0x14),
 499                                  HISI_SAS_QUEUE_SLOTS);
 500
 501                 /* Completion queue */
 502                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_HI + (i * 0x14),
 503                                  upper_32_bits(hisi_hba->complete_hdr_dma[i]));
 504
 505                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_LO + (i * 0x14),
 506                                  lower_32_bits(hisi_hba->complete_hdr_dma[i]));
 507
 508                 hisi_sas_write32(hisi_hba, COMPL_Q_0_DEPTH + (i * 0x14),
 509                                  HISI_SAS_QUEUE_SLOTS);
 510         }
 511
 512         /* itct */
 513         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_LO,
 514                          lower_32_bits(hisi_hba->itct_dma));
 515
 516         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_HI,
 517                          upper_32_bits(hisi_hba->itct_dma));
 518
 519         /* iost */
 520         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_LO,
 521                          lower_32_bits(hisi_hba->iost_dma));
 522
 523         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_HI,
 524                          upper_32_bits(hisi_hba->iost_dma));
 525
 526         /* breakpoint */
 527         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_LO,
 528                          lower_32_bits(hisi_hba->breakpoint_dma));
 529
 530         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_HI,
 531                          upper_32_bits(hisi_hba->breakpoint_dma));
 532
 533         /* SATA broken msg */
 534         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_LO,
 535                          lower_32_bits(hisi_hba->sata_breakpoint_dma));
 536
 537         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_HI,
 538                          upper_32_bits(hisi_hba->sata_breakpoint_dma));
 539
 540         /* SATA initial fis */
 541         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_LO,
 542                          lower_32_bits(hisi_hba->initial_fis_dma));
 543
 544         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_HI,
 545                          upper_32_bits(hisi_hba->initial_fis_dma));
 546
 547         /* RAS registers init */
 548         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, 0x0);
 549         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, 0x0);
 550         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, 0x0);
 551         hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, 0x0);
 552 }
 553
 554 static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 555 {
 556         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 557
 558         cfg &= ~PHY_CFG_DC_OPT_MSK;
 559         cfg |= 1 << PHY_CFG_DC_OPT_OFF;
 560         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 561 }
 562
 563 static void config_id_frame_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 564 {
 565         struct sas_identify_frame identify_frame;
 566         u32 *identify_buffer;
 567
 568         memset(&identify_frame, 0, sizeof(identify_frame));
 569         identify_frame.dev_type = SAS_END_DEVICE;
 570         identify_frame.frame_type = 0;
 571         identify_frame._un1 = 1;
 572         identify_frame.initiator_bits = SAS_PROTOCOL_ALL;
 573         identify_frame.target_bits = SAS_PROTOCOL_NONE;
 574         memcpy(&identify_frame._un4_11[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 575         memcpy(&identify_frame.sas_addr[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 576         identify_frame.phy_id = phy_no;
 577         identify_buffer = (u32 *)(&identify_frame);
 578
 579         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
 580                         __swab32(identify_buffer[0]));
 581         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
 582                         __swab32(identify_buffer[1]));
 583         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
 584                         __swab32(identify_buffer[2]));
 585         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
 586                         __swab32(identify_buffer[3]));
 587         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
 588                         __swab32(identify_buffer[4]));
 589         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
 590                         __swab32(identify_buffer[5]));
 591 }
 592
 593 static void setup_itct_v3_hw(struct hisi_hba *hisi_hba,
 594                              struct hisi_sas_device *sas_dev)
 595 {
 596         struct domain_device *device = sas_dev->sas_device;
 597         struct device *dev = hisi_hba->dev;
 598         u64 qw0, device_id = sas_dev->device_id;
 599         struct hisi_sas_itct *itct = &hisi_hba->itct[device_id];
 600         struct domain_device *parent_dev = device->parent;
 601         struct asd_sas_port *sas_port = device->port;
 602         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
 603
 604         memset(itct, 0, sizeof(*itct));
 605
 606         /* qw0 */
 607         qw0 = 0;
 608         switch (sas_dev->dev_type) {
 609         case SAS_END_DEVICE:
 610         case SAS_EDGE_EXPANDER_DEVICE:
 611         case SAS_FANOUT_EXPANDER_DEVICE:
 612                 qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF;
 613                 break;
 614         case SAS_SATA_DEV:
 615         case SAS_SATA_PENDING:
 616                 if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
 617                         qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF;
 618                 else
 619                         qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF;
 620                 break;
 621         default:
 622                 dev_warn(dev, "setup itct: unsupported dev type (%d)\n",
 623                          sas_dev->dev_type);
 624         }
 625
 626         qw0 |= ((1 << ITCT_HDR_VALID_OFF) |
 627                 (device->linkrate << ITCT_HDR_MCR_OFF) |
 628                 (1 << ITCT_HDR_VLN_OFF) |
 629                 (0xfa << ITCT_HDR_SMP_TIMEOUT_OFF) |
 630                 (1 << ITCT_HDR_AWT_CONTINUE_OFF) |
 631                 (port->id << ITCT_HDR_PORT_ID_OFF));
 632         itct->qw0 = cpu_to_le64(qw0);
 633
 634         /* qw1 */
 635         memcpy(&itct->sas_addr, device->sas_addr, SAS_ADDR_SIZE);
 636         itct->sas_addr = __swab64(itct->sas_addr);
 637
 638         /* qw2 */
 639         if (!dev_is_sata(device))
 640                 itct->qw2 = cpu_to_le64((5000ULL << ITCT_HDR_INLT_OFF) |
 641                                         (0x1ULL << ITCT_HDR_RTOLT_OFF));
 642 }
 643
 644 static void clear_itct_v3_hw(struct hisi_hba *hisi_hba,
 645                               struct hisi_sas_device *sas_dev)
 646 {
 647         DECLARE_COMPLETION_ONSTACK(completion);
 648         u64 dev_id = sas_dev->device_id;
 649         struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
 650         u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
 651
 652         sas_dev->completion = &completion;
 653
 654         /* clear the itct interrupt state */
 655         if (ENT_INT_SRC3_ITC_INT_MSK & reg_val)
 656                 hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
 657                                  ENT_INT_SRC3_ITC_INT_MSK);
 658
 659         /* clear the itct table*/
 660         reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
 661         hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val);
 662
 663         wait_for_completion(sas_dev->completion);
 664         memset(itct, 0, sizeof(struct hisi_sas_itct));
 665 }
 666
 667 static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
 668                                 struct domain_device *device)
 669 {
 670         struct hisi_sas_slot *slot, *slot2;
 671         struct hisi_sas_device *sas_dev = device->lldd_dev;
 672         u32 cfg_abt_set_query_iptt;
 673
 674         cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba,
 675                 CFG_ABT_SET_QUERY_IPTT);
 676         list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) {
 677                 cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK;
 678                 cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) |
 679                         (slot->idx << CFG_SET_ABORTED_IPTT_OFF);
 680                 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 681                         cfg_abt_set_query_iptt);
 682         }
 683         cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF);
 684         hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 685                 cfg_abt_set_query_iptt);
 686         hisi_sas_write32(hisi_hba, CFG_ABT_SET_IPTT_DONE,
 687                                         1 << CFG_ABT_SET_IPTT_DONE_OFF);
 688 }
 689
 690 static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
 691 {
 692         struct device *dev = hisi_hba->dev;
 693         int ret;
 694         u32 val;
 695
 696         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
 697
 698         /* Disable all of the PHYs */
 699         hisi_sas_stop_phys(hisi_hba);
 700         udelay(50);
 701
 702         /* Ensure axi bus idle */
 703         ret = hisi_sas_read32_poll_timeout(AXI_CFG, val, !val,
 704                                            20000, 1000000);
 705         if (ret) {
 706                 dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
 707                 return -EIO;
 708         }
 709
 710         if (ACPI_HANDLE(dev)) {
 711                 acpi_status s;
 712
 713                 s = acpi_evaluate_object(ACPI_HANDLE(dev), "_RST", NULL, NULL);
 714                 if (ACPI_FAILURE(s)) {
 715                         dev_err(dev, "Reset failed\n");
 716                         return -EIO;
 717                 }
 718         } else {
 719                 dev_err(dev, "no reset method!\n");
 720                 return -EINVAL;
 721         }
 722
 723         return 0;
 724 }
 725
 726 static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 727 {
 728         struct device *dev = hisi_hba->dev;
 729         int rc;
 730
 731         rc = reset_hw_v3_hw(hisi_hba);
 732         if (rc) {
 733                 dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc);
 734                 return rc;
 735         }
 736
 737         msleep(100);
 738         init_reg_v3_hw(hisi_hba);
 739
 740         return 0;
 741 }
 742
 743 static void enable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 744 {
 745         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 746
 747         cfg |= PHY_CFG_ENA_MSK;
 748         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 749 }
 750
 751 static void disable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 752 {
 753         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 754
 755         cfg &= ~PHY_CFG_ENA_MSK;
 756         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 757 }
 758
 759 static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 760 {
 761         config_id_frame_v3_hw(hisi_hba, phy_no);
 762         config_phy_opt_mode_v3_hw(hisi_hba, phy_no);
 763         enable_phy_v3_hw(hisi_hba, phy_no);
 764 }
 765
 766 static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 767 {
 768         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 769         u32 txid_auto;
 770
 771         disable_phy_v3_hw(hisi_hba, phy_no);
 772         if (phy->identify.device_type == SAS_END_DEVICE) {
 773                 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 774                 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
 775                                         txid_auto | TX_HARDRST_MSK);
 776         }
 777         msleep(100);
 778         start_phy_v3_hw(hisi_hba, phy_no);
 779 }
 780
 781 static enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
 782 {
 783         return SAS_LINK_RATE_12_0_GBPS;
 784 }
 785
 786 static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
 787 {
 788         int i;
 789
 790         for (i = 0; i < hisi_hba->n_phy; i++) {
 791                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 792                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 793
 794                 if (!sas_phy->phy->enabled)
 795                         continue;
 796
 797                 start_phy_v3_hw(hisi_hba, i);
 798         }
 799 }
 800
 801 static void sl_notify_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 802 {
 803         u32 sl_control;
 804
 805         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 806         sl_control |= SL_CONTROL_NOTIFY_EN_MSK;
 807         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 808         msleep(1);
 809         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 810         sl_control &= ~SL_CONTROL_NOTIFY_EN_MSK;
 811         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 812 }
 813
 814 static int get_wideport_bitmap_v3_hw(struct hisi_hba *hisi_hba, int port_id)
 815 {
 816         int i, bitmap = 0;
 817         u32 phy_port_num_ma = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
 818         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
 819
 820         for (i = 0; i < hisi_hba->n_phy; i++)
 821                 if (phy_state & BIT(i))
 822                         if (((phy_port_num_ma >> (i * 4)) & 0xf) == port_id)
 823                                 bitmap |= BIT(i);
 824
 825         return bitmap;
 826 }
 827
 828 /**
 829  * The callpath to this function and upto writing the write
 830  * queue pointer should be safe from interruption.
 831  */
 832 static int
 833 get_free_slot_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
 834 {
 835         struct device *dev = hisi_hba->dev;
 836         int queue = dq->id;
 837         u32 r, w;
 838
 839         w = dq->wr_point;
 840         r = hisi_sas_read32_relaxed(hisi_hba,
 841                                 DLVRY_Q_0_RD_PTR + (queue * 0x14));
 842         if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
 843                 dev_warn(dev, "full queue=%d r=%d w=%d\n",
 844                                 queue, r, w);
 845                 return -EAGAIN;
 846         }
 847
 848         dq->wr_point = (dq->wr_point + 1) % HISI_SAS_QUEUE_SLOTS;
 849
 850         return w;
 851 }
 852
 853 static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
 854 {
 855         struct hisi_hba *hisi_hba = dq->hisi_hba;
 856         struct hisi_sas_slot *s, *s1;
 857         struct list_head *dq_list;
 858         int dlvry_queue = dq->id;
 859         int wp, count = 0;
 860
 861         dq_list = &dq->list;
 862         list_for_each_entry_safe(s, s1, &dq->list, delivery) {
 863                 if (!s->ready)
 864                         break;
 865                 count++;
 866                 wp = (s->dlvry_queue_slot + 1) % HISI_SAS_QUEUE_SLOTS;
 867                 list_del(&s->delivery);
 868         }
 869
 870         if (!count)
 871                 return;
 872
 873         hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14), wp);
 874 }
 875
 876 static void prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
 877                               struct hisi_sas_slot *slot,
 878                               struct hisi_sas_cmd_hdr *hdr,
 879                               struct scatterlist *scatter,
 880                               int n_elem)
 881 {
 882         struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
 883         struct scatterlist *sg;
 884         int i;
 885
 886         for_each_sg(scatter, sg, n_elem, i) {
 887                 struct hisi_sas_sge *entry = &sge_page->sge[i];
 888
 889                 entry->addr = cpu_to_le64(sg_dma_address(sg));
 890                 entry->page_ctrl_0 = entry->page_ctrl_1 = 0;
 891                 entry->data_len = cpu_to_le32(sg_dma_len(sg));
 892                 entry->data_off = 0;
 893         }
 894
 895         hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 896
 897         hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
 898 }
 899
 900 static void prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 901                           struct hisi_sas_slot *slot, int is_tmf,
 902                           struct hisi_sas_tmf_task *tmf)
 903 {
 904         struct sas_task *task = slot->task;
 905         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 906         struct domain_device *device = task->dev;
 907         struct hisi_sas_device *sas_dev = device->lldd_dev;
 908         struct hisi_sas_port *port = slot->port;
 909         struct sas_ssp_task *ssp_task = &task->ssp_task;
 910         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
 911         int has_data = 0, priority = is_tmf;
 912         u8 *buf_cmd;
 913         u32 dw1 = 0, dw2 = 0;
 914
 915         hdr->dw0 = cpu_to_le32((1 << CMD_HDR_RESP_REPORT_OFF) |
 916                                (2 << CMD_HDR_TLR_CTRL_OFF) |
 917                                (port->id << CMD_HDR_PORT_OFF) |
 918                                (priority << CMD_HDR_PRIORITY_OFF) |
 919                                (1 << CMD_HDR_CMD_OFF)); /* ssp */
 920
 921         dw1 = 1 << CMD_HDR_VDTL_OFF;
 922         if (is_tmf) {
 923                 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
 924                 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
 925         } else {
 926                 dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF;
 927                 switch (scsi_cmnd->sc_data_direction) {
 928                 case DMA_TO_DEVICE:
 929                         has_data = 1;
 930                         dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
 931                         break;
 932                 case DMA_FROM_DEVICE:
 933                         has_data = 1;
 934                         dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
 935                         break;
 936                 default:
 937                         dw1 &= ~CMD_HDR_DIR_MSK;
 938                 }
 939         }
 940
 941         /* map itct entry */
 942         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
 943         hdr->dw1 = cpu_to_le32(dw1);
 944
 945         dw2 = (((sizeof(struct ssp_command_iu) + sizeof(struct ssp_frame_hdr)
 946               + 3) / 4) << CMD_HDR_CFL_OFF) |
 947               ((HISI_SAS_MAX_SSP_RESP_SZ / 4) << CMD_HDR_MRFL_OFF) |
 948               (2 << CMD_HDR_SG_MOD_OFF);
 949         hdr->dw2 = cpu_to_le32(dw2);
 950         hdr->transfer_tags = cpu_to_le32(slot->idx);
 951
 952         if (has_data)
 953                 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
 954                                         slot->n_elem);
 955
 956         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 957         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
 958         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
 959
 960         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot) +
 961                 sizeof(struct ssp_frame_hdr);
 962
 963         memcpy(buf_cmd, &task->ssp_task.LUN, 8);
 964         if (!is_tmf) {
 965                 buf_cmd[9] = ssp_task->task_attr | (ssp_task->task_prio << 3);
 966                 memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
 967         } else {
 968                 buf_cmd[10] = tmf->tmf;
 969                 switch (tmf->tmf) {
 970                 case TMF_ABORT_TASK:
 971                 case TMF_QUERY_TASK:
 972                         buf_cmd[12] =
 973                                 (tmf->tag_of_task_to_be_managed >> 8) & 0xff;
 974                         buf_cmd[13] =
 975                                 tmf->tag_of_task_to_be_managed & 0xff;
 976                         break;
 977                 default:
 978                         break;
 979                 }
 980         }
 981 }
 982
 983 static void prep_smp_v3_hw(struct hisi_hba *hisi_hba,
 984                           struct hisi_sas_slot *slot)
 985 {
 986         struct sas_task *task = slot->task;
 987         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 988         struct domain_device *device = task->dev;
 989         struct hisi_sas_port *port = slot->port;
 990         struct scatterlist *sg_req;
 991         struct hisi_sas_device *sas_dev = device->lldd_dev;
 992         dma_addr_t req_dma_addr;
 993         unsigned int req_len;
 994
 995         /* req */
 996         sg_req = &task->smp_task.smp_req;
 997         req_len = sg_dma_len(sg_req);
 998         req_dma_addr = sg_dma_address(sg_req);
 999
1000         /* create header */
1001         /* dw0 */
1002         hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
1003                                (1 << CMD_HDR_PRIORITY_OFF) | /* high pri */
1004                                (2 << CMD_HDR_CMD_OFF)); /* smp */
1005
1006         /* map itct entry */
1007         hdr->dw1 = cpu_to_le32((sas_dev->device_id << CMD_HDR_DEV_ID_OFF) |
1008                                (1 << CMD_HDR_FRAME_TYPE_OFF) |
1009                                (DIR_NO_DATA << CMD_HDR_DIR_OFF));
1010
1011         /* dw2 */
1012         hdr->dw2 = cpu_to_le32((((req_len - 4) / 4) << CMD_HDR_CFL_OFF) |
1013                                (HISI_SAS_MAX_SMP_RESP_SZ / 4 <<
1014                                CMD_HDR_MRFL_OFF));
1015
1016         hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
1017
1018         hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
1019         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1020
1021 }
1022
1023 static void prep_ata_v3_hw(struct hisi_hba *hisi_hba,
1024                           struct hisi_sas_slot *slot)
1025 {
1026         struct sas_task *task = slot->task;
1027         struct domain_device *device = task->dev;
1028         struct domain_device *parent_dev = device->parent;
1029         struct hisi_sas_device *sas_dev = device->lldd_dev;
1030         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1031         struct asd_sas_port *sas_port = device->port;
1032         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
1033         u8 *buf_cmd;
1034         int has_data = 0, hdr_tag = 0;
1035         u32 dw1 = 0, dw2 = 0;
1036
1037         hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
1038         if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
1039                 hdr->dw0 |= cpu_to_le32(3 << CMD_HDR_CMD_OFF);
1040         else
1041                 hdr->dw0 |= cpu_to_le32(4 << CMD_HDR_CMD_OFF);
1042
1043         switch (task->data_dir) {
1044         case DMA_TO_DEVICE:
1045                 has_data = 1;
1046                 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1047                 break;
1048         case DMA_FROM_DEVICE:
1049                 has_data = 1;
1050                 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1051                 break;
1052         default:
1053                 dw1 &= ~CMD_HDR_DIR_MSK;
1054         }
1055
1056         if ((task->ata_task.fis.command == ATA_CMD_DEV_RESET) &&
1057                         (task->ata_task.fis.control & ATA_SRST))
1058                 dw1 |= 1 << CMD_HDR_RESET_OFF;
1059
1060         dw1 |= (hisi_sas_get_ata_protocol(
1061                 &task->ata_task.fis, task->data_dir))
1062                 << CMD_HDR_FRAME_TYPE_OFF;
1063         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1064
1065         if (FIS_CMD_IS_UNCONSTRAINED(task->ata_task.fis))
1066                 dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
1067
1068         hdr->dw1 = cpu_to_le32(dw1);
1069
1070         /* dw2 */
1071         if (task->ata_task.use_ncq && hisi_sas_get_ncq_tag(task, &hdr_tag)) {
1072                 task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
1073                 dw2 |= hdr_tag << CMD_HDR_NCQ_TAG_OFF;
1074         }
1075
1076         dw2 |= (HISI_SAS_MAX_STP_RESP_SZ / 4) << CMD_HDR_CFL_OFF |
1077                         2 << CMD_HDR_SG_MOD_OFF;
1078         hdr->dw2 = cpu_to_le32(dw2);
1079
1080         /* dw3 */
1081         hdr->transfer_tags = cpu_to_le32(slot->idx);
1082
1083         if (has_data)
1084                 prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
1085                                         slot->n_elem);
1086
1087         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
1088         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1089         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1090
1091         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot);
1092
1093         if (likely(!task->ata_task.device_control_reg_update))
1094                 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
1095         /* fill in command FIS */
1096         memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
1097 }
1098
1099 static void prep_abort_v3_hw(struct hisi_hba *hisi_hba,
1100                 struct hisi_sas_slot *slot,
1101                 int device_id, int abort_flag, int tag_to_abort)
1102 {
1103         struct sas_task *task = slot->task;
1104         struct domain_device *dev = task->dev;
1105         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1106         struct hisi_sas_port *port = slot->port;
1107
1108         /* dw0 */
1109         hdr->dw0 = cpu_to_le32((5 << CMD_HDR_CMD_OFF) | /*abort*/
1110                                (port->id << CMD_HDR_PORT_OFF) |
1111                                    (dev_is_sata(dev)
1112                                         << CMD_HDR_ABORT_DEVICE_TYPE_OFF) |
1113                                         (abort_flag
1114                                          << CMD_HDR_ABORT_FLAG_OFF));
1115
1116         /* dw1 */
1117         hdr->dw1 = cpu_to_le32(device_id
1118                         << CMD_HDR_DEV_ID_OFF);
1119
1120         /* dw7 */
1121         hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
1122         hdr->transfer_tags = cpu_to_le32(slot->idx);
1123
1124 }
1125
1126 static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1127 {
1128         int i, res;
1129         u32 context, port_id, link_rate;
1130         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1131         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1132         struct device *dev = hisi_hba->dev;
1133
1134         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
1135
1136         port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1137         port_id = (port_id >> (4 * phy_no)) & 0xf;
1138         link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE);
1139         link_rate = (link_rate >> (phy_no * 4)) & 0xf;
1140
1141         if (port_id == 0xf) {
1142                 dev_err(dev, "phyup: phy%d invalid portid\n", phy_no);
1143                 res = IRQ_NONE;
1144                 goto end;
1145         }
1146         sas_phy->linkrate = link_rate;
1147         phy->phy_type &= ~(PORT_TYPE_SAS | PORT_TYPE_SATA);
1148
1149         /* Check for SATA dev */
1150         context = hisi_sas_read32(hisi_hba, PHY_CONTEXT);
1151         if (context & (1 << phy_no)) {
1152                 struct hisi_sas_initial_fis *initial_fis;
1153                 struct dev_to_host_fis *fis;
1154                 u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
1155
1156                 dev_info(dev, "phyup: phy%d link_rate=%d(sata)\n", phy_no, link_rate);
1157                 initial_fis = &hisi_hba->initial_fis[phy_no];
1158                 fis = &initial_fis->fis;
1159                 sas_phy->oob_mode = SATA_OOB_MODE;
1160                 attached_sas_addr[0] = 0x50;
1161                 attached_sas_addr[7] = phy_no;
1162                 memcpy(sas_phy->attached_sas_addr,
1163                        attached_sas_addr,
1164                        SAS_ADDR_SIZE);
1165                 memcpy(sas_phy->frame_rcvd, fis,
1166                        sizeof(struct dev_to_host_fis));
1167                 phy->phy_type |= PORT_TYPE_SATA;
1168                 phy->identify.device_type = SAS_SATA_DEV;
1169                 phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
1170                 phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
1171         } else {
1172                 u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
1173                 struct sas_identify_frame *id =
1174                         (struct sas_identify_frame *)frame_rcvd;
1175
1176                 dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate);
1177                 for (i = 0; i < 6; i++) {
1178                         u32 idaf = hisi_sas_phy_read32(hisi_hba, phy_no,
1179                                                RX_IDAF_DWORD0 + (i * 4));
1180                         frame_rcvd[i] = __swab32(idaf);
1181                 }
1182                 sas_phy->oob_mode = SAS_OOB_MODE;
1183                 memcpy(sas_phy->attached_sas_addr,
1184                        &id->sas_addr,
1185                        SAS_ADDR_SIZE);
1186                 phy->phy_type |= PORT_TYPE_SAS;
1187                 phy->identify.device_type = id->dev_type;
1188                 phy->frame_rcvd_size = sizeof(struct sas_identify_frame);
1189                 if (phy->identify.device_type == SAS_END_DEVICE)
1190                         phy->identify.target_port_protocols =
1191                                 SAS_PROTOCOL_SSP;
1192                 else if (phy->identify.device_type != SAS_PHY_UNUSED)
1193                         phy->identify.target_port_protocols =
1194                                 SAS_PROTOCOL_SMP;
1195         }
1196
1197         phy->port_id = port_id;
1198         phy->phy_attached = 1;
1199         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
1200         res = IRQ_HANDLED;
1201 end:
1202         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1203                              CHL_INT0_SL_PHY_ENABLE_MSK);
1204         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0);
1205
1206         return res;
1207 }
1208
1209 static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1210 {
1211         u32 phy_state, sl_ctrl, txid_auto;
1212         struct device *dev = hisi_hba->dev;
1213
1214         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1);
1215
1216         phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1217         dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state);
1218         hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0);
1219
1220         sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1221         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL,
1222                                                 sl_ctrl&(~SL_CTA_MSK));
1223
1224         txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1225         hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1226                                                 txid_auto | CT3_MSK);
1227
1228         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
1229         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0);
1230
1231         return IRQ_HANDLED;
1232 }
1233
1234 static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1235 {
1236         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1237         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1238         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
1239
1240         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1);
1241         sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD);
1242         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1243                              CHL_INT0_SL_RX_BCST_ACK_MSK);
1244         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0);
1245
1246         return IRQ_HANDLED;
1247 }
1248
1249 static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p)
1250 {
1251         struct hisi_hba *hisi_hba = p;
1252         u32 irq_msk;
1253         int phy_no = 0;
1254         irqreturn_t res = IRQ_NONE;
1255
1256         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1257                                 & 0x11111111;
1258         while (irq_msk) {
1259                 if (irq_msk  & 1) {
1260                         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1261                                                             CHL_INT0);
1262                         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1263                         int rdy = phy_state & (1 << phy_no);
1264
1265                         if (rdy) {
1266                                 if (irq_value & CHL_INT0_SL_PHY_ENABLE_MSK)
1267                                         /* phy up */
1268                                         if (phy_up_v3_hw(phy_no, hisi_hba)
1269                                                         == IRQ_HANDLED)
1270                                                 res = IRQ_HANDLED;
1271                                 if (irq_value & CHL_INT0_SL_RX_BCST_ACK_MSK)
1272                                         /* phy bcast */
1273                                         if (phy_bcast_v3_hw(phy_no, hisi_hba)
1274                                                         == IRQ_HANDLED)
1275                                                 res = IRQ_HANDLED;
1276                         } else {
1277                                 if (irq_value & CHL_INT0_NOT_RDY_MSK)
1278                                         /* phy down */
1279                                         if (phy_down_v3_hw(phy_no, hisi_hba)
1280                                                         == IRQ_HANDLED)
1281                                                 res = IRQ_HANDLED;
1282                         }
1283                 }
1284                 irq_msk >>= 4;
1285                 phy_no++;
1286         }
1287
1288         return res;
1289 }
1290
1291 static const struct hisi_sas_hw_error port_axi_error[] = {
1292         {
1293                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF),
1294                 .msg = "dma_tx_axi_wr_err",
1295         },
1296         {
1297                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF),
1298                 .msg = "dma_tx_axi_rd_err",
1299         },
1300         {
1301                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF),
1302                 .msg = "dma_rx_axi_wr_err",
1303         },
1304         {
1305                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF),
1306                 .msg = "dma_rx_axi_rd_err",
1307         },
1308 };
1309
1310 static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
1311 {
1312         struct hisi_hba *hisi_hba = p;
1313         struct device *dev = hisi_hba->dev;
1314         struct pci_dev *pci_dev = hisi_hba->pci_dev;
1315         u32 irq_msk;
1316         int phy_no = 0;
1317
1318         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1319                                 & 0xeeeeeeee;
1320
1321         while (irq_msk) {
1322                 u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no,
1323                                                      CHL_INT0);
1324                 u32 irq_value1 = hisi_sas_phy_read32(hisi_hba, phy_no,
1325                                                      CHL_INT1);
1326                 u32 irq_value2 = hisi_sas_phy_read32(hisi_hba, phy_no,
1327                                                      CHL_INT2);
1328                 u32 irq_msk1 = hisi_sas_phy_read32(hisi_hba, phy_no,
1329                                                         CHL_INT1_MSK);
1330                 u32 irq_msk2 = hisi_sas_phy_read32(hisi_hba, phy_no,
1331                                                         CHL_INT2_MSK);
1332
1333                 irq_value1 &= ~irq_msk1;
1334                 irq_value2 &= ~irq_msk2;
1335
1336                 if ((irq_msk & (4 << (phy_no * 4))) &&
1337                                                 irq_value1) {
1338                         int i;
1339
1340                         for (i = 0; i < ARRAY_SIZE(port_axi_error); i++) {
1341                                 const struct hisi_sas_hw_error *error =
1342                                                 &port_axi_error[i];
1343
1344                                 if (!(irq_value1 & error->irq_msk))
1345                                         continue;
1346
1347                                 dev_err(dev, "%s error (phy%d 0x%x) found!\n",
1348                                         error->msg, phy_no, irq_value1);
1349                                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1350                         }
1351
1352                         hisi_sas_phy_write32(hisi_hba, phy_no,
1353                                              CHL_INT1, irq_value1);
1354                 }
1355
1356                 if (irq_msk & (8 << (phy_no * 4)) && irq_value2) {
1357                         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1358
1359                         if (irq_value2 & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) {
1360                                 dev_warn(dev, "phy%d identify timeout\n",
1361                                                         phy_no);
1362                                 hisi_sas_notify_phy_event(phy,
1363                                         HISI_PHYE_LINK_RESET);
1364
1365                         }
1366
1367                         if (irq_value2 & BIT(CHL_INT2_STP_LINK_TIMEOUT_OFF)) {
1368                                 u32 reg_value = hisi_sas_phy_read32(hisi_hba,
1369                                                 phy_no, STP_LINK_TIMEOUT_STATE);
1370
1371                                 dev_warn(dev, "phy%d stp link timeout (0x%x)\n",
1372                                                         phy_no, reg_value);
1373                                 if (reg_value & BIT(4))
1374                                         hisi_sas_notify_phy_event(phy,
1375                                                 HISI_PHYE_LINK_RESET);
1376                         }
1377
1378                         hisi_sas_phy_write32(hisi_hba, phy_no,
1379                                              CHL_INT2, irq_value2);
1380
1381                         if ((irq_value2 & BIT(CHL_INT2_RX_INVLD_DW_OFF)) &&
1382                             (pci_dev->revision == 0x20)) {
1383                                 u32 reg_value;
1384                                 int rc;
1385
1386                                 rc = hisi_sas_read32_poll_timeout_atomic(
1387                                         HILINK_ERR_DFX, reg_value,
1388                                         !((reg_value >> 8) & BIT(phy_no)),
1389                                         1000, 10000);
1390                                 if (rc) {
1391                                         disable_phy_v3_hw(hisi_hba, phy_no);
1392                                         hisi_sas_phy_write32(hisi_hba, phy_no,
1393                                                 CHL_INT2,
1394                                                 BIT(CHL_INT2_RX_INVLD_DW_OFF));
1395                                         hisi_sas_phy_read32(hisi_hba, phy_no,
1396                                                 ERR_CNT_INVLD_DW);
1397                                         mdelay(1);
1398                                         enable_phy_v3_hw(hisi_hba, phy_no);
1399                                 }
1400                         }
1401                 }
1402
1403                 if (irq_msk & (2 << (phy_no * 4)) && irq_value0) {
1404                         hisi_sas_phy_write32(hisi_hba, phy_no,
1405                                         CHL_INT0, irq_value0
1406                                         & (~CHL_INT0_SL_RX_BCST_ACK_MSK)
1407                                         & (~CHL_INT0_SL_PHY_ENABLE_MSK)
1408                                         & (~CHL_INT0_NOT_RDY_MSK));
1409                 }
1410                 irq_msk &= ~(0xe << (phy_no * 4));
1411                 phy_no++;
1412         }
1413
1414         return IRQ_HANDLED;
1415 }
1416
1417 static const struct hisi_sas_hw_error axi_error[] = {
1418         { .msk = BIT(0), .msg = "IOST_AXI_W_ERR" },
1419         { .msk = BIT(1), .msg = "IOST_AXI_R_ERR" },
1420         { .msk = BIT(2), .msg = "ITCT_AXI_W_ERR" },
1421         { .msk = BIT(3), .msg = "ITCT_AXI_R_ERR" },
1422         { .msk = BIT(4), .msg = "SATA_AXI_W_ERR" },
1423         { .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
1424         { .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
1425         { .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
1426         {},
1427 };
1428
1429 static const struct hisi_sas_hw_error fifo_error[] = {
1430         { .msk = BIT(8),  .msg = "CQE_WINFO_FIFO" },
1431         { .msk = BIT(9),  .msg = "CQE_MSG_FIFIO" },
1432         { .msk = BIT(10), .msg = "GETDQE_FIFO" },
1433         { .msk = BIT(11), .msg = "CMDP_FIFO" },
1434         { .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
1435         {},
1436 };
1437
1438 static const struct hisi_sas_hw_error fatal_axi_error[] = {
1439         {
1440                 .irq_msk = BIT(ENT_INT_SRC3_WP_DEPTH_OFF),
1441                 .msg = "write pointer and depth",
1442         },
1443         {
1444                 .irq_msk = BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF),
1445                 .msg = "iptt no match slot",
1446         },
1447         {
1448                 .irq_msk = BIT(ENT_INT_SRC3_RP_DEPTH_OFF),
1449                 .msg = "read pointer and depth",
1450         },
1451         {
1452                 .irq_msk = BIT(ENT_INT_SRC3_AXI_OFF),
1453                 .reg = HGC_AXI_FIFO_ERR_INFO,
1454                 .sub = axi_error,
1455         },
1456         {
1457                 .irq_msk = BIT(ENT_INT_SRC3_FIFO_OFF),
1458                 .reg = HGC_AXI_FIFO_ERR_INFO,
1459                 .sub = fifo_error,
1460         },
1461         {
1462                 .irq_msk = BIT(ENT_INT_SRC3_LM_OFF),
1463                 .msg = "LM add/fetch list",
1464         },
1465         {
1466                 .irq_msk = BIT(ENT_INT_SRC3_ABT_OFF),
1467                 .msg = "SAS_HGC_ABT fetch LM list",
1468         },
1469 };
1470
1471 static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
1472 {
1473         u32 irq_value, irq_msk;
1474         struct hisi_hba *hisi_hba = p;
1475         struct device *dev = hisi_hba->dev;
1476         int i;
1477
1478         irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
1479         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0x1df00);
1480
1481         irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
1482         irq_value &= ~irq_msk;
1483
1484         for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
1485                 const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
1486
1487                 if (!(irq_value & error->irq_msk))
1488                         continue;
1489
1490                 if (error->sub) {
1491                         const struct hisi_sas_hw_error *sub = error->sub;
1492                         u32 err_value = hisi_sas_read32(hisi_hba, error->reg);
1493
1494                         for (; sub->msk || sub->msg; sub++) {
1495                                 if (!(err_value & sub->msk))
1496                                         continue;
1497
1498                                 dev_err(dev, "%s error (0x%x) found!\n",
1499                                         sub->msg, irq_value);
1500                                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1501                         }
1502                 } else {
1503                         dev_err(dev, "%s error (0x%x) found!\n",
1504                                 error->msg, irq_value);
1505                         queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1506                 }
1507         }
1508
1509         if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
1510                 u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
1511                 u32 dev_id = reg_val & ITCT_DEV_MSK;
1512                 struct hisi_sas_device *sas_dev =
1513                                 &hisi_hba->devices[dev_id];
1514
1515                 hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
1516                 dev_dbg(dev, "clear ITCT ok\n");
1517                 complete(sas_dev->completion);
1518         }
1519
1520         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, irq_value & 0x1df00);
1521         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk);
1522
1523         return IRQ_HANDLED;
1524 }
1525
1526 static void
1527 slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task,
1528                struct hisi_sas_slot *slot)
1529 {
1530         struct task_status_struct *ts = &task->task_status;
1531         struct hisi_sas_complete_v3_hdr *complete_queue =
1532                         hisi_hba->complete_hdr[slot->cmplt_queue];
1533         struct hisi_sas_complete_v3_hdr *complete_hdr =
1534                         &complete_queue[slot->cmplt_queue_slot];
1535         struct hisi_sas_err_record_v3 *record =
1536                         hisi_sas_status_buf_addr_mem(slot);
1537         u32 dma_rx_err_type = record->dma_rx_err_type;
1538         u32 trans_tx_fail_type = record->trans_tx_fail_type;
1539
1540         switch (task->task_proto) {
1541         case SAS_PROTOCOL_SSP:
1542                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1543                         ts->residual = trans_tx_fail_type;
1544                         ts->stat = SAS_DATA_UNDERRUN;
1545                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1546                         ts->stat = SAS_QUEUE_FULL;
1547                         slot->abort = 1;
1548                 } else {
1549                         ts->stat = SAS_OPEN_REJECT;
1550                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1551                 }
1552                 break;
1553         case SAS_PROTOCOL_SATA:
1554         case SAS_PROTOCOL_STP:
1555         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1556                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1557                         ts->residual = trans_tx_fail_type;
1558                         ts->stat = SAS_DATA_UNDERRUN;
1559                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1560                         ts->stat = SAS_PHY_DOWN;
1561                         slot->abort = 1;
1562                 } else {
1563                         ts->stat = SAS_OPEN_REJECT;
1564                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1565                 }
1566                 hisi_sas_sata_done(task, slot);
1567                 break;
1568         case SAS_PROTOCOL_SMP:
1569                 ts->stat = SAM_STAT_CHECK_CONDITION;
1570                 break;
1571         default:
1572                 break;
1573         }
1574 }
1575
1576 static int
1577 slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
1578 {
1579         struct sas_task *task = slot->task;
1580         struct hisi_sas_device *sas_dev;
1581         struct device *dev = hisi_hba->dev;
1582         struct task_status_struct *ts;
1583         struct domain_device *device;
1584         struct sas_ha_struct *ha;
1585         enum exec_status sts;
1586         struct hisi_sas_complete_v3_hdr *complete_queue =
1587                         hisi_hba->complete_hdr[slot->cmplt_queue];
1588         struct hisi_sas_complete_v3_hdr *complete_hdr =
1589                         &complete_queue[slot->cmplt_queue_slot];
1590         unsigned long flags;
1591         bool is_internal = slot->is_internal;
1592
1593         if (unlikely(!task || !task->lldd_task || !task->dev))
1594                 return -EINVAL;
1595
1596         ts = &task->task_status;
1597         device = task->dev;
1598         ha = device->port->ha;
1599         sas_dev = device->lldd_dev;
1600
1601         spin_lock_irqsave(&task->task_state_lock, flags);
1602         task->task_state_flags &=
1603                 ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
1604         spin_unlock_irqrestore(&task->task_state_lock, flags);
1605
1606         memset(ts, 0, sizeof(*ts));
1607         ts->resp = SAS_TASK_COMPLETE;
1608
1609         if (unlikely(!sas_dev)) {
1610                 dev_dbg(dev, "slot complete: port has not device\n");
1611                 ts->stat = SAS_PHY_DOWN;
1612                 goto out;
1613         }
1614
1615         /*
1616          * Use SAS+TMF status codes
1617          */
1618         switch ((complete_hdr->dw0 & CMPLT_HDR_ABORT_STAT_MSK)
1619                         >> CMPLT_HDR_ABORT_STAT_OFF) {
1620         case STAT_IO_ABORTED:
1621                 /* this IO has been aborted by abort command */
1622                 ts->stat = SAS_ABORTED_TASK;
1623                 goto out;
1624         case STAT_IO_COMPLETE:
1625                 /* internal abort command complete */
1626                 ts->stat = TMF_RESP_FUNC_SUCC;
1627                 goto out;
1628         case STAT_IO_NO_DEVICE:
1629                 ts->stat = TMF_RESP_FUNC_COMPLETE;
1630                 goto out;
1631         case STAT_IO_NOT_VALID:
1632                 /*
1633                  * abort single IO, the controller can't find the IO
1634                  */
1635                 ts->stat = TMF_RESP_FUNC_FAILED;
1636                 goto out;
1637         default:
1638                 break;
1639         }
1640
1641         /* check for erroneous completion */
1642         if ((complete_hdr->dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) {
1643                 u32 *error_info = hisi_sas_status_buf_addr_mem(slot);
1644
1645                 slot_err_v3_hw(hisi_hba, task, slot);
1646                 if (ts->stat != SAS_DATA_UNDERRUN)
1647                         dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
1648                                 "CQ hdr: 0x%x 0x%x 0x%x 0x%x "
1649                                 "Error info: 0x%x 0x%x 0x%x 0x%x\n",
1650                                 slot->idx, task, sas_dev->device_id,
1651                                 complete_hdr->dw0, complete_hdr->dw1,
1652                                 complete_hdr->act, complete_hdr->dw3,
1653                                 error_info[0], error_info[1],
1654                                 error_info[2], error_info[3]);
1655                 if (unlikely(slot->abort))
1656                         return ts->stat;
1657                 goto out;
1658         }
1659
1660         switch (task->task_proto) {
1661         case SAS_PROTOCOL_SSP: {
1662                 struct ssp_response_iu *iu =
1663                         hisi_sas_status_buf_addr_mem(slot) +
1664                         sizeof(struct hisi_sas_err_record);
1665
1666                 sas_ssp_task_response(dev, task, iu);
1667                 break;
1668         }
1669         case SAS_PROTOCOL_SMP: {
1670                 struct scatterlist *sg_resp = &task->smp_task.smp_resp;
1671                 void *to;
1672
1673                 ts->stat = SAM_STAT_GOOD;
1674                 to = kmap_atomic(sg_page(sg_resp));
1675
1676                 dma_unmap_sg(dev, &task->smp_task.smp_resp, 1,
1677                              DMA_FROM_DEVICE);
1678                 dma_unmap_sg(dev, &task->smp_task.smp_req, 1,
1679                              DMA_TO_DEVICE);
1680                 memcpy(to + sg_resp->offset,
1681                         hisi_sas_status_buf_addr_mem(slot) +
1682                        sizeof(struct hisi_sas_err_record),
1683                        sg_dma_len(sg_resp));
1684                 kunmap_atomic(to);
1685                 break;
1686         }
1687         case SAS_PROTOCOL_SATA:
1688         case SAS_PROTOCOL_STP:
1689         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1690                 ts->stat = SAM_STAT_GOOD;
1691                 hisi_sas_sata_done(task, slot);
1692                 break;
1693         default:
1694                 ts->stat = SAM_STAT_CHECK_CONDITION;
1695                 break;
1696         }
1697
1698         if (!slot->port->port_attached) {
1699                 dev_warn(dev, "slot complete: port %d has removed\n",
1700                         slot->port->sas_port.id);
1701                 ts->stat = SAS_PHY_DOWN;
1702         }
1703
1704 out:
1705         hisi_sas_slot_task_free(hisi_hba, task, slot);
1706         sts = ts->stat;
1707         spin_lock_irqsave(&task->task_state_lock, flags);
1708         if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
1709                 spin_unlock_irqrestore(&task->task_state_lock, flags);
1710                 dev_info(dev, "slot complete: task(%p) aborted\n", task);
1711                 return SAS_ABORTED_TASK;
1712         }
1713         task->task_state_flags |= SAS_TASK_STATE_DONE;
1714         spin_unlock_irqrestore(&task->task_state_lock, flags);
1715
1716         if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
1717                 spin_lock_irqsave(&device->done_lock, flags);
1718                 if (test_bit(SAS_HA_FROZEN, &ha->state)) {
1719                         spin_unlock_irqrestore(&device->done_lock, flags);
1720                         dev_info(dev, "slot complete: task(%p) ignored\n ",
1721                                  task);
1722                         return sts;
1723                 }
1724                 spin_unlock_irqrestore(&device->done_lock, flags);
1725         }
1726
1727         if (task->task_done)
1728                 task->task_done(task);
1729
1730         return sts;
1731 }
1732
1733 static void cq_tasklet_v3_hw(unsigned long val)
1734 {
1735         struct hisi_sas_cq *cq = (struct hisi_sas_cq *)val;
1736         struct hisi_hba *hisi_hba = cq->hisi_hba;
1737         struct hisi_sas_slot *slot;
1738         struct hisi_sas_complete_v3_hdr *complete_queue;
1739         u32 rd_point = cq->rd_point, wr_point;
1740         int queue = cq->id;
1741
1742         complete_queue = hisi_hba->complete_hdr[queue];
1743
1744         wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
1745                                    (0x14 * queue));
1746
1747         while (rd_point != wr_point) {
1748                 struct hisi_sas_complete_v3_hdr *complete_hdr;
1749                 struct device *dev = hisi_hba->dev;
1750                 int iptt;
1751
1752                 complete_hdr = &complete_queue[rd_point];
1753
1754                 iptt = (complete_hdr->dw1) & CMPLT_HDR_IPTT_MSK;
1755                 if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
1756                         slot = &hisi_hba->slot_info[iptt];
1757                         slot->cmplt_queue_slot = rd_point;
1758                         slot->cmplt_queue = queue;
1759                         slot_complete_v3_hw(hisi_hba, slot);
1760                 } else
1761                         dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
1762
1763                 if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
1764                         rd_point = 0;
1765         }
1766
1767         /* update rd_point */
1768         cq->rd_point = rd_point;
1769         hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
1770 }
1771
1772 static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
1773 {
1774         struct hisi_sas_cq *cq = p;
1775         struct hisi_hba *hisi_hba = cq->hisi_hba;
1776         int queue = cq->id;
1777
1778         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue);
1779
1780         tasklet_schedule(&cq->tasklet);
1781
1782         return IRQ_HANDLED;
1783 }
1784
1785 static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
1786 {
1787         struct device *dev = hisi_hba->dev;
1788         struct pci_dev *pdev = hisi_hba->pci_dev;
1789         int vectors, rc;
1790         int i, k;
1791         int max_msi = HISI_SAS_MSI_COUNT_V3_HW;
1792
1793         vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, 1,
1794                                         max_msi, PCI_IRQ_MSI);
1795         if (vectors < max_msi) {
1796                 dev_err(dev, "could not allocate all msi (%d)\n", vectors);
1797                 return -ENOENT;
1798         }
1799
1800         rc = devm_request_irq(dev, pci_irq_vector(pdev, 1),
1801                               int_phy_up_down_bcast_v3_hw, 0,
1802                               DRV_NAME " phy", hisi_hba);
1803         if (rc) {
1804                 dev_err(dev, "could not request phy interrupt, rc=%d\n", rc);
1805                 rc = -ENOENT;
1806                 goto free_irq_vectors;
1807         }
1808
1809         rc = devm_request_irq(dev, pci_irq_vector(pdev, 2),
1810                               int_chnl_int_v3_hw, 0,
1811                               DRV_NAME " channel", hisi_hba);
1812         if (rc) {
1813                 dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc);
1814                 rc = -ENOENT;
1815                 goto free_phy_irq;
1816         }
1817
1818         rc = devm_request_irq(dev, pci_irq_vector(pdev, 11),
1819                               fatal_axi_int_v3_hw, 0,
1820                               DRV_NAME " fatal", hisi_hba);
1821         if (rc) {
1822                 dev_err(dev, "could not request fatal interrupt, rc=%d\n", rc);
1823                 rc = -ENOENT;
1824                 goto free_chnl_interrupt;
1825         }
1826
1827         /* Init tasklets for cq only */
1828         for (i = 0; i < hisi_hba->queue_count; i++) {
1829                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
1830                 struct tasklet_struct *t = &cq->tasklet;
1831
1832                 rc = devm_request_irq(dev, pci_irq_vector(pdev, i+16),
1833                                           cq_interrupt_v3_hw, 0,
1834                                           DRV_NAME " cq", cq);
1835                 if (rc) {
1836                         dev_err(dev,
1837                                 "could not request cq%d interrupt, rc=%d\n",
1838                                 i, rc);
1839                         rc = -ENOENT;
1840                         goto free_cq_irqs;
1841                 }
1842
1843                 tasklet_init(t, cq_tasklet_v3_hw, (unsigned long)cq);
1844         }
1845
1846         return 0;
1847
1848 free_cq_irqs:
1849         for (k = 0; k < i; k++) {
1850                 struct hisi_sas_cq *cq = &hisi_hba->cq[k];
1851
1852                 free_irq(pci_irq_vector(pdev, k+16), cq);
1853         }
1854         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
1855 free_chnl_interrupt:
1856         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
1857 free_phy_irq:
1858         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
1859 free_irq_vectors:
1860         pci_free_irq_vectors(pdev);
1861         return rc;
1862 }
1863
1864 static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
1865 {
1866         int rc;
1867
1868         rc = hw_init_v3_hw(hisi_hba);
1869         if (rc)
1870                 return rc;
1871
1872         rc = interrupt_init_v3_hw(hisi_hba);
1873         if (rc)
1874                 return rc;
1875
1876         return 0;
1877 }
1878
1879 static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
1880                 struct sas_phy_linkrates *r)
1881 {
1882         enum sas_linkrate max = r->maximum_linkrate;
1883         u32 prog_phy_link_rate = 0x800;
1884
1885         prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
1886         hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
1887                              prog_phy_link_rate);
1888 }
1889
1890 static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
1891 {
1892         struct pci_dev *pdev = hisi_hba->pci_dev;
1893         int i;
1894
1895         synchronize_irq(pci_irq_vector(pdev, 1));
1896         synchronize_irq(pci_irq_vector(pdev, 2));
1897         synchronize_irq(pci_irq_vector(pdev, 11));
1898         for (i = 0; i < hisi_hba->queue_count; i++) {
1899                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0x1);
1900                 synchronize_irq(pci_irq_vector(pdev, i + 16));
1901         }
1902
1903         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xffffffff);
1904         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xffffffff);
1905         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffffffff);
1906         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
1907
1908         for (i = 0; i < hisi_hba->n_phy; i++) {
1909                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
1910                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffffff);
1911                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x1);
1912                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x1);
1913                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x1);
1914         }
1915 }
1916
1917 static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba)
1918 {
1919         return hisi_sas_read32(hisi_hba, PHY_STATE);
1920 }
1921
1922 static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1923 {
1924         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1925         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1926         struct sas_phy *sphy = sas_phy->phy;
1927         u32 reg_value;
1928
1929         /* loss dword sync */
1930         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST);
1931         sphy->loss_of_dword_sync_count += reg_value;
1932
1933         /* phy reset problem */
1934         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB);
1935         sphy->phy_reset_problem_count += reg_value;
1936
1937         /* invalid dword */
1938         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1939         sphy->invalid_dword_count += reg_value;
1940
1941         /* disparity err */
1942         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
1943         sphy->running_disparity_error_count += reg_value;
1944
1945 }
1946
1947 static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
1948 {
1949         struct device *dev = hisi_hba->dev;
1950         int rc;
1951         u32 status;
1952
1953         interrupt_disable_v3_hw(hisi_hba);
1954         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
1955         hisi_sas_kill_tasklets(hisi_hba);
1956
1957         hisi_sas_stop_phys(hisi_hba);
1958
1959         mdelay(10);
1960
1961         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE + AM_CTRL_GLOBAL, 0x1);
1962
1963         /* wait until bus idle */
1964         rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
1965                                           AM_CURR_TRANS_RETURN, status,
1966                                           status == 0x3, 10, 100);
1967         if (rc) {
1968                 dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
1969                 return rc;
1970         }
1971
1972         hisi_sas_init_mem(hisi_hba);
1973
1974         return hw_init_v3_hw(hisi_hba);
1975 }
1976
1977 static const struct hisi_sas_hw hisi_sas_v3_hw = {
1978         .hw_init = hisi_sas_v3_init,
1979         .setup_itct = setup_itct_v3_hw,
1980         .max_command_entries = HISI_SAS_COMMAND_ENTRIES_V3_HW,
1981         .get_wideport_bitmap = get_wideport_bitmap_v3_hw,
1982         .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr),
1983         .clear_itct = clear_itct_v3_hw,
1984         .sl_notify = sl_notify_v3_hw,
1985         .prep_ssp = prep_ssp_v3_hw,
1986         .prep_smp = prep_smp_v3_hw,
1987         .prep_stp = prep_ata_v3_hw,
1988         .prep_abort = prep_abort_v3_hw,
1989         .get_free_slot = get_free_slot_v3_hw,
1990         .start_delivery = start_delivery_v3_hw,
1991         .slot_complete = slot_complete_v3_hw,
1992         .phys_init = phys_init_v3_hw,
1993         .phy_start = start_phy_v3_hw,
1994         .phy_disable = disable_phy_v3_hw,
1995         .phy_hard_reset = phy_hard_reset_v3_hw,
1996         .phy_get_max_linkrate = phy_get_max_linkrate_v3_hw,
1997         .phy_set_linkrate = phy_set_linkrate_v3_hw,
1998         .dereg_device = dereg_device_v3_hw,
1999         .soft_reset = soft_reset_v3_hw,
2000         .get_phys_state = get_phys_state_v3_hw,
2001         .get_events = phy_get_events_v3_hw,
2002 };
2003
2004 static struct Scsi_Host *
2005 hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
2006 {
2007         struct Scsi_Host *shost;
2008         struct hisi_hba *hisi_hba;
2009         struct device *dev = &pdev->dev;
2010
2011         shost = scsi_host_alloc(hisi_sas_sht, sizeof(*hisi_hba));
2012         if (!shost) {
2013                 dev_err(dev, "shost alloc failed\n");
2014                 return NULL;
2015         }
2016         hisi_hba = shost_priv(shost);
2017
2018         INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler);
2019         hisi_hba->hw = &hisi_sas_v3_hw;
2020         hisi_hba->pci_dev = pdev;
2021         hisi_hba->dev = dev;
2022         hisi_hba->shost = shost;
2023         SHOST_TO_SAS_HA(shost) = &hisi_hba->sha;
2024
2025         timer_setup(&hisi_hba->timer, NULL, 0);
2026
2027         if (hisi_sas_get_fw_info(hisi_hba) < 0)
2028                 goto err_out;
2029
2030         if (hisi_sas_alloc(hisi_hba, shost)) {
2031                 hisi_sas_free(hisi_hba);
2032                 goto err_out;
2033         }
2034
2035         return shost;
2036 err_out:
2037         scsi_host_put(shost);
2038         dev_err(dev, "shost alloc failed\n");
2039         return NULL;
2040 }
2041
2042 static int
2043 hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2044 {
2045         struct Scsi_Host *shost;
2046         struct hisi_hba *hisi_hba;
2047         struct device *dev = &pdev->dev;
2048         struct asd_sas_phy **arr_phy;
2049         struct asd_sas_port **arr_port;
2050         struct sas_ha_struct *sha;
2051         int rc, phy_nr, port_nr, i;
2052
2053         rc = pci_enable_device(pdev);
2054         if (rc)
2055                 goto err_out;
2056
2057         pci_set_master(pdev);
2058
2059         rc = pci_request_regions(pdev, DRV_NAME);
2060         if (rc)
2061                 goto err_out_disable_device;
2062
2063         if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) ||
2064             (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0)) {
2065                 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) ||
2066                    (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)) != 0)) {
2067                         dev_err(dev, "No usable DMA addressing method\n");
2068                         rc = -EIO;
2069                         goto err_out_regions;
2070                 }
2071         }
2072
2073         shost = hisi_sas_shost_alloc_pci(pdev);
2074         if (!shost) {
2075                 rc = -ENOMEM;
2076                 goto err_out_regions;
2077         }
2078
2079         sha = SHOST_TO_SAS_HA(shost);
2080         hisi_hba = shost_priv(shost);
2081         dev_set_drvdata(dev, sha);
2082
2083         hisi_hba->regs = pcim_iomap(pdev, 5, 0);
2084         if (!hisi_hba->regs) {
2085                 dev_err(dev, "cannot map register.\n");
2086                 rc = -ENOMEM;
2087                 goto err_out_ha;
2088         }
2089
2090         phy_nr = port_nr = hisi_hba->n_phy;
2091
2092         arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
2093         arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL);
2094         if (!arr_phy || !arr_port) {
2095                 rc = -ENOMEM;
2096                 goto err_out_ha;
2097         }
2098
2099         sha->sas_phy = arr_phy;
2100         sha->sas_port = arr_port;
2101         sha->core.shost = shost;
2102         sha->lldd_ha = hisi_hba;
2103
2104         shost->transportt = hisi_sas_stt;
2105         shost->max_id = HISI_SAS_MAX_DEVICES;
2106         shost->max_lun = ~0;
2107         shost->max_channel = 1;
2108         shost->max_cmd_len = 16;
2109         shost->sg_tablesize = min_t(u16, SG_ALL, HISI_SAS_SGE_PAGE_CNT);
2110         shost->can_queue = hisi_hba->hw->max_command_entries;
2111         shost->cmd_per_lun = hisi_hba->hw->max_command_entries;
2112
2113         sha->sas_ha_name = DRV_NAME;
2114         sha->dev = dev;
2115         sha->lldd_module = THIS_MODULE;
2116         sha->sas_addr = &hisi_hba->sas_addr[0];
2117         sha->num_phys = hisi_hba->n_phy;
2118         sha->core.shost = hisi_hba->shost;
2119
2120         for (i = 0; i < hisi_hba->n_phy; i++) {
2121                 sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy;
2122                 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
2123         }
2124
2125         rc = scsi_add_host(shost, dev);
2126         if (rc)
2127                 goto err_out_ha;
2128
2129         rc = sas_register_ha(sha);
2130         if (rc)
2131                 goto err_out_register_ha;
2132
2133         rc = hisi_hba->hw->hw_init(hisi_hba);
2134         if (rc)
2135                 goto err_out_register_ha;
2136
2137         scsi_scan_host(shost);
2138
2139         return 0;
2140
2141 err_out_register_ha:
2142         scsi_remove_host(shost);
2143 err_out_ha:
2144         scsi_host_put(shost);
2145 err_out_regions:
2146         pci_release_regions(pdev);
2147 err_out_disable_device:
2148         pci_disable_device(pdev);
2149 err_out:
2150         return rc;
2151 }
2152
2153 static void
2154 hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
2155 {
2156         int i;
2157
2158         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
2159         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
2160         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
2161         for (i = 0; i < hisi_hba->queue_count; i++) {
2162                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2163
2164                 free_irq(pci_irq_vector(pdev, i+16), cq);
2165         }
2166         pci_free_irq_vectors(pdev);
2167 }
2168
2169 static void hisi_sas_v3_remove(struct pci_dev *pdev)
2170 {
2171         struct device *dev = &pdev->dev;
2172         struct sas_ha_struct *sha = dev_get_drvdata(dev);
2173         struct hisi_hba *hisi_hba = sha->lldd_ha;
2174         struct Scsi_Host *shost = sha->core.shost;
2175
2176         if (timer_pending(&hisi_hba->timer))
2177                 del_timer(&hisi_hba->timer);
2178
2179         sas_unregister_ha(sha);
2180         sas_remove_host(sha->core.shost);
2181
2182         hisi_sas_v3_destroy_irqs(pdev, hisi_hba);
2183         hisi_sas_kill_tasklets(hisi_hba);
2184         pci_release_regions(pdev);
2185         pci_disable_device(pdev);
2186         hisi_sas_free(hisi_hba);
2187         scsi_host_put(shost);
2188 }
2189
2190 static const struct hisi_sas_hw_error sas_ras_intr0_nfe[] = {
2191         { .irq_msk = BIT(19), .msg = "HILINK_INT" },
2192         { .irq_msk = BIT(20), .msg = "HILINK_PLL0_OUT_OF_LOCK" },
2193         { .irq_msk = BIT(21), .msg = "HILINK_PLL1_OUT_OF_LOCK" },
2194         { .irq_msk = BIT(22), .msg = "HILINK_LOSS_OF_REFCLK0" },
2195         { .irq_msk = BIT(23), .msg = "HILINK_LOSS_OF_REFCLK1" },
2196         { .irq_msk = BIT(24), .msg = "DMAC0_TX_POISON" },
2197         { .irq_msk = BIT(25), .msg = "DMAC1_TX_POISON" },
2198         { .irq_msk = BIT(26), .msg = "DMAC2_TX_POISON" },
2199         { .irq_msk = BIT(27), .msg = "DMAC3_TX_POISON" },
2200         { .irq_msk = BIT(28), .msg = "DMAC4_TX_POISON" },
2201         { .irq_msk = BIT(29), .msg = "DMAC5_TX_POISON" },
2202         { .irq_msk = BIT(30), .msg = "DMAC6_TX_POISON" },
2203         { .irq_msk = BIT(31), .msg = "DMAC7_TX_POISON" },
2204 };
2205
2206 static const struct hisi_sas_hw_error sas_ras_intr1_nfe[] = {
2207         { .irq_msk = BIT(0), .msg = "RXM_CFG_MEM3_ECC2B_INTR" },
2208         { .irq_msk = BIT(1), .msg = "RXM_CFG_MEM2_ECC2B_INTR" },
2209         { .irq_msk = BIT(2), .msg = "RXM_CFG_MEM1_ECC2B_INTR" },
2210         { .irq_msk = BIT(3), .msg = "RXM_CFG_MEM0_ECC2B_INTR" },
2211         { .irq_msk = BIT(4), .msg = "HGC_CQE_ECC2B_INTR" },
2212         { .irq_msk = BIT(5), .msg = "LM_CFG_IOSTL_ECC2B_INTR" },
2213         { .irq_msk = BIT(6), .msg = "LM_CFG_ITCTL_ECC2B_INTR" },
2214         { .irq_msk = BIT(7), .msg = "HGC_ITCT_ECC2B_INTR" },
2215         { .irq_msk = BIT(8), .msg = "HGC_IOST_ECC2B_INTR" },
2216         { .irq_msk = BIT(9), .msg = "HGC_DQE_ECC2B_INTR" },
2217         { .irq_msk = BIT(10), .msg = "DMAC0_RAM_ECC2B_INTR" },
2218         { .irq_msk = BIT(11), .msg = "DMAC1_RAM_ECC2B_INTR" },
2219         { .irq_msk = BIT(12), .msg = "DMAC2_RAM_ECC2B_INTR" },
2220         { .irq_msk = BIT(13), .msg = "DMAC3_RAM_ECC2B_INTR" },
2221         { .irq_msk = BIT(14), .msg = "DMAC4_RAM_ECC2B_INTR" },
2222         { .irq_msk = BIT(15), .msg = "DMAC5_RAM_ECC2B_INTR" },
2223         { .irq_msk = BIT(16), .msg = "DMAC6_RAM_ECC2B_INTR" },
2224         { .irq_msk = BIT(17), .msg = "DMAC7_RAM_ECC2B_INTR" },
2225         { .irq_msk = BIT(18), .msg = "OOO_RAM_ECC2B_INTR" },
2226         { .irq_msk = BIT(20), .msg = "HGC_DQE_POISON_INTR" },
2227         { .irq_msk = BIT(21), .msg = "HGC_IOST_POISON_INTR" },
2228         { .irq_msk = BIT(22), .msg = "HGC_ITCT_POISON_INTR" },
2229         { .irq_msk = BIT(23), .msg = "HGC_ITCT_NCQ_POISON_INTR" },
2230         { .irq_msk = BIT(24), .msg = "DMAC0_RX_POISON" },
2231         { .irq_msk = BIT(25), .msg = "DMAC1_RX_POISON" },
2232         { .irq_msk = BIT(26), .msg = "DMAC2_RX_POISON" },
2233         { .irq_msk = BIT(27), .msg = "DMAC3_RX_POISON" },
2234         { .irq_msk = BIT(28), .msg = "DMAC4_RX_POISON" },
2235         { .irq_msk = BIT(29), .msg = "DMAC5_RX_POISON" },
2236         { .irq_msk = BIT(30), .msg = "DMAC6_RX_POISON" },
2237         { .irq_msk = BIT(31), .msg = "DMAC7_RX_POISON" },
2238 };
2239
2240 static const struct hisi_sas_hw_error sas_ras_intr2_nfe[] = {
2241         { .irq_msk = BIT(0), .msg = "DMAC0_AXI_BUS_ERR" },
2242         { .irq_msk = BIT(1), .msg = "DMAC1_AXI_BUS_ERR" },
2243         { .irq_msk = BIT(2), .msg = "DMAC2_AXI_BUS_ERR" },
2244         { .irq_msk = BIT(3), .msg = "DMAC3_AXI_BUS_ERR" },
2245         { .irq_msk = BIT(4), .msg = "DMAC4_AXI_BUS_ERR" },
2246         { .irq_msk = BIT(5), .msg = "DMAC5_AXI_BUS_ERR" },
2247         { .irq_msk = BIT(6), .msg = "DMAC6_AXI_BUS_ERR" },
2248         { .irq_msk = BIT(7), .msg = "DMAC7_AXI_BUS_ERR" },
2249         { .irq_msk = BIT(8), .msg = "DMAC0_FIFO_OMIT_ERR" },
2250         { .irq_msk = BIT(9), .msg = "DMAC1_FIFO_OMIT_ERR" },
2251         { .irq_msk = BIT(10), .msg = "DMAC2_FIFO_OMIT_ERR" },
2252         { .irq_msk = BIT(11), .msg = "DMAC3_FIFO_OMIT_ERR" },
2253         { .irq_msk = BIT(12), .msg = "DMAC4_FIFO_OMIT_ERR" },
2254         { .irq_msk = BIT(13), .msg = "DMAC5_FIFO_OMIT_ERR" },
2255         { .irq_msk = BIT(14), .msg = "DMAC6_FIFO_OMIT_ERR" },
2256         { .irq_msk = BIT(15), .msg = "DMAC7_FIFO_OMIT_ERR" },
2257         { .irq_msk = BIT(16), .msg = "HGC_RLSE_SLOT_UNMATCH" },
2258         { .irq_msk = BIT(17), .msg = "HGC_LM_ADD_FCH_LIST_ERR" },
2259         { .irq_msk = BIT(18), .msg = "HGC_AXI_BUS_ERR" },
2260         { .irq_msk = BIT(19), .msg = "HGC_FIFO_OMIT_ERR" },
2261 };
2262
2263 static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
2264 {
2265         struct device *dev = hisi_hba->dev;
2266         const struct hisi_sas_hw_error *ras_error;
2267         bool need_reset = false;
2268         u32 irq_value;
2269         int i;
2270
2271         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR0);
2272         for (i = 0; i < ARRAY_SIZE(sas_ras_intr0_nfe); i++) {
2273                 ras_error = &sas_ras_intr0_nfe[i];
2274                 if (ras_error->irq_msk & irq_value) {
2275                         dev_warn(dev, "SAS_RAS_INTR0: %s(irq_value=0x%x) found.\n",
2276                                         ras_error->msg, irq_value);
2277                         need_reset = true;
2278                 }
2279         }
2280         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0, irq_value);
2281
2282         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR1);
2283         for (i = 0; i < ARRAY_SIZE(sas_ras_intr1_nfe); i++) {
2284                 ras_error = &sas_ras_intr1_nfe[i];
2285                 if (ras_error->irq_msk & irq_value) {
2286                         dev_warn(dev, "SAS_RAS_INTR1: %s(irq_value=0x%x) found.\n",
2287                                         ras_error->msg, irq_value);
2288                         need_reset = true;
2289                 }
2290         }
2291         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1, irq_value);
2292
2293         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR2);
2294         for (i = 0; i < ARRAY_SIZE(sas_ras_intr2_nfe); i++) {
2295                 ras_error = &sas_ras_intr2_nfe[i];
2296                 if (ras_error->irq_msk & irq_value) {
2297                         dev_warn(dev, "SAS_RAS_INTR2: %s(irq_value=0x%x) found.\n",
2298                                         ras_error->msg, irq_value);
2299                         need_reset = true;
2300                 }
2301         }
2302         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2, irq_value);
2303
2304         return need_reset;
2305 }
2306
2307 static pci_ers_result_t hisi_sas_error_detected_v3_hw(struct pci_dev *pdev,
2308                 pci_channel_state_t state)
2309 {
2310         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2311         struct hisi_hba *hisi_hba = sha->lldd_ha;
2312         struct device *dev = hisi_hba->dev;
2313
2314         dev_info(dev, "PCI error: detected callback, state(%d)!!\n", state);
2315         if (state == pci_channel_io_perm_failure)
2316                 return PCI_ERS_RESULT_DISCONNECT;
2317
2318         if (process_non_fatal_error_v3_hw(hisi_hba))
2319                 return PCI_ERS_RESULT_NEED_RESET;
2320
2321         return PCI_ERS_RESULT_CAN_RECOVER;
2322 }
2323
2324 static pci_ers_result_t hisi_sas_mmio_enabled_v3_hw(struct pci_dev *pdev)
2325 {
2326         return PCI_ERS_RESULT_RECOVERED;
2327 }
2328
2329 static pci_ers_result_t hisi_sas_slot_reset_v3_hw(struct pci_dev *pdev)
2330 {
2331         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2332         struct hisi_hba *hisi_hba = sha->lldd_ha;
2333         struct device *dev = hisi_hba->dev;
2334         HISI_SAS_DECLARE_RST_WORK_ON_STACK(r);
2335
2336         dev_info(dev, "PCI error: slot reset callback!!\n");
2337         queue_work(hisi_hba->wq, &r.work);
2338         wait_for_completion(r.completion);
2339         if (r.done)
2340                 return PCI_ERS_RESULT_RECOVERED;
2341
2342         return PCI_ERS_RESULT_DISCONNECT;
2343 }
2344
2345 enum {
2346         /* instances of the controller */
2347         hip08,
2348 };
2349
2350 static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
2351 {
2352         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2353         struct hisi_hba *hisi_hba = sha->lldd_ha;
2354         struct device *dev = hisi_hba->dev;
2355         struct Scsi_Host *shost = hisi_hba->shost;
2356         u32 device_state, status;
2357         int rc;
2358         u32 reg_val;
2359
2360         if (!pdev->pm_cap) {
2361                 dev_err(dev, "PCI PM not supported\n");
2362                 return -ENODEV;
2363         }
2364
2365         set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2366         scsi_block_requests(shost);
2367         set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2368         flush_workqueue(hisi_hba->wq);
2369         /* disable DQ/PHY/bus */
2370         interrupt_disable_v3_hw(hisi_hba);
2371         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
2372         hisi_sas_kill_tasklets(hisi_hba);
2373
2374         hisi_sas_stop_phys(hisi_hba);
2375
2376         reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
2377                 AM_CTRL_GLOBAL);
2378         reg_val |= 0x1;
2379         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2380                 AM_CTRL_GLOBAL, reg_val);
2381
2382         /* wait until bus idle */
2383         rc = hisi_sas_read32_poll_timeout(AXI_MASTER_CFG_BASE +
2384                                           AM_CURR_TRANS_RETURN, status,
2385                                           status == 0x3, 10, 100);
2386         if (rc) {
2387                 dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
2388                 clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2389                 clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2390                 scsi_unblock_requests(shost);
2391                 return rc;
2392         }
2393
2394         hisi_sas_init_mem(hisi_hba);
2395
2396         device_state = pci_choose_state(pdev, state);
2397         dev_warn(dev, "entering operating state [D%d]\n",
2398                         device_state);
2399         pci_save_state(pdev);
2400         pci_disable_device(pdev);
2401         pci_set_power_state(pdev, device_state);
2402
2403         hisi_sas_release_tasks(hisi_hba);
2404
2405         sas_suspend_ha(sha);
2406         return 0;
2407 }
2408
2409 static int hisi_sas_v3_resume(struct pci_dev *pdev)
2410 {
2411         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2412         struct hisi_hba *hisi_hba = sha->lldd_ha;
2413         struct Scsi_Host *shost = hisi_hba->shost;
2414         struct device *dev = hisi_hba->dev;
2415         unsigned int rc;
2416         u32 device_state = pdev->current_state;
2417
2418         dev_warn(dev, "resuming from operating state [D%d]\n",
2419                         device_state);
2420         pci_set_power_state(pdev, PCI_D0);
2421         pci_enable_wake(pdev, PCI_D0, 0);
2422         pci_restore_state(pdev);
2423         rc = pci_enable_device(pdev);
2424         if (rc)
2425                 dev_err(dev, "enable device failed during resume (%d)\n", rc);
2426
2427         pci_set_master(pdev);
2428         scsi_unblock_requests(shost);
2429         clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2430
2431         sas_prep_resume_ha(sha);
2432         init_reg_v3_hw(hisi_hba);
2433         hisi_hba->hw->phys_init(hisi_hba);
2434         sas_resume_ha(sha);
2435         clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2436
2437         return 0;
2438 }
2439
2440 static const struct pci_device_id sas_v3_pci_table[] = {
2441         { PCI_VDEVICE(HUAWEI, 0xa230), hip08 },
2442         {}
2443 };
2444 MODULE_DEVICE_TABLE(pci, sas_v3_pci_table);
2445
2446 static const struct pci_error_handlers hisi_sas_err_handler = {
2447         .error_detected = hisi_sas_error_detected_v3_hw,
2448         .mmio_enabled   = hisi_sas_mmio_enabled_v3_hw,
2449         .slot_reset     = hisi_sas_slot_reset_v3_hw,
2450 };
2451
2452 static struct pci_driver sas_v3_pci_driver = {
2453         .name           = DRV_NAME,
2454         .id_table       = sas_v3_pci_table,
2455         .probe          = hisi_sas_v3_probe,
2456         .remove         = hisi_sas_v3_remove,
2457         .suspend        = hisi_sas_v3_suspend,
2458         .resume         = hisi_sas_v3_resume,
2459         .err_handler    = &hisi_sas_err_handler,
2460 };
2461
2462 module_pci_driver(sas_v3_pci_driver);
2463
2464 MODULE_LICENSE("GPL");
2465 MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
2466 MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device");
2467 MODULE_ALIAS("pci:" DRV_NAME);