drivers/scsi/mpi3mr/mpi3mr_fw.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Driver for Broadcom MPI3 Storage Controllers
   4  *
   5  * Copyright (C) 2017-2022 Broadcom Inc.
   6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
   7  *
   8  */
   9
  10 #include "mpi3mr.h"
  11 #include <linux/io-64-nonatomic-lo-hi.h>
  12
  13 static int
  14 mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type, u32 reset_reason);
  15 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc);
  16 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
  17         struct mpi3_ioc_facts_data *facts_data);
  18 static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
  19         struct mpi3mr_drv_cmd *drv_cmd);
  20
  21 static int poll_queues;
  22 module_param(poll_queues, int, 0444);
  23 MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)");
  24
  25 #if defined(writeq) && defined(CONFIG_64BIT)
  26 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  27 {
  28         writeq(b, addr);
  29 }
  30 #else
  31 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  32 {
  33         __u64 data_out = b;
  34
  35         writel((u32)(data_out), addr);
  36         writel((u32)(data_out >> 32), (addr + 4));
  37 }
  38 #endif
  39
  40 static inline bool
  41 mpi3mr_check_req_qfull(struct op_req_qinfo *op_req_q)
  42 {
  43         u16 pi, ci, max_entries;
  44         bool is_qfull = false;
  45
  46         pi = op_req_q->pi;
  47         ci = READ_ONCE(op_req_q->ci);
  48         max_entries = op_req_q->num_requests;
  49
  50         if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))
  51                 is_qfull = true;
  52
  53         return is_qfull;
  54 }
  55
  56 static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
  57 {
  58         u16 i, max_vectors;
  59
  60         max_vectors = mrioc->intr_info_count;
  61
  62         for (i = 0; i < max_vectors; i++)
  63                 synchronize_irq(pci_irq_vector(mrioc->pdev, i));
  64 }
  65
  66 void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
  67 {
  68         mrioc->intr_enabled = 0;
  69         mpi3mr_sync_irqs(mrioc);
  70 }
  71
  72 void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
  73 {
  74         mrioc->intr_enabled = 1;
  75 }
  76
  77 static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
  78 {
  79         u16 i;
  80
  81         mpi3mr_ioc_disable_intr(mrioc);
  82
  83         if (!mrioc->intr_info)
  84                 return;
  85
  86         for (i = 0; i < mrioc->intr_info_count; i++)
  87                 free_irq(pci_irq_vector(mrioc->pdev, i),
  88                     (mrioc->intr_info + i));
  89
  90         kfree(mrioc->intr_info);
  91         mrioc->intr_info = NULL;
  92         mrioc->intr_info_count = 0;
  93         mrioc->is_intr_info_set = false;
  94         pci_free_irq_vectors(mrioc->pdev);
  95 }
  96
  97 void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
  98         dma_addr_t dma_addr)
  99 {
 100         struct mpi3_sge_common *sgel = paddr;
 101
 102         sgel->flags = flags;
 103         sgel->length = cpu_to_le32(length);
 104         sgel->address = cpu_to_le64(dma_addr);
 105 }
 106
 107 void mpi3mr_build_zero_len_sge(void *paddr)
 108 {
 109         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
 110
 111         mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);
 112 }
 113
 114 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 115         dma_addr_t phys_addr)
 116 {
 117         if (!phys_addr)
 118                 return NULL;
 119
 120         if ((phys_addr < mrioc->reply_buf_dma) ||
 121             (phys_addr > mrioc->reply_buf_dma_max_address))
 122                 return NULL;
 123
 124         return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
 125 }
 126
 127 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
 128         dma_addr_t phys_addr)
 129 {
 130         if (!phys_addr)
 131                 return NULL;
 132
 133         return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
 134 }
 135
 136 static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
 137         u64 reply_dma)
 138 {
 139         u32 old_idx = 0;
 140         unsigned long flags;
 141
 142         spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);
 143         old_idx  =  mrioc->reply_free_queue_host_index;
 144         mrioc->reply_free_queue_host_index = (
 145             (mrioc->reply_free_queue_host_index ==
 146             (mrioc->reply_free_qsz - 1)) ? 0 :
 147             (mrioc->reply_free_queue_host_index + 1));
 148         mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
 149         writel(mrioc->reply_free_queue_host_index,
 150             &mrioc->sysif_regs->reply_free_host_index);
 151         spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);
 152 }
 153
 154 void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
 155         u64 sense_buf_dma)
 156 {
 157         u32 old_idx = 0;
 158         unsigned long flags;
 159
 160         spin_lock_irqsave(&mrioc->sbq_lock, flags);
 161         old_idx  =  mrioc->sbq_host_index;
 162         mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
 163             (mrioc->sense_buf_q_sz - 1)) ? 0 :
 164             (mrioc->sbq_host_index + 1));
 165         mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
 166         writel(mrioc->sbq_host_index,
 167             &mrioc->sysif_regs->sense_buffer_free_host_index);
 168         spin_unlock_irqrestore(&mrioc->sbq_lock, flags);
 169 }
 170
 171 static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
 172         struct mpi3_event_notification_reply *event_reply)
 173 {
 174         char *desc = NULL;
 175         u16 event;
 176
 177         event = event_reply->event;
 178
 179         switch (event) {
 180         case MPI3_EVENT_LOG_DATA:
 181                 desc = "Log Data";
 182                 break;
 183         case MPI3_EVENT_CHANGE:
 184                 desc = "Event Change";
 185                 break;
 186         case MPI3_EVENT_GPIO_INTERRUPT:
 187                 desc = "GPIO Interrupt";
 188                 break;
 189         case MPI3_EVENT_CABLE_MGMT:
 190                 desc = "Cable Management";
 191                 break;
 192         case MPI3_EVENT_ENERGY_PACK_CHANGE:
 193                 desc = "Energy Pack Change";
 194                 break;
 195         case MPI3_EVENT_DEVICE_ADDED:
 196         {
 197                 struct mpi3_device_page0 *event_data =
 198                     (struct mpi3_device_page0 *)event_reply->event_data;
 199                 ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
 200                     event_data->dev_handle, event_data->device_form);
 201                 return;
 202         }
 203         case MPI3_EVENT_DEVICE_INFO_CHANGED:
 204         {
 205                 struct mpi3_device_page0 *event_data =
 206                     (struct mpi3_device_page0 *)event_reply->event_data;
 207                 ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
 208                     event_data->dev_handle, event_data->device_form);
 209                 return;
 210         }
 211         case MPI3_EVENT_DEVICE_STATUS_CHANGE:
 212         {
 213                 struct mpi3_event_data_device_status_change *event_data =
 214                     (struct mpi3_event_data_device_status_change *)event_reply->event_data;
 215                 ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
 216                     event_data->dev_handle, event_data->reason_code);
 217                 return;
 218         }
 219         case MPI3_EVENT_SAS_DISCOVERY:
 220         {
 221                 struct mpi3_event_data_sas_discovery *event_data =
 222                     (struct mpi3_event_data_sas_discovery *)event_reply->event_data;
 223                 ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
 224                     (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?
 225                     "start" : "stop",
 226                     le32_to_cpu(event_data->discovery_status));
 227                 return;
 228         }
 229         case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
 230                 desc = "SAS Broadcast Primitive";
 231                 break;
 232         case MPI3_EVENT_SAS_NOTIFY_PRIMITIVE:
 233                 desc = "SAS Notify Primitive";
 234                 break;
 235         case MPI3_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
 236                 desc = "SAS Init Device Status Change";
 237                 break;
 238         case MPI3_EVENT_SAS_INIT_TABLE_OVERFLOW:
 239                 desc = "SAS Init Table Overflow";
 240                 break;
 241         case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 242                 desc = "SAS Topology Change List";
 243                 break;
 244         case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
 245                 desc = "Enclosure Device Status Change";
 246                 break;
 247         case MPI3_EVENT_ENCL_DEVICE_ADDED:
 248                 desc = "Enclosure Added";
 249                 break;
 250         case MPI3_EVENT_HARD_RESET_RECEIVED:
 251                 desc = "Hard Reset Received";
 252                 break;
 253         case MPI3_EVENT_SAS_PHY_COUNTER:
 254                 desc = "SAS PHY Counter";
 255                 break;
 256         case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
 257                 desc = "SAS Device Discovery Error";
 258                 break;
 259         case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
 260                 desc = "PCIE Topology Change List";
 261                 break;
 262         case MPI3_EVENT_PCIE_ENUMERATION:
 263         {
 264                 struct mpi3_event_data_pcie_enumeration *event_data =
 265                     (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;
 266                 ioc_info(mrioc, "PCIE Enumeration: (%s)",
 267                     (event_data->reason_code ==
 268                     MPI3_EVENT_PCIE_ENUM_RC_STARTED) ? "start" : "stop");
 269                 if (event_data->enumeration_status)
 270                         ioc_info(mrioc, "enumeration_status(0x%08x)\n",
 271                             le32_to_cpu(event_data->enumeration_status));
 272                 return;
 273         }
 274         case MPI3_EVENT_PREPARE_FOR_RESET:
 275                 desc = "Prepare For Reset";
 276                 break;
 277         }
 278
 279         if (!desc)
 280                 return;
 281
 282         ioc_info(mrioc, "%s\n", desc);
 283 }
 284
 285 static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
 286         struct mpi3_default_reply *def_reply)
 287 {
 288         struct mpi3_event_notification_reply *event_reply =
 289             (struct mpi3_event_notification_reply *)def_reply;
 290
 291         mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
 292         mpi3mr_print_event_data(mrioc, event_reply);
 293         mpi3mr_os_handle_events(mrioc, event_reply);
 294 }
 295
 296 static struct mpi3mr_drv_cmd *
 297 mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
 298         struct mpi3_default_reply *def_reply)
 299 {
 300         u16 idx;
 301
 302         switch (host_tag) {
 303         case MPI3MR_HOSTTAG_INITCMDS:
 304                 return &mrioc->init_cmds;
 305         case MPI3MR_HOSTTAG_CFG_CMDS:
 306                 return &mrioc->cfg_cmds;
 307         case MPI3MR_HOSTTAG_BSG_CMDS:
 308                 return &mrioc->bsg_cmds;
 309         case MPI3MR_HOSTTAG_BLK_TMS:
 310                 return &mrioc->host_tm_cmds;
 311         case MPI3MR_HOSTTAG_PEL_ABORT:
 312                 return &mrioc->pel_abort_cmd;
 313         case MPI3MR_HOSTTAG_PEL_WAIT:
 314                 return &mrioc->pel_cmds;
 315         case MPI3MR_HOSTTAG_TRANSPORT_CMDS:
 316                 return &mrioc->transport_cmds;
 317         case MPI3MR_HOSTTAG_INVALID:
 318                 if (def_reply && def_reply->function ==
 319                     MPI3_FUNCTION_EVENT_NOTIFICATION)
 320                         mpi3mr_handle_events(mrioc, def_reply);
 321                 return NULL;
 322         default:
 323                 break;
 324         }
 325         if (host_tag >= MPI3MR_HOSTTAG_DEVRMCMD_MIN &&
 326             host_tag <= MPI3MR_HOSTTAG_DEVRMCMD_MAX) {
 327                 idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
 328                 return &mrioc->dev_rmhs_cmds[idx];
 329         }
 330
 331         if (host_tag >= MPI3MR_HOSTTAG_EVTACKCMD_MIN &&
 332             host_tag <= MPI3MR_HOSTTAG_EVTACKCMD_MAX) {
 333                 idx = host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;
 334                 return &mrioc->evtack_cmds[idx];
 335         }
 336
 337         return NULL;
 338 }
 339
 340 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
 341         struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
 342 {
 343         u16 reply_desc_type, host_tag = 0;
 344         u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
 345         u32 ioc_loginfo = 0;
 346         struct mpi3_status_reply_descriptor *status_desc;
 347         struct mpi3_address_reply_descriptor *addr_desc;
 348         struct mpi3_success_reply_descriptor *success_desc;
 349         struct mpi3_default_reply *def_reply = NULL;
 350         struct mpi3mr_drv_cmd *cmdptr = NULL;
 351         struct mpi3_scsi_io_reply *scsi_reply;
 352         u8 *sense_buf = NULL;
 353
 354         *reply_dma = 0;
 355         reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
 356             MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
 357         switch (reply_desc_type) {
 358         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
 359                 status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
 360                 host_tag = le16_to_cpu(status_desc->host_tag);
 361                 ioc_status = le16_to_cpu(status_desc->ioc_status);
 362                 if (ioc_status &
 363                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 364                         ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
 365                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 366                 break;
 367         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
 368                 addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
 369                 *reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
 370                 def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
 371                 if (!def_reply)
 372                         goto out;
 373                 host_tag = le16_to_cpu(def_reply->host_tag);
 374                 ioc_status = le16_to_cpu(def_reply->ioc_status);
 375                 if (ioc_status &
 376                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 377                         ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
 378                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 379                 if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
 380                         scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
 381                         sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
 382                             le64_to_cpu(scsi_reply->sense_data_buffer_address));
 383                 }
 384                 break;
 385         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
 386                 success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
 387                 host_tag = le16_to_cpu(success_desc->host_tag);
 388                 break;
 389         default:
 390                 break;
 391         }
 392
 393         cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
 394         if (cmdptr) {
 395                 if (cmdptr->state & MPI3MR_CMD_PENDING) {
 396                         cmdptr->state |= MPI3MR_CMD_COMPLETE;
 397                         cmdptr->ioc_loginfo = ioc_loginfo;
 398                         cmdptr->ioc_status = ioc_status;
 399                         cmdptr->state &= ~MPI3MR_CMD_PENDING;
 400                         if (def_reply) {
 401                                 cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
 402                                 memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
 403                                     mrioc->reply_sz);
 404                         }
 405                         if (cmdptr->is_waiting) {
 406                                 complete(&cmdptr->done);
 407                                 cmdptr->is_waiting = 0;
 408                         } else if (cmdptr->callback)
 409                                 cmdptr->callback(mrioc, cmdptr);
 410                 }
 411         }
 412 out:
 413         if (sense_buf)
 414                 mpi3mr_repost_sense_buf(mrioc,
 415                     le64_to_cpu(scsi_reply->sense_data_buffer_address));
 416 }
 417
 418 static int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 419 {
 420         u32 exp_phase = mrioc->admin_reply_ephase;
 421         u32 admin_reply_ci = mrioc->admin_reply_ci;
 422         u32 num_admin_replies = 0;
 423         u64 reply_dma = 0;
 424         struct mpi3_default_reply_descriptor *reply_desc;
 425
 426         reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 427             admin_reply_ci;
 428
 429         if ((le16_to_cpu(reply_desc->reply_flags) &
 430             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 431                 return 0;
 432
 433         do {
 434                 if (mrioc->unrecoverable)
 435                         break;
 436
 437                 mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
 438                 mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
 439                 if (reply_dma)
 440                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 441                 num_admin_replies++;
 442                 if (++admin_reply_ci == mrioc->num_admin_replies) {
 443                         admin_reply_ci = 0;
 444                         exp_phase ^= 1;
 445                 }
 446                 reply_desc =
 447                     (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 448                     admin_reply_ci;
 449                 if ((le16_to_cpu(reply_desc->reply_flags) &
 450                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 451                         break;
 452         } while (1);
 453
 454         writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
 455         mrioc->admin_reply_ci = admin_reply_ci;
 456         mrioc->admin_reply_ephase = exp_phase;
 457
 458         return num_admin_replies;
 459 }
 460
 461 /**
 462  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
 463  *      queue's consumer index from operational reply descriptor queue.
 464  * @op_reply_q: op_reply_qinfo object
 465  * @reply_ci: operational reply descriptor's queue consumer index
 466  *
 467  * Returns reply descriptor frame address
 468  */
 469 static inline struct mpi3_default_reply_descriptor *
 470 mpi3mr_get_reply_desc(struct op_reply_qinfo *op_reply_q, u32 reply_ci)
 471 {
 472         void *segment_base_addr;
 473         struct segments *segments = op_reply_q->q_segments;
 474         struct mpi3_default_reply_descriptor *reply_desc = NULL;
 475
 476         segment_base_addr =
 477             segments[reply_ci / op_reply_q->segment_qd].segment;
 478         reply_desc = (struct mpi3_default_reply_descriptor *)segment_base_addr +
 479             (reply_ci % op_reply_q->segment_qd);
 480         return reply_desc;
 481 }
 482
 483 /**
 484  * mpi3mr_process_op_reply_q - Operational reply queue handler
 485  * @mrioc: Adapter instance reference
 486  * @op_reply_q: Operational reply queue info
 487  *
 488  * Checks the specific operational reply queue and drains the
 489  * reply queue entries until the queue is empty and process the
 490  * individual reply descriptors.
 491  *
 492  * Return: 0 if queue is already processed,or number of reply
 493  *          descriptors processed.
 494  */
 495 int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 496         struct op_reply_qinfo *op_reply_q)
 497 {
 498         struct op_req_qinfo *op_req_q;
 499         u32 exp_phase;
 500         u32 reply_ci;
 501         u32 num_op_reply = 0;
 502         u64 reply_dma = 0;
 503         struct mpi3_default_reply_descriptor *reply_desc;
 504         u16 req_q_idx = 0, reply_qidx;
 505
 506         reply_qidx = op_reply_q->qid - 1;
 507
 508         if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))
 509                 return 0;
 510
 511         exp_phase = op_reply_q->ephase;
 512         reply_ci = op_reply_q->ci;
 513
 514         reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 515         if ((le16_to_cpu(reply_desc->reply_flags) &
 516             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
 517                 atomic_dec(&op_reply_q->in_use);
 518                 return 0;
 519         }
 520
 521         do {
 522                 if (mrioc->unrecoverable)
 523                         break;
 524
 525                 req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
 526                 op_req_q = &mrioc->req_qinfo[req_q_idx];
 527
 528                 WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
 529                 mpi3mr_process_op_reply_desc(mrioc, reply_desc, &reply_dma,
 530                     reply_qidx);
 531                 atomic_dec(&op_reply_q->pend_ios);
 532                 if (reply_dma)
 533                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 534                 num_op_reply++;
 535
 536                 if (++reply_ci == op_reply_q->num_replies) {
 537                         reply_ci = 0;
 538                         exp_phase ^= 1;
 539                 }
 540
 541                 reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 542
 543                 if ((le16_to_cpu(reply_desc->reply_flags) &
 544                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 545                         break;
 546 #ifndef CONFIG_PREEMPT_RT
 547                 /*
 548                  * Exit completion loop to avoid CPU lockup
 549                  * Ensure remaining completion happens from threaded ISR.
 550                  */
 551                 if (num_op_reply > mrioc->max_host_ios) {
 552                         op_reply_q->enable_irq_poll = true;
 553                         break;
 554                 }
 555 #endif
 556         } while (1);
 557
 558         writel(reply_ci,
 559             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
 560         op_reply_q->ci = reply_ci;
 561         op_reply_q->ephase = exp_phase;
 562
 563         atomic_dec(&op_reply_q->in_use);
 564         return num_op_reply;
 565 }
 566
 567 /**
 568  * mpi3mr_blk_mq_poll - Operational reply queue handler
 569  * @shost: SCSI Host reference
 570  * @queue_num: Request queue number (w.r.t OS it is hardware context number)
 571  *
 572  * Checks the specific operational reply queue and drains the
 573  * reply queue entries until the queue is empty and process the
 574  * individual reply descriptors.
 575  *
 576  * Return: 0 if queue is already processed,or number of reply
 577  *          descriptors processed.
 578  */
 579 int mpi3mr_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
 580 {
 581         int num_entries = 0;
 582         struct mpi3mr_ioc *mrioc;
 583
 584         mrioc = (struct mpi3mr_ioc *)shost->hostdata;
 585
 586         if ((mrioc->reset_in_progress || mrioc->prepare_for_reset ||
 587             mrioc->unrecoverable))
 588                 return 0;
 589
 590         num_entries = mpi3mr_process_op_reply_q(mrioc,
 591                         &mrioc->op_reply_qinfo[queue_num]);
 592
 593         return num_entries;
 594 }
 595
 596 static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 597 {
 598         struct mpi3mr_intr_info *intr_info = privdata;
 599         struct mpi3mr_ioc *mrioc;
 600         u16 midx;
 601         u32 num_admin_replies = 0, num_op_reply = 0;
 602
 603         if (!intr_info)
 604                 return IRQ_NONE;
 605
 606         mrioc = intr_info->mrioc;
 607
 608         if (!mrioc->intr_enabled)
 609                 return IRQ_NONE;
 610
 611         midx = intr_info->msix_index;
 612
 613         if (!midx)
 614                 num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
 615         if (intr_info->op_reply_q)
 616                 num_op_reply = mpi3mr_process_op_reply_q(mrioc,
 617                     intr_info->op_reply_q);
 618
 619         if (num_admin_replies || num_op_reply)
 620                 return IRQ_HANDLED;
 621         else
 622                 return IRQ_NONE;
 623 }
 624
 625 #ifndef CONFIG_PREEMPT_RT
 626
 627 static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 628 {
 629         struct mpi3mr_intr_info *intr_info = privdata;
 630         int ret;
 631
 632         if (!intr_info)
 633                 return IRQ_NONE;
 634
 635         /* Call primary ISR routine */
 636         ret = mpi3mr_isr_primary(irq, privdata);
 637
 638         /*
 639          * If more IOs are expected, schedule IRQ polling thread.
 640          * Otherwise exit from ISR.
 641          */
 642         if (!intr_info->op_reply_q)
 643                 return ret;
 644
 645         if (!intr_info->op_reply_q->enable_irq_poll ||
 646             !atomic_read(&intr_info->op_reply_q->pend_ios))
 647                 return ret;
 648
 649         disable_irq_nosync(intr_info->os_irq);
 650
 651         return IRQ_WAKE_THREAD;
 652 }
 653
 654 /**
 655  * mpi3mr_isr_poll - Reply queue polling routine
 656  * @irq: IRQ
 657  * @privdata: Interrupt info
 658  *
 659  * poll for pending I/O completions in a loop until pending I/Os
 660  * present or controller queue depth I/Os are processed.
 661  *
 662  * Return: IRQ_NONE or IRQ_HANDLED
 663  */
 664 static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 665 {
 666         struct mpi3mr_intr_info *intr_info = privdata;
 667         struct mpi3mr_ioc *mrioc;
 668         u16 midx;
 669         u32 num_op_reply = 0;
 670
 671         if (!intr_info || !intr_info->op_reply_q)
 672                 return IRQ_NONE;
 673
 674         mrioc = intr_info->mrioc;
 675         midx = intr_info->msix_index;
 676
 677         /* Poll for pending IOs completions */
 678         do {
 679                 if (!mrioc->intr_enabled || mrioc->unrecoverable)
 680                         break;
 681
 682                 if (!midx)
 683                         mpi3mr_process_admin_reply_q(mrioc);
 684                 if (intr_info->op_reply_q)
 685                         num_op_reply +=
 686                             mpi3mr_process_op_reply_q(mrioc,
 687                                 intr_info->op_reply_q);
 688
 689                 usleep_range(MPI3MR_IRQ_POLL_SLEEP, 10 * MPI3MR_IRQ_POLL_SLEEP);
 690
 691         } while (atomic_read(&intr_info->op_reply_q->pend_ios) &&
 692             (num_op_reply < mrioc->max_host_ios));
 693
 694         intr_info->op_reply_q->enable_irq_poll = false;
 695         enable_irq(intr_info->os_irq);
 696
 697         return IRQ_HANDLED;
 698 }
 699
 700 #endif
 701
 702 /**
 703  * mpi3mr_request_irq - Request IRQ and register ISR
 704  * @mrioc: Adapter instance reference
 705  * @index: IRQ vector index
 706  *
 707  * Request threaded ISR with primary ISR and secondary
 708  *
 709  * Return: 0 on success and non zero on failures.
 710  */
 711 static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
 712 {
 713         struct pci_dev *pdev = mrioc->pdev;
 714         struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
 715         int retval = 0;
 716
 717         intr_info->mrioc = mrioc;
 718         intr_info->msix_index = index;
 719         intr_info->op_reply_q = NULL;
 720
 721         snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",
 722             mrioc->driver_name, mrioc->id, index);
 723
 724 #ifndef CONFIG_PREEMPT_RT
 725         retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr,
 726             mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);
 727 #else
 728         retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr_primary,
 729             NULL, IRQF_SHARED, intr_info->name, intr_info);
 730 #endif
 731         if (retval) {
 732                 ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
 733                     intr_info->name, pci_irq_vector(pdev, index));
 734                 return retval;
 735         }
 736
 737         intr_info->os_irq = pci_irq_vector(pdev, index);
 738         return retval;
 739 }
 740
 741 static void mpi3mr_calc_poll_queues(struct mpi3mr_ioc *mrioc, u16 max_vectors)
 742 {
 743         if (!mrioc->requested_poll_qcount)
 744                 return;
 745
 746         /* Reserved for Admin and Default Queue */
 747         if (max_vectors > 2 &&
 748                 (mrioc->requested_poll_qcount < max_vectors - 2)) {
 749                 ioc_info(mrioc,
 750                     "enabled polled queues (%d) msix (%d)\n",
 751                     mrioc->requested_poll_qcount, max_vectors);
 752         } else {
 753                 ioc_info(mrioc,
 754                     "disabled polled queues (%d) msix (%d) because of no resources for default queue\n",
 755                     mrioc->requested_poll_qcount, max_vectors);
 756                 mrioc->requested_poll_qcount = 0;
 757         }
 758 }
 759
 760 /**
 761  * mpi3mr_setup_isr - Setup ISR for the controller
 762  * @mrioc: Adapter instance reference
 763  * @setup_one: Request one IRQ or more
 764  *
 765  * Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
 766  *
 767  * Return: 0 on success and non zero on failures.
 768  */
 769 static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
 770 {
 771         unsigned int irq_flags = PCI_IRQ_MSIX;
 772         int max_vectors, min_vec;
 773         int retval;
 774         int i;
 775         struct irq_affinity desc = { .pre_vectors =  1, .post_vectors = 1 };
 776
 777         if (mrioc->is_intr_info_set)
 778                 return 0;
 779
 780         mpi3mr_cleanup_isr(mrioc);
 781
 782         if (setup_one || reset_devices) {
 783                 max_vectors = 1;
 784                 retval = pci_alloc_irq_vectors(mrioc->pdev,
 785                     1, max_vectors, irq_flags);
 786                 if (retval < 0) {
 787                         ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
 788                             retval);
 789                         goto out_failed;
 790                 }
 791         } else {
 792                 max_vectors =
 793                     min_t(int, mrioc->cpu_count + 1 +
 794                         mrioc->requested_poll_qcount, mrioc->msix_count);
 795
 796                 mpi3mr_calc_poll_queues(mrioc, max_vectors);
 797
 798                 ioc_info(mrioc,
 799                     "MSI-X vectors supported: %d, no of cores: %d,",
 800                     mrioc->msix_count, mrioc->cpu_count);
 801                 ioc_info(mrioc,
 802                     "MSI-x vectors requested: %d poll_queues %d\n",
 803                     max_vectors, mrioc->requested_poll_qcount);
 804
 805                 desc.post_vectors = mrioc->requested_poll_qcount;
 806                 min_vec = desc.pre_vectors + desc.post_vectors;
 807                 irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
 808
 809                 retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,
 810                         min_vec, max_vectors, irq_flags, &desc);
 811
 812                 if (retval < 0) {
 813                         ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
 814                             retval);
 815                         goto out_failed;
 816                 }
 817
 818
 819                 /*
 820                  * If only one MSI-x is allocated, then MSI-x 0 will be shared
 821                  * between Admin queue and operational queue
 822                  */
 823                 if (retval == min_vec)
 824                         mrioc->op_reply_q_offset = 0;
 825                 else if (retval != (max_vectors)) {
 826                         ioc_info(mrioc,
 827                             "allocated vectors (%d) are less than configured (%d)\n",
 828                             retval, max_vectors);
 829                 }
 830
 831                 max_vectors = retval;
 832                 mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
 833
 834                 mpi3mr_calc_poll_queues(mrioc, max_vectors);
 835
 836         }
 837
 838         mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
 839             GFP_KERNEL);
 840         if (!mrioc->intr_info) {
 841                 retval = -ENOMEM;
 842                 pci_free_irq_vectors(mrioc->pdev);
 843                 goto out_failed;
 844         }
 845         for (i = 0; i < max_vectors; i++) {
 846                 retval = mpi3mr_request_irq(mrioc, i);
 847                 if (retval) {
 848                         mrioc->intr_info_count = i;
 849                         goto out_failed;
 850                 }
 851         }
 852         if (reset_devices || !setup_one)
 853                 mrioc->is_intr_info_set = true;
 854         mrioc->intr_info_count = max_vectors;
 855         mpi3mr_ioc_enable_intr(mrioc);
 856         return 0;
 857
 858 out_failed:
 859         mpi3mr_cleanup_isr(mrioc);
 860
 861         return retval;
 862 }
 863
 864 static const struct {
 865         enum mpi3mr_iocstate value;
 866         char *name;
 867 } mrioc_states[] = {
 868         { MRIOC_STATE_READY, "ready" },
 869         { MRIOC_STATE_FAULT, "fault" },
 870         { MRIOC_STATE_RESET, "reset" },
 871         { MRIOC_STATE_BECOMING_READY, "becoming ready" },
 872         { MRIOC_STATE_RESET_REQUESTED, "reset requested" },
 873         { MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
 874 };
 875
 876 static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
 877 {
 878         int i;
 879         char *name = NULL;
 880
 881         for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
 882                 if (mrioc_states[i].value == mrioc_state) {
 883                         name = mrioc_states[i].name;
 884                         break;
 885                 }
 886         }
 887         return name;
 888 }
 889
 890 /* Reset reason to name mapper structure*/
 891 static const struct {
 892         enum mpi3mr_reset_reason value;
 893         char *name;
 894 } mpi3mr_reset_reason_codes[] = {
 895         { MPI3MR_RESET_FROM_BRINGUP, "timeout in bringup" },
 896         { MPI3MR_RESET_FROM_FAULT_WATCH, "fault" },
 897         { MPI3MR_RESET_FROM_APP, "application invocation" },
 898         { MPI3MR_RESET_FROM_EH_HOS, "error handling" },
 899         { MPI3MR_RESET_FROM_TM_TIMEOUT, "TM timeout" },
 900         { MPI3MR_RESET_FROM_APP_TIMEOUT, "application command timeout" },
 901         { MPI3MR_RESET_FROM_MUR_FAILURE, "MUR failure" },
 902         { MPI3MR_RESET_FROM_CTLR_CLEANUP, "timeout in controller cleanup" },
 903         { MPI3MR_RESET_FROM_CIACTIV_FAULT, "component image activation fault" },
 904         { MPI3MR_RESET_FROM_PE_TIMEOUT, "port enable timeout" },
 905         { MPI3MR_RESET_FROM_TSU_TIMEOUT, "time stamp update timeout" },
 906         { MPI3MR_RESET_FROM_DELREQQ_TIMEOUT, "delete request queue timeout" },
 907         { MPI3MR_RESET_FROM_DELREPQ_TIMEOUT, "delete reply queue timeout" },
 908         {
 909                 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT,
 910                 "create request queue timeout"
 911         },
 912         {
 913                 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT,
 914                 "create reply queue timeout"
 915         },
 916         { MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT, "IOC facts timeout" },
 917         { MPI3MR_RESET_FROM_IOCINIT_TIMEOUT, "IOC init timeout" },
 918         { MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT, "event notify timeout" },
 919         { MPI3MR_RESET_FROM_EVTACK_TIMEOUT, "event acknowledgment timeout" },
 920         {
 921                 MPI3MR_RESET_FROM_CIACTVRST_TIMER,
 922                 "component image activation timeout"
 923         },
 924         {
 925                 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT,
 926                 "get package version timeout"
 927         },
 928         { MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" },
 929         { MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" },
 930         { MPI3MR_RESET_FROM_FIRMWARE, "firmware asynchronous reset" },
 931         { MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT, "configuration request timeout"},
 932         { MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT, "timeout of a SAS transport layer request" },
 933 };
 934
 935 /**
 936  * mpi3mr_reset_rc_name - get reset reason code name
 937  * @reason_code: reset reason code value
 938  *
 939  * Map reset reason to an NULL terminated ASCII string
 940  *
 941  * Return: name corresponding to reset reason value or NULL.
 942  */
 943 static const char *mpi3mr_reset_rc_name(enum mpi3mr_reset_reason reason_code)
 944 {
 945         int i;
 946         char *name = NULL;
 947
 948         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_reason_codes); i++) {
 949                 if (mpi3mr_reset_reason_codes[i].value == reason_code) {
 950                         name = mpi3mr_reset_reason_codes[i].name;
 951                         break;
 952                 }
 953         }
 954         return name;
 955 }
 956
 957 /* Reset type to name mapper structure*/
 958 static const struct {
 959         u16 reset_type;
 960         char *name;
 961 } mpi3mr_reset_types[] = {
 962         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, "soft" },
 963         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
 964 };
 965
 966 /**
 967  * mpi3mr_reset_type_name - get reset type name
 968  * @reset_type: reset type value
 969  *
 970  * Map reset type to an NULL terminated ASCII string
 971  *
 972  * Return: name corresponding to reset type value or NULL.
 973  */
 974 static const char *mpi3mr_reset_type_name(u16 reset_type)
 975 {
 976         int i;
 977         char *name = NULL;
 978
 979         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_types); i++) {
 980                 if (mpi3mr_reset_types[i].reset_type == reset_type) {
 981                         name = mpi3mr_reset_types[i].name;
 982                         break;
 983                 }
 984         }
 985         return name;
 986 }
 987
 988 /**
 989  * mpi3mr_print_fault_info - Display fault information
 990  * @mrioc: Adapter instance reference
 991  *
 992  * Display the controller fault information if there is a
 993  * controller fault.
 994  *
 995  * Return: Nothing.
 996  */
 997 void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
 998 {
 999         u32 ioc_status, code, code1, code2, code3;
1000
1001         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1002
1003         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
1004                 code = readl(&mrioc->sysif_regs->fault);
1005                 code1 = readl(&mrioc->sysif_regs->fault_info[0]);
1006                 code2 = readl(&mrioc->sysif_regs->fault_info[1]);
1007                 code3 = readl(&mrioc->sysif_regs->fault_info[2]);
1008
1009                 ioc_info(mrioc,
1010                     "fault code(0x%08X): Additional code: (0x%08X:0x%08X:0x%08X)\n",
1011                     code, code1, code2, code3);
1012         }
1013 }
1014
1015 /**
1016  * mpi3mr_get_iocstate - Get IOC State
1017  * @mrioc: Adapter instance reference
1018  *
1019  * Return a proper IOC state enum based on the IOC status and
1020  * IOC configuration and unrcoverable state of the controller.
1021  *
1022  * Return: Current IOC state.
1023  */
1024 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
1025 {
1026         u32 ioc_status, ioc_config;
1027         u8 ready, enabled;
1028
1029         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1030         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1031
1032         if (mrioc->unrecoverable)
1033                 return MRIOC_STATE_UNRECOVERABLE;
1034         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)
1035                 return MRIOC_STATE_FAULT;
1036
1037         ready = (ioc_status & MPI3_SYSIF_IOC_STATUS_READY);
1038         enabled = (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC);
1039
1040         if (ready && enabled)
1041                 return MRIOC_STATE_READY;
1042         if ((!ready) && (!enabled))
1043                 return MRIOC_STATE_RESET;
1044         if ((!ready) && (enabled))
1045                 return MRIOC_STATE_BECOMING_READY;
1046
1047         return MRIOC_STATE_RESET_REQUESTED;
1048 }
1049
1050 /**
1051  * mpi3mr_clear_reset_history - clear reset history
1052  * @mrioc: Adapter instance reference
1053  *
1054  * Write the reset history bit in IOC status to clear the bit,
1055  * if it is already set.
1056  *
1057  * Return: Nothing.
1058  */
1059 static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
1060 {
1061         u32 ioc_status;
1062
1063         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1064         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1065                 writel(ioc_status, &mrioc->sysif_regs->ioc_status);
1066 }
1067
1068 /**
1069  * mpi3mr_issue_and_process_mur - Message unit Reset handler
1070  * @mrioc: Adapter instance reference
1071  * @reset_reason: Reset reason code
1072  *
1073  * Issue Message unit Reset to the controller and wait for it to
1074  * be complete.
1075  *
1076  * Return: 0 on success, -1 on failure.
1077  */
1078 static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
1079         u32 reset_reason)
1080 {
1081         u32 ioc_config, timeout, ioc_status;
1082         int retval = -1;
1083
1084         ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
1085         if (mrioc->unrecoverable) {
1086                 ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
1087                 return retval;
1088         }
1089         mpi3mr_clear_reset_history(mrioc);
1090         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1091         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1092         ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1093         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1094
1095         timeout = MPI3MR_RESET_ACK_TIMEOUT * 10;
1096         do {
1097                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1098                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
1099                         mpi3mr_clear_reset_history(mrioc);
1100                         break;
1101                 }
1102                 if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
1103                         mpi3mr_print_fault_info(mrioc);
1104                         break;
1105                 }
1106                 msleep(100);
1107         } while (--timeout);
1108
1109         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1110         if (timeout && !((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1111               (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
1112               (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1113                 retval = 0;
1114
1115         ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
1116             (!retval) ? "successful" : "failed", ioc_status, ioc_config);
1117         return retval;
1118 }
1119
1120 /**
1121  * mpi3mr_revalidate_factsdata - validate IOCFacts parameters
1122  * during reset/resume
1123  * @mrioc: Adapter instance reference
1124  *
1125  * Return zero if the new IOCFacts parameters value is compatible with
1126  * older values else return -EPERM
1127  */
1128 static int
1129 mpi3mr_revalidate_factsdata(struct mpi3mr_ioc *mrioc)
1130 {
1131         void *removepend_bitmap;
1132
1133         if (mrioc->facts.reply_sz > mrioc->reply_sz) {
1134                 ioc_err(mrioc,
1135                     "cannot increase reply size from %d to %d\n",
1136                     mrioc->reply_sz, mrioc->facts.reply_sz);
1137                 return -EPERM;
1138         }
1139
1140         if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {
1141                 ioc_err(mrioc,
1142                     "cannot reduce number of operational reply queues from %d to %d\n",
1143                     mrioc->num_op_reply_q,
1144                     mrioc->facts.max_op_reply_q);
1145                 return -EPERM;
1146         }
1147
1148         if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {
1149                 ioc_err(mrioc,
1150                     "cannot reduce number of operational request queues from %d to %d\n",
1151                     mrioc->num_op_req_q, mrioc->facts.max_op_req_q);
1152                 return -EPERM;
1153         }
1154
1155         if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities &
1156             MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED))
1157                 ioc_err(mrioc,
1158                     "critical error: multipath capability is enabled at the\n"
1159                     "\tcontroller while sas transport support is enabled at the\n"
1160                     "\tdriver, please reboot the system or reload the driver\n");
1161
1162         if (mrioc->facts.max_devhandle > mrioc->dev_handle_bitmap_bits) {
1163                 removepend_bitmap = bitmap_zalloc(mrioc->facts.max_devhandle,
1164                                                   GFP_KERNEL);
1165                 if (!removepend_bitmap) {
1166                         ioc_err(mrioc,
1167                                 "failed to increase removepend_bitmap bits from %d to %d\n",
1168                                 mrioc->dev_handle_bitmap_bits,
1169                                 mrioc->facts.max_devhandle);
1170                         return -EPERM;
1171                 }
1172                 bitmap_free(mrioc->removepend_bitmap);
1173                 mrioc->removepend_bitmap = removepend_bitmap;
1174                 ioc_info(mrioc,
1175                          "increased bits of dev_handle_bitmap from %d to %d\n",
1176                          mrioc->dev_handle_bitmap_bits,
1177                          mrioc->facts.max_devhandle);
1178                 mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
1179         }
1180
1181         return 0;
1182 }
1183
1184 /**
1185  * mpi3mr_bring_ioc_ready - Bring controller to ready state
1186  * @mrioc: Adapter instance reference
1187  *
1188  * Set Enable IOC bit in IOC configuration register and wait for
1189  * the controller to become ready.
1190  *
1191  * Return: 0 on success, appropriate error on failure.
1192  */
1193 static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
1194 {
1195         u32 ioc_config, ioc_status, timeout;
1196         int retval = 0;
1197         enum mpi3mr_iocstate ioc_state;
1198         u64 base_info;
1199
1200         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1201         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1202         base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);
1203         ioc_info(mrioc, "ioc_status(0x%08x), ioc_config(0x%08x), ioc_info(0x%016llx) at the bringup\n",
1204             ioc_status, ioc_config, base_info);
1205
1206         /*The timeout value is in 2sec unit, changing it to seconds*/
1207         mrioc->ready_timeout =
1208             ((base_info & MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_MASK) >>
1209             MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_SHIFT) * 2;
1210
1211         ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);
1212
1213         ioc_state = mpi3mr_get_iocstate(mrioc);
1214         ioc_info(mrioc, "controller is in %s state during detection\n",
1215             mpi3mr_iocstate_name(ioc_state));
1216
1217         if (ioc_state == MRIOC_STATE_BECOMING_READY ||
1218             ioc_state == MRIOC_STATE_RESET_REQUESTED) {
1219                 timeout = mrioc->ready_timeout * 10;
1220                 do {
1221                         msleep(100);
1222                 } while (--timeout);
1223
1224                 if (!pci_device_is_present(mrioc->pdev)) {
1225                         mrioc->unrecoverable = 1;
1226                         ioc_err(mrioc,
1227                             "controller is not present while waiting to reset\n");
1228                         retval = -1;
1229                         goto out_device_not_present;
1230                 }
1231
1232                 ioc_state = mpi3mr_get_iocstate(mrioc);
1233                 ioc_info(mrioc,
1234                     "controller is in %s state after waiting to reset\n",
1235                     mpi3mr_iocstate_name(ioc_state));
1236         }
1237
1238         if (ioc_state == MRIOC_STATE_READY) {
1239                 ioc_info(mrioc, "issuing message unit reset (MUR) to bring to reset state\n");
1240                 retval = mpi3mr_issue_and_process_mur(mrioc,
1241                     MPI3MR_RESET_FROM_BRINGUP);
1242                 ioc_state = mpi3mr_get_iocstate(mrioc);
1243                 if (retval)
1244                         ioc_err(mrioc,
1245                             "message unit reset failed with error %d current state %s\n",
1246                             retval, mpi3mr_iocstate_name(ioc_state));
1247         }
1248         if (ioc_state != MRIOC_STATE_RESET) {
1249                 mpi3mr_print_fault_info(mrioc);
1250                 ioc_info(mrioc, "issuing soft reset to bring to reset state\n");
1251                 retval = mpi3mr_issue_reset(mrioc,
1252                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
1253                     MPI3MR_RESET_FROM_BRINGUP);
1254                 if (retval) {
1255                         ioc_err(mrioc,
1256                             "soft reset failed with error %d\n", retval);
1257                         goto out_failed;
1258                 }
1259         }
1260         ioc_state = mpi3mr_get_iocstate(mrioc);
1261         if (ioc_state != MRIOC_STATE_RESET) {
1262                 ioc_err(mrioc,
1263                     "cannot bring controller to reset state, current state: %s\n",
1264                     mpi3mr_iocstate_name(ioc_state));
1265                 goto out_failed;
1266         }
1267         mpi3mr_clear_reset_history(mrioc);
1268         retval = mpi3mr_setup_admin_qpair(mrioc);
1269         if (retval) {
1270                 ioc_err(mrioc, "failed to setup admin queues: error %d\n",
1271                     retval);
1272                 goto out_failed;
1273         }
1274
1275         ioc_info(mrioc, "bringing controller to ready state\n");
1276         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1277         ioc_config |= MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1278         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1279
1280         timeout = mrioc->ready_timeout * 10;
1281         do {
1282                 ioc_state = mpi3mr_get_iocstate(mrioc);
1283                 if (ioc_state == MRIOC_STATE_READY) {
1284                         ioc_info(mrioc,
1285                             "successfully transitioned to %s state\n",
1286                             mpi3mr_iocstate_name(ioc_state));
1287                         return 0;
1288                 }
1289                 if (!pci_device_is_present(mrioc->pdev)) {
1290                         mrioc->unrecoverable = 1;
1291                         ioc_err(mrioc,
1292                             "controller is not present at the bringup\n");
1293                         retval = -1;
1294                         goto out_device_not_present;
1295                 }
1296                 msleep(100);
1297         } while (--timeout);
1298
1299 out_failed:
1300         ioc_state = mpi3mr_get_iocstate(mrioc);
1301         ioc_err(mrioc,
1302             "failed to bring to ready state,  current state: %s\n",
1303             mpi3mr_iocstate_name(ioc_state));
1304 out_device_not_present:
1305         return retval;
1306 }
1307
1308 /**
1309  * mpi3mr_soft_reset_success - Check softreset is success or not
1310  * @ioc_status: IOC status register value
1311  * @ioc_config: IOC config register value
1312  *
1313  * Check whether the soft reset is successful or not based on
1314  * IOC status and IOC config register values.
1315  *
1316  * Return: True when the soft reset is success, false otherwise.
1317  */
1318 static inline bool
1319 mpi3mr_soft_reset_success(u32 ioc_status, u32 ioc_config)
1320 {
1321         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1322             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1323                 return true;
1324         return false;
1325 }
1326
1327 /**
1328  * mpi3mr_diagfault_success - Check diag fault is success or not
1329  * @mrioc: Adapter reference
1330  * @ioc_status: IOC status register value
1331  *
1332  * Check whether the controller hit diag reset fault code.
1333  *
1334  * Return: True when there is diag fault, false otherwise.
1335  */
1336 static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1337         u32 ioc_status)
1338 {
1339         u32 fault;
1340
1341         if (!(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT))
1342                 return false;
1343         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1344         if (fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) {
1345                 mpi3mr_print_fault_info(mrioc);
1346                 return true;
1347         }
1348         return false;
1349 }
1350
1351 /**
1352  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1353  * @mrioc: Adapter reference
1354  *
1355  * Set diag save bit in IOC configuration register to enable
1356  * snapdump.
1357  *
1358  * Return: Nothing.
1359  */
1360 static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1361 {
1362         u32 ioc_config;
1363
1364         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1365         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DIAG_SAVE;
1366         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1367 }
1368
1369 /**
1370  * mpi3mr_issue_reset - Issue reset to the controller
1371  * @mrioc: Adapter reference
1372  * @reset_type: Reset type
1373  * @reset_reason: Reset reason code
1374  *
1375  * Unlock the host diagnostic registers and write the specific
1376  * reset type to that, wait for reset acknowledgment from the
1377  * controller, if the reset is not successful retry for the
1378  * predefined number of times.
1379  *
1380  * Return: 0 on success, non-zero on failure.
1381  */
1382 static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1383         u32 reset_reason)
1384 {
1385         int retval = -1;
1386         u8 unlock_retry_count = 0;
1387         u32 host_diagnostic, ioc_status, ioc_config;
1388         u32 timeout = MPI3MR_RESET_ACK_TIMEOUT * 10;
1389
1390         if ((reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) &&
1391             (reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT))
1392                 return retval;
1393         if (mrioc->unrecoverable)
1394                 return retval;
1395         if (reset_reason == MPI3MR_RESET_FROM_FIRMWARE) {
1396                 retval = 0;
1397                 return retval;
1398         }
1399
1400         ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1401             mpi3mr_reset_type_name(reset_type),
1402             mpi3mr_reset_rc_name(reset_reason), reset_reason);
1403
1404         mpi3mr_clear_reset_history(mrioc);
1405         do {
1406                 ioc_info(mrioc,
1407                     "Write magic sequence to unlock host diag register (retry=%d)\n",
1408                     ++unlock_retry_count);
1409                 if (unlock_retry_count >= MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT) {
1410                         ioc_err(mrioc,
1411                             "%s reset failed due to unlock failure, host_diagnostic(0x%08x)\n",
1412                             mpi3mr_reset_type_name(reset_type),
1413                             host_diagnostic);
1414                         mrioc->unrecoverable = 1;
1415                         return retval;
1416                 }
1417
1418                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_FLUSH,
1419                     &mrioc->sysif_regs->write_sequence);
1420                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_1ST,
1421                     &mrioc->sysif_regs->write_sequence);
1422                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1423                     &mrioc->sysif_regs->write_sequence);
1424                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_3RD,
1425                     &mrioc->sysif_regs->write_sequence);
1426                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_4TH,
1427                     &mrioc->sysif_regs->write_sequence);
1428                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_5TH,
1429                     &mrioc->sysif_regs->write_sequence);
1430                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_6TH,
1431                     &mrioc->sysif_regs->write_sequence);
1432                 usleep_range(1000, 1100);
1433                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
1434                 ioc_info(mrioc,
1435                     "wrote magic sequence: retry_count(%d), host_diagnostic(0x%08x)\n",
1436                     unlock_retry_count, host_diagnostic);
1437         } while (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_DIAG_WRITE_ENABLE));
1438
1439         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1440         writel(host_diagnostic | reset_type,
1441             &mrioc->sysif_regs->host_diagnostic);
1442         switch (reset_type) {
1443         case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET:
1444                 do {
1445                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1446                         ioc_config =
1447                             readl(&mrioc->sysif_regs->ioc_configuration);
1448                         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1449                             && mpi3mr_soft_reset_success(ioc_status, ioc_config)
1450                             ) {
1451                                 mpi3mr_clear_reset_history(mrioc);
1452                                 retval = 0;
1453                                 break;
1454                         }
1455                         msleep(100);
1456                 } while (--timeout);
1457                 mpi3mr_print_fault_info(mrioc);
1458                 break;
1459         case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT:
1460                 do {
1461                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1462                         if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1463                                 retval = 0;
1464                                 break;
1465                         }
1466                         msleep(100);
1467                 } while (--timeout);
1468                 break;
1469         default:
1470                 break;
1471         }
1472
1473         writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1474             &mrioc->sysif_regs->write_sequence);
1475
1476         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1477         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1478         ioc_info(mrioc,
1479             "ioc_status/ioc_onfig after %s reset is (0x%x)/(0x%x)\n",
1480             (!retval)?"successful":"failed", ioc_status,
1481             ioc_config);
1482         if (retval)
1483                 mrioc->unrecoverable = 1;
1484         return retval;
1485 }
1486
1487 /**
1488  * mpi3mr_admin_request_post - Post request to admin queue
1489  * @mrioc: Adapter reference
1490  * @admin_req: MPI3 request
1491  * @admin_req_sz: Request size
1492  * @ignore_reset: Ignore reset in process
1493  *
1494  * Post the MPI3 request into admin request queue and
1495  * inform the controller, if the queue is full return
1496  * appropriate error.
1497  *
1498  * Return: 0 on success, non-zero on failure.
1499  */
1500 int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1501         u16 admin_req_sz, u8 ignore_reset)
1502 {
1503         u16 areq_pi = 0, areq_ci = 0, max_entries = 0;
1504         int retval = 0;
1505         unsigned long flags;
1506         u8 *areq_entry;
1507
1508         if (mrioc->unrecoverable) {
1509                 ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1510                 return -EFAULT;
1511         }
1512
1513         spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1514         areq_pi = mrioc->admin_req_pi;
1515         areq_ci = mrioc->admin_req_ci;
1516         max_entries = mrioc->num_admin_req;
1517         if ((areq_ci == (areq_pi + 1)) || ((!areq_ci) &&
1518             (areq_pi == (max_entries - 1)))) {
1519                 ioc_err(mrioc, "AdminReqQ full condition detected\n");
1520                 retval = -EAGAIN;
1521                 goto out;
1522         }
1523         if (!ignore_reset && mrioc->reset_in_progress) {
1524                 ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1525                 retval = -EAGAIN;
1526                 goto out;
1527         }
1528         areq_entry = (u8 *)mrioc->admin_req_base +
1529             (areq_pi * MPI3MR_ADMIN_REQ_FRAME_SZ);
1530         memset(areq_entry, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
1531         memcpy(areq_entry, (u8 *)admin_req, admin_req_sz);
1532
1533         if (++areq_pi == max_entries)
1534                 areq_pi = 0;
1535         mrioc->admin_req_pi = areq_pi;
1536
1537         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
1538
1539 out:
1540         spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);
1541
1542         return retval;
1543 }
1544
1545 /**
1546  * mpi3mr_free_op_req_q_segments - free request memory segments
1547  * @mrioc: Adapter instance reference
1548  * @q_idx: operational request queue index
1549  *
1550  * Free memory segments allocated for operational request queue
1551  *
1552  * Return: Nothing.
1553  */
1554 static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1555 {
1556         u16 j;
1557         int size;
1558         struct segments *segments;
1559
1560         segments = mrioc->req_qinfo[q_idx].q_segments;
1561         if (!segments)
1562                 return;
1563
1564         if (mrioc->enable_segqueue) {
1565                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1566                 if (mrioc->req_qinfo[q_idx].q_segment_list) {
1567                         dma_free_coherent(&mrioc->pdev->dev,
1568                             MPI3MR_MAX_SEG_LIST_SIZE,
1569                             mrioc->req_qinfo[q_idx].q_segment_list,
1570                             mrioc->req_qinfo[q_idx].q_segment_list_dma);
1571                         mrioc->req_qinfo[q_idx].q_segment_list = NULL;
1572                 }
1573         } else
1574                 size = mrioc->req_qinfo[q_idx].segment_qd *
1575                     mrioc->facts.op_req_sz;
1576
1577         for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1578                 if (!segments[j].segment)
1579                         continue;
1580                 dma_free_coherent(&mrioc->pdev->dev,
1581                     size, segments[j].segment, segments[j].segment_dma);
1582                 segments[j].segment = NULL;
1583         }
1584         kfree(mrioc->req_qinfo[q_idx].q_segments);
1585         mrioc->req_qinfo[q_idx].q_segments = NULL;
1586         mrioc->req_qinfo[q_idx].qid = 0;
1587 }
1588
1589 /**
1590  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1591  * @mrioc: Adapter instance reference
1592  * @q_idx: operational reply queue index
1593  *
1594  * Free memory segments allocated for operational reply queue
1595  *
1596  * Return: Nothing.
1597  */
1598 static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1599 {
1600         u16 j;
1601         int size;
1602         struct segments *segments;
1603
1604         segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1605         if (!segments)
1606                 return;
1607
1608         if (mrioc->enable_segqueue) {
1609                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1610                 if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1611                         dma_free_coherent(&mrioc->pdev->dev,
1612                             MPI3MR_MAX_SEG_LIST_SIZE,
1613                             mrioc->op_reply_qinfo[q_idx].q_segment_list,
1614                             mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1615                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1616                 }
1617         } else
1618                 size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1619                     mrioc->op_reply_desc_sz;
1620
1621         for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1622                 if (!segments[j].segment)
1623                         continue;
1624                 dma_free_coherent(&mrioc->pdev->dev,
1625                     size, segments[j].segment, segments[j].segment_dma);
1626                 segments[j].segment = NULL;
1627         }
1628
1629         kfree(mrioc->op_reply_qinfo[q_idx].q_segments);
1630         mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1631         mrioc->op_reply_qinfo[q_idx].qid = 0;
1632 }
1633
1634 /**
1635  * mpi3mr_delete_op_reply_q - delete operational reply queue
1636  * @mrioc: Adapter instance reference
1637  * @qidx: operational reply queue index
1638  *
1639  * Delete operatinal reply queue by issuing MPI request
1640  * through admin queue.
1641  *
1642  * Return:  0 on success, non-zero on failure.
1643  */
1644 static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1645 {
1646         struct mpi3_delete_reply_queue_request delq_req;
1647         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1648         int retval = 0;
1649         u16 reply_qid = 0, midx;
1650
1651         reply_qid = op_reply_q->qid;
1652
1653         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1654
1655         if (!reply_qid) {
1656                 retval = -1;
1657                 ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1658                 goto out;
1659         }
1660
1661         (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :
1662             mrioc->active_poll_qcount--;
1663
1664         memset(&delq_req, 0, sizeof(delq_req));
1665         mutex_lock(&mrioc->init_cmds.mutex);
1666         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1667                 retval = -1;
1668                 ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1669                 mutex_unlock(&mrioc->init_cmds.mutex);
1670                 goto out;
1671         }
1672         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1673         mrioc->init_cmds.is_waiting = 1;
1674         mrioc->init_cmds.callback = NULL;
1675         delq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1676         delq_req.function = MPI3_FUNCTION_DELETE_REPLY_QUEUE;
1677         delq_req.queue_id = cpu_to_le16(reply_qid);
1678
1679         init_completion(&mrioc->init_cmds.done);
1680         retval = mpi3mr_admin_request_post(mrioc, &delq_req, sizeof(delq_req),
1681             1);
1682         if (retval) {
1683                 ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1684                 goto out_unlock;
1685         }
1686         wait_for_completion_timeout(&mrioc->init_cmds.done,
1687             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1688         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1689                 ioc_err(mrioc, "delete reply queue timed out\n");
1690                 mpi3mr_check_rh_fault_ioc(mrioc,
1691                     MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
1692                 retval = -1;
1693                 goto out_unlock;
1694         }
1695         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1696             != MPI3_IOCSTATUS_SUCCESS) {
1697                 ioc_err(mrioc,
1698                     "Issue DelRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1699                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1700                     mrioc->init_cmds.ioc_loginfo);
1701                 retval = -1;
1702                 goto out_unlock;
1703         }
1704         mrioc->intr_info[midx].op_reply_q = NULL;
1705
1706         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1707 out_unlock:
1708         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1709         mutex_unlock(&mrioc->init_cmds.mutex);
1710 out:
1711
1712         return retval;
1713 }
1714
1715 /**
1716  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1717  * @mrioc: Adapter instance reference
1718  * @qidx: request queue index
1719  *
1720  * Allocate segmented memory pools for operational reply
1721  * queue.
1722  *
1723  * Return: 0 on success, non-zero on failure.
1724  */
1725 static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1726 {
1727         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1728         int i, size;
1729         u64 *q_segment_list_entry = NULL;
1730         struct segments *segments;
1731
1732         if (mrioc->enable_segqueue) {
1733                 op_reply_q->segment_qd =
1734                     MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1735
1736                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1737
1738                 op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1739                     MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,
1740                     GFP_KERNEL);
1741                 if (!op_reply_q->q_segment_list)
1742                         return -ENOMEM;
1743                 q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;
1744         } else {
1745                 op_reply_q->segment_qd = op_reply_q->num_replies;
1746                 size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1747         }
1748
1749         op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,
1750             op_reply_q->segment_qd);
1751
1752         op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,
1753             sizeof(struct segments), GFP_KERNEL);
1754         if (!op_reply_q->q_segments)
1755                 return -ENOMEM;
1756
1757         segments = op_reply_q->q_segments;
1758         for (i = 0; i < op_reply_q->num_segments; i++) {
1759                 segments[i].segment =
1760                     dma_alloc_coherent(&mrioc->pdev->dev,
1761                     size, &segments[i].segment_dma, GFP_KERNEL);
1762                 if (!segments[i].segment)
1763                         return -ENOMEM;
1764                 if (mrioc->enable_segqueue)
1765                         q_segment_list_entry[i] =
1766                             (unsigned long)segments[i].segment_dma;
1767         }
1768
1769         return 0;
1770 }
1771
1772 /**
1773  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1774  * @mrioc: Adapter instance reference
1775  * @qidx: request queue index
1776  *
1777  * Allocate segmented memory pools for operational request
1778  * queue.
1779  *
1780  * Return: 0 on success, non-zero on failure.
1781  */
1782 static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1783 {
1784         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1785         int i, size;
1786         u64 *q_segment_list_entry = NULL;
1787         struct segments *segments;
1788
1789         if (mrioc->enable_segqueue) {
1790                 op_req_q->segment_qd =
1791                     MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1792
1793                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1794
1795                 op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1796                     MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,
1797                     GFP_KERNEL);
1798                 if (!op_req_q->q_segment_list)
1799                         return -ENOMEM;
1800                 q_segment_list_entry = (u64 *)op_req_q->q_segment_list;
1801
1802         } else {
1803                 op_req_q->segment_qd = op_req_q->num_requests;
1804                 size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1805         }
1806
1807         op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,
1808             op_req_q->segment_qd);
1809
1810         op_req_q->q_segments = kcalloc(op_req_q->num_segments,
1811             sizeof(struct segments), GFP_KERNEL);
1812         if (!op_req_q->q_segments)
1813                 return -ENOMEM;
1814
1815         segments = op_req_q->q_segments;
1816         for (i = 0; i < op_req_q->num_segments; i++) {
1817                 segments[i].segment =
1818                     dma_alloc_coherent(&mrioc->pdev->dev,
1819                     size, &segments[i].segment_dma, GFP_KERNEL);
1820                 if (!segments[i].segment)
1821                         return -ENOMEM;
1822                 if (mrioc->enable_segqueue)
1823                         q_segment_list_entry[i] =
1824                             (unsigned long)segments[i].segment_dma;
1825         }
1826
1827         return 0;
1828 }
1829
1830 /**
1831  * mpi3mr_create_op_reply_q - create operational reply queue
1832  * @mrioc: Adapter instance reference
1833  * @qidx: operational reply queue index
1834  *
1835  * Create operatinal reply queue by issuing MPI request
1836  * through admin queue.
1837  *
1838  * Return:  0 on success, non-zero on failure.
1839  */
1840 static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1841 {
1842         struct mpi3_create_reply_queue_request create_req;
1843         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1844         int retval = 0;
1845         u16 reply_qid = 0, midx;
1846
1847         reply_qid = op_reply_q->qid;
1848
1849         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1850
1851         if (reply_qid) {
1852                 retval = -1;
1853                 ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1854                     reply_qid);
1855
1856                 return retval;
1857         }
1858
1859         reply_qid = qidx + 1;
1860         op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
1861         if (!mrioc->pdev->revision)
1862                 op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;
1863         op_reply_q->ci = 0;
1864         op_reply_q->ephase = 1;
1865         atomic_set(&op_reply_q->pend_ios, 0);
1866         atomic_set(&op_reply_q->in_use, 0);
1867         op_reply_q->enable_irq_poll = false;
1868
1869         if (!op_reply_q->q_segments) {
1870                 retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
1871                 if (retval) {
1872                         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1873                         goto out;
1874                 }
1875         }
1876
1877         memset(&create_req, 0, sizeof(create_req));
1878         mutex_lock(&mrioc->init_cmds.mutex);
1879         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1880                 retval = -1;
1881                 ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
1882                 goto out_unlock;
1883         }
1884         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1885         mrioc->init_cmds.is_waiting = 1;
1886         mrioc->init_cmds.callback = NULL;
1887         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1888         create_req.function = MPI3_FUNCTION_CREATE_REPLY_QUEUE;
1889         create_req.queue_id = cpu_to_le16(reply_qid);
1890
1891         if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))
1892                 op_reply_q->qtype = MPI3MR_DEFAULT_QUEUE;
1893         else
1894                 op_reply_q->qtype = MPI3MR_POLL_QUEUE;
1895
1896         if (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) {
1897                 create_req.flags =
1898                         MPI3_CREATE_REPLY_QUEUE_FLAGS_INT_ENABLE_ENABLE;
1899                 create_req.msix_index =
1900                         cpu_to_le16(mrioc->intr_info[midx].msix_index);
1901         } else {
1902                 create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);
1903                 ioc_info(mrioc, "create reply queue(polled): for qid(%d), midx(%d)\n",
1904                         reply_qid, midx);
1905                 if (!mrioc->active_poll_qcount)
1906                         disable_irq_nosync(pci_irq_vector(mrioc->pdev,
1907                             mrioc->intr_info_count - 1));
1908         }
1909
1910         if (mrioc->enable_segqueue) {
1911                 create_req.flags |=
1912                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1913                 create_req.base_address = cpu_to_le64(
1914                     op_reply_q->q_segment_list_dma);
1915         } else
1916                 create_req.base_address = cpu_to_le64(
1917                     op_reply_q->q_segments[0].segment_dma);
1918
1919         create_req.size = cpu_to_le16(op_reply_q->num_replies);
1920
1921         init_completion(&mrioc->init_cmds.done);
1922         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1923             sizeof(create_req), 1);
1924         if (retval) {
1925                 ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
1926                 goto out_unlock;
1927         }
1928         wait_for_completion_timeout(&mrioc->init_cmds.done,
1929             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1930         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1931                 ioc_err(mrioc, "create reply queue timed out\n");
1932                 mpi3mr_check_rh_fault_ioc(mrioc,
1933                     MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
1934                 retval = -1;
1935                 goto out_unlock;
1936         }
1937         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1938             != MPI3_IOCSTATUS_SUCCESS) {
1939                 ioc_err(mrioc,
1940                     "CreateRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1941                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1942                     mrioc->init_cmds.ioc_loginfo);
1943                 retval = -1;
1944                 goto out_unlock;
1945         }
1946         op_reply_q->qid = reply_qid;
1947         if (midx < mrioc->intr_info_count)
1948                 mrioc->intr_info[midx].op_reply_q = op_reply_q;
1949
1950         (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :
1951             mrioc->active_poll_qcount++;
1952
1953 out_unlock:
1954         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1955         mutex_unlock(&mrioc->init_cmds.mutex);
1956 out:
1957
1958         return retval;
1959 }
1960
1961 /**
1962  * mpi3mr_create_op_req_q - create operational request queue
1963  * @mrioc: Adapter instance reference
1964  * @idx: operational request queue index
1965  * @reply_qid: Reply queue ID
1966  *
1967  * Create operatinal request queue by issuing MPI request
1968  * through admin queue.
1969  *
1970  * Return:  0 on success, non-zero on failure.
1971  */
1972 static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
1973         u16 reply_qid)
1974 {
1975         struct mpi3_create_request_queue_request create_req;
1976         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
1977         int retval = 0;
1978         u16 req_qid = 0;
1979
1980         req_qid = op_req_q->qid;
1981
1982         if (req_qid) {
1983                 retval = -1;
1984                 ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
1985                     req_qid);
1986
1987                 return retval;
1988         }
1989         req_qid = idx + 1;
1990
1991         op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;
1992         op_req_q->ci = 0;
1993         op_req_q->pi = 0;
1994         op_req_q->reply_qid = reply_qid;
1995         spin_lock_init(&op_req_q->q_lock);
1996
1997         if (!op_req_q->q_segments) {
1998                 retval = mpi3mr_alloc_op_req_q_segments(mrioc, idx);
1999                 if (retval) {
2000                         mpi3mr_free_op_req_q_segments(mrioc, idx);
2001                         goto out;
2002                 }
2003         }
2004
2005         memset(&create_req, 0, sizeof(create_req));
2006         mutex_lock(&mrioc->init_cmds.mutex);
2007         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2008                 retval = -1;
2009                 ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
2010                 goto out_unlock;
2011         }
2012         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2013         mrioc->init_cmds.is_waiting = 1;
2014         mrioc->init_cmds.callback = NULL;
2015         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2016         create_req.function = MPI3_FUNCTION_CREATE_REQUEST_QUEUE;
2017         create_req.queue_id = cpu_to_le16(req_qid);
2018         if (mrioc->enable_segqueue) {
2019                 create_req.flags =
2020                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
2021                 create_req.base_address = cpu_to_le64(
2022                     op_req_q->q_segment_list_dma);
2023         } else
2024                 create_req.base_address = cpu_to_le64(
2025                     op_req_q->q_segments[0].segment_dma);
2026         create_req.reply_queue_id = cpu_to_le16(reply_qid);
2027         create_req.size = cpu_to_le16(op_req_q->num_requests);
2028
2029         init_completion(&mrioc->init_cmds.done);
2030         retval = mpi3mr_admin_request_post(mrioc, &create_req,
2031             sizeof(create_req), 1);
2032         if (retval) {
2033                 ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
2034                 goto out_unlock;
2035         }
2036         wait_for_completion_timeout(&mrioc->init_cmds.done,
2037             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2038         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2039                 ioc_err(mrioc, "create request queue timed out\n");
2040                 mpi3mr_check_rh_fault_ioc(mrioc,
2041                     MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT);
2042                 retval = -1;
2043                 goto out_unlock;
2044         }
2045         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2046             != MPI3_IOCSTATUS_SUCCESS) {
2047                 ioc_err(mrioc,
2048                     "CreateReqQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2049                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2050                     mrioc->init_cmds.ioc_loginfo);
2051                 retval = -1;
2052                 goto out_unlock;
2053         }
2054         op_req_q->qid = req_qid;
2055
2056 out_unlock:
2057         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2058         mutex_unlock(&mrioc->init_cmds.mutex);
2059 out:
2060
2061         return retval;
2062 }
2063
2064 /**
2065  * mpi3mr_create_op_queues - create operational queue pairs
2066  * @mrioc: Adapter instance reference
2067  *
2068  * Allocate memory for operational queue meta data and call
2069  * create request and reply queue functions.
2070  *
2071  * Return: 0 on success, non-zero on failures.
2072  */
2073 static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
2074 {
2075         int retval = 0;
2076         u16 num_queues = 0, i = 0, msix_count_op_q = 1;
2077
2078         num_queues = min_t(int, mrioc->facts.max_op_reply_q,
2079             mrioc->facts.max_op_req_q);
2080
2081         msix_count_op_q =
2082             mrioc->intr_info_count - mrioc->op_reply_q_offset;
2083         if (!mrioc->num_queues)
2084                 mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
2085         /*
2086          * During reset set the num_queues to the number of queues
2087          * that was set before the reset.
2088          */
2089         num_queues = mrioc->num_op_reply_q ?
2090             mrioc->num_op_reply_q : mrioc->num_queues;
2091         ioc_info(mrioc, "trying to create %d operational queue pairs\n",
2092             num_queues);
2093
2094         if (!mrioc->req_qinfo) {
2095                 mrioc->req_qinfo = kcalloc(num_queues,
2096                     sizeof(struct op_req_qinfo), GFP_KERNEL);
2097                 if (!mrioc->req_qinfo) {
2098                         retval = -1;
2099                         goto out_failed;
2100                 }
2101
2102                 mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *
2103                     num_queues, GFP_KERNEL);
2104                 if (!mrioc->op_reply_qinfo) {
2105                         retval = -1;
2106                         goto out_failed;
2107                 }
2108         }
2109
2110         if (mrioc->enable_segqueue)
2111                 ioc_info(mrioc,
2112                     "allocating operational queues through segmented queues\n");
2113
2114         for (i = 0; i < num_queues; i++) {
2115                 if (mpi3mr_create_op_reply_q(mrioc, i)) {
2116                         ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
2117                         break;
2118                 }
2119                 if (mpi3mr_create_op_req_q(mrioc, i,
2120                     mrioc->op_reply_qinfo[i].qid)) {
2121                         ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
2122                         mpi3mr_delete_op_reply_q(mrioc, i);
2123                         break;
2124                 }
2125         }
2126
2127         if (i == 0) {
2128                 /* Not even one queue is created successfully*/
2129                 retval = -1;
2130                 goto out_failed;
2131         }
2132         mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
2133         ioc_info(mrioc,
2134             "successfully created %d operational queue pairs(default/polled) queue = (%d/%d)\n",
2135             mrioc->num_op_reply_q, mrioc->default_qcount,
2136             mrioc->active_poll_qcount);
2137
2138         return retval;
2139 out_failed:
2140         kfree(mrioc->req_qinfo);
2141         mrioc->req_qinfo = NULL;
2142
2143         kfree(mrioc->op_reply_qinfo);
2144         mrioc->op_reply_qinfo = NULL;
2145
2146         return retval;
2147 }
2148
2149 /**
2150  * mpi3mr_op_request_post - Post request to operational queue
2151  * @mrioc: Adapter reference
2152  * @op_req_q: Operational request queue info
2153  * @req: MPI3 request
2154  *
2155  * Post the MPI3 request into operational request queue and
2156  * inform the controller, if the queue is full return
2157  * appropriate error.
2158  *
2159  * Return: 0 on success, non-zero on failure.
2160  */
2161 int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
2162         struct op_req_qinfo *op_req_q, u8 *req)
2163 {
2164         u16 pi = 0, max_entries, reply_qidx = 0, midx;
2165         int retval = 0;
2166         unsigned long flags;
2167         u8 *req_entry;
2168         void *segment_base_addr;
2169         u16 req_sz = mrioc->facts.op_req_sz;
2170         struct segments *segments = op_req_q->q_segments;
2171
2172         reply_qidx = op_req_q->reply_qid - 1;
2173
2174         if (mrioc->unrecoverable)
2175                 return -EFAULT;
2176
2177         spin_lock_irqsave(&op_req_q->q_lock, flags);
2178         pi = op_req_q->pi;
2179         max_entries = op_req_q->num_requests;
2180
2181         if (mpi3mr_check_req_qfull(op_req_q)) {
2182                 midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
2183                     reply_qidx, mrioc->op_reply_q_offset);
2184                 mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);
2185
2186                 if (mpi3mr_check_req_qfull(op_req_q)) {
2187                         retval = -EAGAIN;
2188                         goto out;
2189                 }
2190         }
2191
2192         if (mrioc->reset_in_progress) {
2193                 ioc_err(mrioc, "OpReqQ submit reset in progress\n");
2194                 retval = -EAGAIN;
2195                 goto out;
2196         }
2197
2198         segment_base_addr = segments[pi / op_req_q->segment_qd].segment;
2199         req_entry = (u8 *)segment_base_addr +
2200             ((pi % op_req_q->segment_qd) * req_sz);
2201
2202         memset(req_entry, 0, req_sz);
2203         memcpy(req_entry, req, MPI3MR_ADMIN_REQ_FRAME_SZ);
2204
2205         if (++pi == max_entries)
2206                 pi = 0;
2207         op_req_q->pi = pi;
2208
2209 #ifndef CONFIG_PREEMPT_RT
2210         if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)
2211             > MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT)
2212                 mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
2213 #else
2214         atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios);
2215 #endif
2216
2217         writel(op_req_q->pi,
2218             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
2219
2220 out:
2221         spin_unlock_irqrestore(&op_req_q->q_lock, flags);
2222         return retval;
2223 }
2224
2225 /**
2226  * mpi3mr_check_rh_fault_ioc - check reset history and fault
2227  * controller
2228  * @mrioc: Adapter instance reference
2229  * @reason_code: reason code for the fault.
2230  *
2231  * This routine will save snapdump and fault the controller with
2232  * the given reason code if it is not already in the fault or
2233  * not asynchronosuly reset. This will be used to handle
2234  * initilaization time faults/resets/timeout as in those cases
2235  * immediate soft reset invocation is not required.
2236  *
2237  * Return:  None.
2238  */
2239 void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code)
2240 {
2241         u32 ioc_status, host_diagnostic, timeout;
2242
2243         if (mrioc->unrecoverable) {
2244                 ioc_err(mrioc, "controller is unrecoverable\n");
2245                 return;
2246         }
2247
2248         if (!pci_device_is_present(mrioc->pdev)) {
2249                 mrioc->unrecoverable = 1;
2250                 ioc_err(mrioc, "controller is not present\n");
2251                 return;
2252         }
2253
2254         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2255         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
2256             (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
2257                 mpi3mr_print_fault_info(mrioc);
2258                 return;
2259         }
2260         mpi3mr_set_diagsave(mrioc);
2261         mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2262             reason_code);
2263         timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
2264         do {
2265                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2266                 if (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
2267                         break;
2268                 msleep(100);
2269         } while (--timeout);
2270 }
2271
2272 /**
2273  * mpi3mr_sync_timestamp - Issue time stamp sync request
2274  * @mrioc: Adapter reference
2275  *
2276  * Issue IO unit control MPI request to synchornize firmware
2277  * timestamp with host time.
2278  *
2279  * Return: 0 on success, non-zero on failure.
2280  */
2281 static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
2282 {
2283         ktime_t current_time;
2284         struct mpi3_iounit_control_request iou_ctrl;
2285         int retval = 0;
2286
2287         memset(&iou_ctrl, 0, sizeof(iou_ctrl));
2288         mutex_lock(&mrioc->init_cmds.mutex);
2289         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2290                 retval = -1;
2291                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
2292                 mutex_unlock(&mrioc->init_cmds.mutex);
2293                 goto out;
2294         }
2295         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2296         mrioc->init_cmds.is_waiting = 1;
2297         mrioc->init_cmds.callback = NULL;
2298         iou_ctrl.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2299         iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
2300         iou_ctrl.operation = MPI3_CTRL_OP_UPDATE_TIMESTAMP;
2301         current_time = ktime_get_real();
2302         iou_ctrl.param64[0] = cpu_to_le64(ktime_to_ms(current_time));
2303
2304         init_completion(&mrioc->init_cmds.done);
2305         retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl,
2306             sizeof(iou_ctrl), 0);
2307         if (retval) {
2308                 ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
2309                 goto out_unlock;
2310         }
2311
2312         wait_for_completion_timeout(&mrioc->init_cmds.done,
2313             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2314         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2315                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
2316                 mrioc->init_cmds.is_waiting = 0;
2317                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
2318                         mpi3mr_soft_reset_handler(mrioc,
2319                             MPI3MR_RESET_FROM_TSU_TIMEOUT, 1);
2320                 retval = -1;
2321                 goto out_unlock;
2322         }
2323         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2324             != MPI3_IOCSTATUS_SUCCESS) {
2325                 ioc_err(mrioc,
2326                     "Issue IOUCTL time_stamp: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2327                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2328                     mrioc->init_cmds.ioc_loginfo);
2329                 retval = -1;
2330                 goto out_unlock;
2331         }
2332
2333 out_unlock:
2334         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2335         mutex_unlock(&mrioc->init_cmds.mutex);
2336
2337 out:
2338         return retval;
2339 }
2340
2341 /**
2342  * mpi3mr_print_pkg_ver - display controller fw package version
2343  * @mrioc: Adapter reference
2344  *
2345  * Retrieve firmware package version from the component image
2346  * header of the controller flash and display it.
2347  *
2348  * Return: 0 on success and non-zero on failure.
2349  */
2350 static int mpi3mr_print_pkg_ver(struct mpi3mr_ioc *mrioc)
2351 {
2352         struct mpi3_ci_upload_request ci_upload;
2353         int retval = -1;
2354         void *data = NULL;
2355         dma_addr_t data_dma;
2356         struct mpi3_ci_manifest_mpi *manifest;
2357         u32 data_len = sizeof(struct mpi3_ci_manifest_mpi);
2358         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2359
2360         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2361             GFP_KERNEL);
2362         if (!data)
2363                 return -ENOMEM;
2364
2365         memset(&ci_upload, 0, sizeof(ci_upload));
2366         mutex_lock(&mrioc->init_cmds.mutex);
2367         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2368                 ioc_err(mrioc, "sending get package version failed due to command in use\n");
2369                 mutex_unlock(&mrioc->init_cmds.mutex);
2370                 goto out;
2371         }
2372         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2373         mrioc->init_cmds.is_waiting = 1;
2374         mrioc->init_cmds.callback = NULL;
2375         ci_upload.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2376         ci_upload.function = MPI3_FUNCTION_CI_UPLOAD;
2377         ci_upload.msg_flags = MPI3_CI_UPLOAD_MSGFLAGS_LOCATION_PRIMARY;
2378         ci_upload.signature1 = cpu_to_le32(MPI3_IMAGE_HEADER_SIGNATURE1_MANIFEST);
2379         ci_upload.image_offset = cpu_to_le32(MPI3_IMAGE_HEADER_SIZE);
2380         ci_upload.segment_size = cpu_to_le32(data_len);
2381
2382         mpi3mr_add_sg_single(&ci_upload.sgl, sgl_flags, data_len,
2383             data_dma);
2384         init_completion(&mrioc->init_cmds.done);
2385         retval = mpi3mr_admin_request_post(mrioc, &ci_upload,
2386             sizeof(ci_upload), 1);
2387         if (retval) {
2388                 ioc_err(mrioc, "posting get package version failed\n");
2389                 goto out_unlock;
2390         }
2391         wait_for_completion_timeout(&mrioc->init_cmds.done,
2392             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2393         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2394                 ioc_err(mrioc, "get package version timed out\n");
2395                 mpi3mr_check_rh_fault_ioc(mrioc,
2396                     MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT);
2397                 retval = -1;
2398                 goto out_unlock;
2399         }
2400         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2401             == MPI3_IOCSTATUS_SUCCESS) {
2402                 manifest = (struct mpi3_ci_manifest_mpi *) data;
2403                 if (manifest->manifest_type == MPI3_CI_MANIFEST_TYPE_MPI) {
2404                         ioc_info(mrioc,
2405                             "firmware package version(%d.%d.%d.%d.%05d-%05d)\n",
2406                             manifest->package_version.gen_major,
2407                             manifest->package_version.gen_minor,
2408                             manifest->package_version.phase_major,
2409                             manifest->package_version.phase_minor,
2410                             manifest->package_version.customer_id,
2411                             manifest->package_version.build_num);
2412                 }
2413         }
2414         retval = 0;
2415 out_unlock:
2416         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2417         mutex_unlock(&mrioc->init_cmds.mutex);
2418
2419 out:
2420         if (data)
2421                 dma_free_coherent(&mrioc->pdev->dev, data_len, data,
2422                     data_dma);
2423         return retval;
2424 }
2425
2426 /**
2427  * mpi3mr_watchdog_work - watchdog thread to monitor faults
2428  * @work: work struct
2429  *
2430  * Watch dog work periodically executed (1 second interval) to
2431  * monitor firmware fault and to issue periodic timer sync to
2432  * the firmware.
2433  *
2434  * Return: Nothing.
2435  */
2436 static void mpi3mr_watchdog_work(struct work_struct *work)
2437 {
2438         struct mpi3mr_ioc *mrioc =
2439             container_of(work, struct mpi3mr_ioc, watchdog_work.work);
2440         unsigned long flags;
2441         enum mpi3mr_iocstate ioc_state;
2442         u32 fault, host_diagnostic, ioc_status;
2443         u32 reset_reason = MPI3MR_RESET_FROM_FAULT_WATCH;
2444
2445         if (mrioc->reset_in_progress)
2446                 return;
2447
2448         if (!mrioc->unrecoverable && !pci_device_is_present(mrioc->pdev)) {
2449                 ioc_err(mrioc, "watchdog could not detect the controller\n");
2450                 mrioc->unrecoverable = 1;
2451         }
2452
2453         if (mrioc->unrecoverable) {
2454                 ioc_err(mrioc,
2455                     "flush pending commands for unrecoverable controller\n");
2456                 mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
2457                 return;
2458         }
2459
2460         if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
2461                 mrioc->ts_update_counter = 0;
2462                 mpi3mr_sync_timestamp(mrioc);
2463         }
2464
2465         if ((mrioc->prepare_for_reset) &&
2466             ((mrioc->prepare_for_reset_timeout_counter++) >=
2467              MPI3MR_PREPARE_FOR_RESET_TIMEOUT)) {
2468                 mpi3mr_soft_reset_handler(mrioc,
2469                     MPI3MR_RESET_FROM_CIACTVRST_TIMER, 1);
2470                 return;
2471         }
2472
2473         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2474         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) {
2475                 mpi3mr_soft_reset_handler(mrioc, MPI3MR_RESET_FROM_FIRMWARE, 0);
2476                 return;
2477         }
2478
2479         /*Check for fault state every one second and issue Soft reset*/
2480         ioc_state = mpi3mr_get_iocstate(mrioc);
2481         if (ioc_state != MRIOC_STATE_FAULT)
2482                 goto schedule_work;
2483
2484         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
2485         host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2486         if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
2487                 if (!mrioc->diagsave_timeout) {
2488                         mpi3mr_print_fault_info(mrioc);
2489                         ioc_warn(mrioc, "diag save in progress\n");
2490                 }
2491                 if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2492                         goto schedule_work;
2493         }
2494
2495         mpi3mr_print_fault_info(mrioc);
2496         mrioc->diagsave_timeout = 0;
2497
2498         switch (fault) {
2499         case MPI3_SYSIF_FAULT_CODE_COMPLETE_RESET_NEEDED:
2500         case MPI3_SYSIF_FAULT_CODE_POWER_CYCLE_REQUIRED:
2501                 ioc_warn(mrioc,
2502                     "controller requires system power cycle, marking controller as unrecoverable\n");
2503                 mrioc->unrecoverable = 1;
2504                 goto schedule_work;
2505         case MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS:
2506                 return;
2507         case MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET:
2508                 reset_reason = MPI3MR_RESET_FROM_CIACTIV_FAULT;
2509                 break;
2510         default:
2511                 break;
2512         }
2513         mpi3mr_soft_reset_handler(mrioc, reset_reason, 0);
2514         return;
2515
2516 schedule_work:
2517         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2518         if (mrioc->watchdog_work_q)
2519                 queue_delayed_work(mrioc->watchdog_work_q,
2520                     &mrioc->watchdog_work,
2521                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2522         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2523         return;
2524 }
2525
2526 /**
2527  * mpi3mr_start_watchdog - Start watchdog
2528  * @mrioc: Adapter instance reference
2529  *
2530  * Create and start the watchdog thread to monitor controller
2531  * faults.
2532  *
2533  * Return: Nothing.
2534  */
2535 void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2536 {
2537         if (mrioc->watchdog_work_q)
2538                 return;
2539
2540         INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2541         snprintf(mrioc->watchdog_work_q_name,
2542             sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
2543             mrioc->id);
2544         mrioc->watchdog_work_q =
2545             create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2546         if (!mrioc->watchdog_work_q) {
2547                 ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2548                 return;
2549         }
2550
2551         if (mrioc->watchdog_work_q)
2552                 queue_delayed_work(mrioc->watchdog_work_q,
2553                     &mrioc->watchdog_work,
2554                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2555 }
2556
2557 /**
2558  * mpi3mr_stop_watchdog - Stop watchdog
2559  * @mrioc: Adapter instance reference
2560  *
2561  * Stop the watchdog thread created to monitor controller
2562  * faults.
2563  *
2564  * Return: Nothing.
2565  */
2566 void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2567 {
2568         unsigned long flags;
2569         struct workqueue_struct *wq;
2570
2571         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2572         wq = mrioc->watchdog_work_q;
2573         mrioc->watchdog_work_q = NULL;
2574         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2575         if (wq) {
2576                 if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
2577                         flush_workqueue(wq);
2578                 destroy_workqueue(wq);
2579         }
2580 }
2581
2582 /**
2583  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2584  * @mrioc: Adapter instance reference
2585  *
2586  * Allocate memory for admin queue pair if required and register
2587  * the admin queue with the controller.
2588  *
2589  * Return: 0 on success, non-zero on failures.
2590  */
2591 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2592 {
2593         int retval = 0;
2594         u32 num_admin_entries = 0;
2595
2596         mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2597         mrioc->num_admin_req = mrioc->admin_req_q_sz /
2598             MPI3MR_ADMIN_REQ_FRAME_SZ;
2599         mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
2600         mrioc->admin_req_base = NULL;
2601
2602         mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2603         mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2604             MPI3MR_ADMIN_REPLY_FRAME_SZ;
2605         mrioc->admin_reply_ci = 0;
2606         mrioc->admin_reply_ephase = 1;
2607         mrioc->admin_reply_base = NULL;
2608
2609         if (!mrioc->admin_req_base) {
2610                 mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,
2611                     mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);
2612
2613                 if (!mrioc->admin_req_base) {
2614                         retval = -1;
2615                         goto out_failed;
2616                 }
2617
2618                 mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,
2619                     mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,
2620                     GFP_KERNEL);
2621
2622                 if (!mrioc->admin_reply_base) {
2623                         retval = -1;
2624                         goto out_failed;
2625                 }
2626         }
2627
2628         num_admin_entries = (mrioc->num_admin_replies << 16) |
2629             (mrioc->num_admin_req);
2630         writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);
2631         mpi3mr_writeq(mrioc->admin_req_dma,
2632             &mrioc->sysif_regs->admin_request_queue_address);
2633         mpi3mr_writeq(mrioc->admin_reply_dma,
2634             &mrioc->sysif_regs->admin_reply_queue_address);
2635         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
2636         writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
2637         return retval;
2638
2639 out_failed:
2640
2641         if (mrioc->admin_reply_base) {
2642                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
2643                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
2644                 mrioc->admin_reply_base = NULL;
2645         }
2646         if (mrioc->admin_req_base) {
2647                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
2648                     mrioc->admin_req_base, mrioc->admin_req_dma);
2649                 mrioc->admin_req_base = NULL;
2650         }
2651         return retval;
2652 }
2653
2654 /**
2655  * mpi3mr_issue_iocfacts - Send IOC Facts
2656  * @mrioc: Adapter instance reference
2657  * @facts_data: Cached IOC facts data
2658  *
2659  * Issue IOC Facts MPI request through admin queue and wait for
2660  * the completion of it or time out.
2661  *
2662  * Return: 0 on success, non-zero on failures.
2663  */
2664 static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2665         struct mpi3_ioc_facts_data *facts_data)
2666 {
2667         struct mpi3_ioc_facts_request iocfacts_req;
2668         void *data = NULL;
2669         dma_addr_t data_dma;
2670         u32 data_len = sizeof(*facts_data);
2671         int retval = 0;
2672         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2673
2674         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2675             GFP_KERNEL);
2676
2677         if (!data) {
2678                 retval = -1;
2679                 goto out;
2680         }
2681
2682         memset(&iocfacts_req, 0, sizeof(iocfacts_req));
2683         mutex_lock(&mrioc->init_cmds.mutex);
2684         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2685                 retval = -1;
2686                 ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2687                 mutex_unlock(&mrioc->init_cmds.mutex);
2688                 goto out;
2689         }
2690         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2691         mrioc->init_cmds.is_waiting = 1;
2692         mrioc->init_cmds.callback = NULL;
2693         iocfacts_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2694         iocfacts_req.function = MPI3_FUNCTION_IOC_FACTS;
2695
2696         mpi3mr_add_sg_single(&iocfacts_req.sgl, sgl_flags, data_len,
2697             data_dma);
2698
2699         init_completion(&mrioc->init_cmds.done);
2700         retval = mpi3mr_admin_request_post(mrioc, &iocfacts_req,
2701             sizeof(iocfacts_req), 1);
2702         if (retval) {
2703                 ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2704                 goto out_unlock;
2705         }
2706         wait_for_completion_timeout(&mrioc->init_cmds.done,
2707             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2708         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2709                 ioc_err(mrioc, "ioc_facts timed out\n");
2710                 mpi3mr_check_rh_fault_ioc(mrioc,
2711                     MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
2712                 retval = -1;
2713                 goto out_unlock;
2714         }
2715         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2716             != MPI3_IOCSTATUS_SUCCESS) {
2717                 ioc_err(mrioc,
2718                     "Issue IOCFacts: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2719                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2720                     mrioc->init_cmds.ioc_loginfo);
2721                 retval = -1;
2722                 goto out_unlock;
2723         }
2724         memcpy(facts_data, (u8 *)data, data_len);
2725         mpi3mr_process_factsdata(mrioc, facts_data);
2726 out_unlock:
2727         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2728         mutex_unlock(&mrioc->init_cmds.mutex);
2729
2730 out:
2731         if (data)
2732                 dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);
2733
2734         return retval;
2735 }
2736
2737 /**
2738  * mpi3mr_check_reset_dma_mask - Process IOC facts data
2739  * @mrioc: Adapter instance reference
2740  *
2741  * Check whether the new DMA mask requested through IOCFacts by
2742  * firmware needs to be set, if so set it .
2743  *
2744  * Return: 0 on success, non-zero on failure.
2745  */
2746 static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2747 {
2748         struct pci_dev *pdev = mrioc->pdev;
2749         int r;
2750         u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2751
2752         if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2753                 return 0;
2754
2755         ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2756             mrioc->dma_mask, facts_dma_mask);
2757
2758         r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);
2759         if (r) {
2760                 ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2761                     facts_dma_mask, r);
2762                 return r;
2763         }
2764         mrioc->dma_mask = facts_dma_mask;
2765         return r;
2766 }
2767
2768 /**
2769  * mpi3mr_process_factsdata - Process IOC facts data
2770  * @mrioc: Adapter instance reference
2771  * @facts_data: Cached IOC facts data
2772  *
2773  * Convert IOC facts data into cpu endianness and cache it in
2774  * the driver .
2775  *
2776  * Return: Nothing.
2777  */
2778 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2779         struct mpi3_ioc_facts_data *facts_data)
2780 {
2781         u32 ioc_config, req_sz, facts_flags;
2782
2783         if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=
2784             (sizeof(*facts_data) / 4)) {
2785                 ioc_warn(mrioc,
2786                     "IOCFactsdata length mismatch driver_sz(%zu) firmware_sz(%d)\n",
2787                     sizeof(*facts_data),
2788                     le16_to_cpu(facts_data->ioc_facts_data_length) * 4);
2789         }
2790
2791         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
2792         req_sz = 1 << ((ioc_config & MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ) >>
2793             MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ_SHIFT);
2794         if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {
2795                 ioc_err(mrioc,
2796                     "IOCFacts data reqFrameSize mismatch hw_size(%d) firmware_sz(%d)\n",
2797                     req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));
2798         }
2799
2800         memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2801
2802         facts_flags = le32_to_cpu(facts_data->flags);
2803         mrioc->facts.op_req_sz = req_sz;
2804         mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2805             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ) >>
2806             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ_SHIFT);
2807
2808         mrioc->facts.ioc_num = facts_data->ioc_number;
2809         mrioc->facts.who_init = facts_data->who_init;
2810         mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2811         mrioc->facts.personality = (facts_flags &
2812             MPI3_IOCFACTS_FLAGS_PERSONALITY_MASK);
2813         mrioc->facts.dma_mask = (facts_flags &
2814             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
2815             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
2816         mrioc->facts.protocol_flags = facts_data->protocol_flags;
2817         mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2818         mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);
2819         mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2820         mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2821         mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2822         mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2823         mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2824         mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2825         mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);
2826         mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);
2827         mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2828         mrioc->facts.max_pcie_switches =
2829             le16_to_cpu(facts_data->max_pcie_switches);
2830         mrioc->facts.max_sasexpanders =
2831             le16_to_cpu(facts_data->max_sas_expanders);
2832         mrioc->facts.max_sasinitiators =
2833             le16_to_cpu(facts_data->max_sas_initiators);
2834         mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2835         mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2836         mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2837         mrioc->facts.max_op_req_q =
2838             le16_to_cpu(facts_data->max_operational_request_queues);
2839         mrioc->facts.max_op_reply_q =
2840             le16_to_cpu(facts_data->max_operational_reply_queues);
2841         mrioc->facts.ioc_capabilities =
2842             le32_to_cpu(facts_data->ioc_capabilities);
2843         mrioc->facts.fw_ver.build_num =
2844             le16_to_cpu(facts_data->fw_version.build_num);
2845         mrioc->facts.fw_ver.cust_id =
2846             le16_to_cpu(facts_data->fw_version.customer_id);
2847         mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2848         mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2849         mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2850         mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2851         mrioc->msix_count = min_t(int, mrioc->msix_count,
2852             mrioc->facts.max_msix_vectors);
2853         mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2854         mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2855         mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2856         mrioc->facts.shutdown_timeout =
2857             le16_to_cpu(facts_data->shutdown_timeout);
2858
2859         mrioc->facts.max_dev_per_tg =
2860             facts_data->max_devices_per_throttle_group;
2861         mrioc->facts.io_throttle_data_length =
2862             le16_to_cpu(facts_data->io_throttle_data_length);
2863         mrioc->facts.max_io_throttle_group =
2864             le16_to_cpu(facts_data->max_io_throttle_group);
2865         mrioc->facts.io_throttle_low = le16_to_cpu(facts_data->io_throttle_low);
2866         mrioc->facts.io_throttle_high =
2867             le16_to_cpu(facts_data->io_throttle_high);
2868
2869         /* Store in 512b block count */
2870         if (mrioc->facts.io_throttle_data_length)
2871                 mrioc->io_throttle_data_length =
2872                     (mrioc->facts.io_throttle_data_length * 2 * 4);
2873         else
2874                 /* set the length to 1MB + 1K to disable throttle */
2875                 mrioc->io_throttle_data_length = MPI3MR_MAX_SECTORS + 2;
2876
2877         mrioc->io_throttle_high = (mrioc->facts.io_throttle_high * 2 * 1024);
2878         mrioc->io_throttle_low = (mrioc->facts.io_throttle_low * 2 * 1024);
2879
2880         ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
2881             mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
2882             mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
2883         ioc_info(mrioc,
2884             "maxreqs(%d), mindh(%d) maxvectors(%d) maxperids(%d)\n",
2885             mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
2886             mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);
2887         ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
2888             mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
2889             mrioc->facts.sge_mod_shift);
2890         ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x\n",
2891             mrioc->facts.dma_mask, (facts_flags &
2892             MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK));
2893         ioc_info(mrioc,
2894             "max_dev_per_throttle_group(%d), max_throttle_groups(%d)\n",
2895             mrioc->facts.max_dev_per_tg, mrioc->facts.max_io_throttle_group);
2896         ioc_info(mrioc,
2897            "io_throttle_data_len(%dKiB), io_throttle_high(%dMiB), io_throttle_low(%dMiB)\n",
2898            mrioc->facts.io_throttle_data_length * 4,
2899            mrioc->facts.io_throttle_high, mrioc->facts.io_throttle_low);
2900 }
2901
2902 /**
2903  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
2904  * @mrioc: Adapter instance reference
2905  *
2906  * Allocate and initialize the reply free buffers, sense
2907  * buffers, reply free queue and sense buffer queue.
2908  *
2909  * Return: 0 on success, non-zero on failures.
2910  */
2911 static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
2912 {
2913         int retval = 0;
2914         u32 sz, i;
2915
2916         if (mrioc->init_cmds.reply)
2917                 return retval;
2918
2919         mrioc->init_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2920         if (!mrioc->init_cmds.reply)
2921                 goto out_failed;
2922
2923         mrioc->bsg_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2924         if (!mrioc->bsg_cmds.reply)
2925                 goto out_failed;
2926
2927         mrioc->transport_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2928         if (!mrioc->transport_cmds.reply)
2929                 goto out_failed;
2930
2931         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
2932                 mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz,
2933                     GFP_KERNEL);
2934                 if (!mrioc->dev_rmhs_cmds[i].reply)
2935                         goto out_failed;
2936         }
2937
2938         for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
2939                 mrioc->evtack_cmds[i].reply = kzalloc(mrioc->reply_sz,
2940                     GFP_KERNEL);
2941                 if (!mrioc->evtack_cmds[i].reply)
2942                         goto out_failed;
2943         }
2944
2945         mrioc->host_tm_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2946         if (!mrioc->host_tm_cmds.reply)
2947                 goto out_failed;
2948
2949         mrioc->pel_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2950         if (!mrioc->pel_cmds.reply)
2951                 goto out_failed;
2952
2953         mrioc->pel_abort_cmd.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
2954         if (!mrioc->pel_abort_cmd.reply)
2955                 goto out_failed;
2956
2957         mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
2958         mrioc->removepend_bitmap = bitmap_zalloc(mrioc->dev_handle_bitmap_bits,
2959                                                  GFP_KERNEL);
2960         if (!mrioc->removepend_bitmap)
2961                 goto out_failed;
2962
2963         mrioc->devrem_bitmap = bitmap_zalloc(MPI3MR_NUM_DEVRMCMD, GFP_KERNEL);
2964         if (!mrioc->devrem_bitmap)
2965                 goto out_failed;
2966
2967         mrioc->evtack_cmds_bitmap = bitmap_zalloc(MPI3MR_NUM_EVTACKCMD,
2968                                                   GFP_KERNEL);
2969         if (!mrioc->evtack_cmds_bitmap)
2970                 goto out_failed;
2971
2972         mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
2973         mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
2974         mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
2975         mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
2976
2977         /* reply buffer pool, 16 byte align */
2978         sz = mrioc->num_reply_bufs * mrioc->reply_sz;
2979         mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",
2980             &mrioc->pdev->dev, sz, 16, 0);
2981         if (!mrioc->reply_buf_pool) {
2982                 ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
2983                 goto out_failed;
2984         }
2985
2986         mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,
2987             &mrioc->reply_buf_dma);
2988         if (!mrioc->reply_buf)
2989                 goto out_failed;
2990
2991         mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
2992
2993         /* reply free queue, 8 byte align */
2994         sz = mrioc->reply_free_qsz * 8;
2995         mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",
2996             &mrioc->pdev->dev, sz, 8, 0);
2997         if (!mrioc->reply_free_q_pool) {
2998                 ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
2999                 goto out_failed;
3000         }
3001         mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,
3002             GFP_KERNEL, &mrioc->reply_free_q_dma);
3003         if (!mrioc->reply_free_q)
3004                 goto out_failed;
3005
3006         /* sense buffer pool,  4 byte align */
3007         sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3008         mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",
3009             &mrioc->pdev->dev, sz, 4, 0);
3010         if (!mrioc->sense_buf_pool) {
3011                 ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
3012                 goto out_failed;
3013         }
3014         mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,
3015             &mrioc->sense_buf_dma);
3016         if (!mrioc->sense_buf)
3017                 goto out_failed;
3018
3019         /* sense buffer queue, 8 byte align */
3020         sz = mrioc->sense_buf_q_sz * 8;
3021         mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",
3022             &mrioc->pdev->dev, sz, 8, 0);
3023         if (!mrioc->sense_buf_q_pool) {
3024                 ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
3025                 goto out_failed;
3026         }
3027         mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,
3028             GFP_KERNEL, &mrioc->sense_buf_q_dma);
3029         if (!mrioc->sense_buf_q)
3030                 goto out_failed;
3031
3032         return retval;
3033
3034 out_failed:
3035         retval = -1;
3036         return retval;
3037 }
3038
3039 /**
3040  * mpimr_initialize_reply_sbuf_queues - initialize reply sense
3041  * buffers
3042  * @mrioc: Adapter instance reference
3043  *
3044  * Helper function to initialize reply and sense buffers along
3045  * with some debug prints.
3046  *
3047  * Return:  None.
3048  */
3049 static void mpimr_initialize_reply_sbuf_queues(struct mpi3mr_ioc *mrioc)
3050 {
3051         u32 sz, i;
3052         dma_addr_t phy_addr;
3053
3054         sz = mrioc->num_reply_bufs * mrioc->reply_sz;
3055         ioc_info(mrioc,
3056             "reply buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
3057             mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,
3058             (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
3059         sz = mrioc->reply_free_qsz * 8;
3060         ioc_info(mrioc,
3061             "reply_free_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
3062             mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
3063             (unsigned long long)mrioc->reply_free_q_dma);
3064         sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3065         ioc_info(mrioc,
3066             "sense_buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
3067             mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,
3068             (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
3069         sz = mrioc->sense_buf_q_sz * 8;
3070         ioc_info(mrioc,
3071             "sense_buf_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
3072             mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
3073             (unsigned long long)mrioc->sense_buf_q_dma);
3074
3075         /* initialize Reply buffer Queue */
3076         for (i = 0, phy_addr = mrioc->reply_buf_dma;
3077             i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)
3078                 mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
3079         mrioc->reply_free_q[i] = cpu_to_le64(0);
3080
3081         /* initialize Sense Buffer Queue */
3082         for (i = 0, phy_addr = mrioc->sense_buf_dma;
3083             i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)
3084                 mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
3085         mrioc->sense_buf_q[i] = cpu_to_le64(0);
3086 }
3087
3088 /**
3089  * mpi3mr_issue_iocinit - Send IOC Init
3090  * @mrioc: Adapter instance reference
3091  *
3092  * Issue IOC Init MPI request through admin queue and wait for
3093  * the completion of it or time out.
3094  *
3095  * Return: 0 on success, non-zero on failures.
3096  */
3097 static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
3098 {
3099         struct mpi3_ioc_init_request iocinit_req;
3100         struct mpi3_driver_info_layout *drv_info;
3101         dma_addr_t data_dma;
3102         u32 data_len = sizeof(*drv_info);
3103         int retval = 0;
3104         ktime_t current_time;
3105
3106         drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
3107             GFP_KERNEL);
3108         if (!drv_info) {
3109                 retval = -1;
3110                 goto out;
3111         }
3112         mpimr_initialize_reply_sbuf_queues(mrioc);
3113
3114         drv_info->information_length = cpu_to_le32(data_len);
3115         strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));
3116         strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));
3117         strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));
3118         strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));
3119         strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));
3120         strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,
3121             sizeof(drv_info->driver_release_date));
3122         drv_info->driver_capabilities = 0;
3123         memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
3124             sizeof(mrioc->driver_info));
3125
3126         memset(&iocinit_req, 0, sizeof(iocinit_req));
3127         mutex_lock(&mrioc->init_cmds.mutex);
3128         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3129                 retval = -1;
3130                 ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
3131                 mutex_unlock(&mrioc->init_cmds.mutex);
3132                 goto out;
3133         }
3134         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3135         mrioc->init_cmds.is_waiting = 1;
3136         mrioc->init_cmds.callback = NULL;
3137         iocinit_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3138         iocinit_req.function = MPI3_FUNCTION_IOC_INIT;
3139         iocinit_req.mpi_version.mpi3_version.dev = MPI3_VERSION_DEV;
3140         iocinit_req.mpi_version.mpi3_version.unit = MPI3_VERSION_UNIT;
3141         iocinit_req.mpi_version.mpi3_version.major = MPI3_VERSION_MAJOR;
3142         iocinit_req.mpi_version.mpi3_version.minor = MPI3_VERSION_MINOR;
3143         iocinit_req.who_init = MPI3_WHOINIT_HOST_DRIVER;
3144         iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
3145         iocinit_req.reply_free_queue_address =
3146             cpu_to_le64(mrioc->reply_free_q_dma);
3147         iocinit_req.sense_buffer_length = cpu_to_le16(MPI3MR_SENSE_BUF_SZ);
3148         iocinit_req.sense_buffer_free_queue_depth =
3149             cpu_to_le16(mrioc->sense_buf_q_sz);
3150         iocinit_req.sense_buffer_free_queue_address =
3151             cpu_to_le64(mrioc->sense_buf_q_dma);
3152         iocinit_req.driver_information_address = cpu_to_le64(data_dma);
3153
3154         current_time = ktime_get_real();
3155         iocinit_req.time_stamp = cpu_to_le64(ktime_to_ms(current_time));
3156
3157         init_completion(&mrioc->init_cmds.done);
3158         retval = mpi3mr_admin_request_post(mrioc, &iocinit_req,
3159             sizeof(iocinit_req), 1);
3160         if (retval) {
3161                 ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
3162                 goto out_unlock;
3163         }
3164         wait_for_completion_timeout(&mrioc->init_cmds.done,
3165             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3166         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3167                 mpi3mr_check_rh_fault_ioc(mrioc,
3168                     MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
3169                 ioc_err(mrioc, "ioc_init timed out\n");
3170                 retval = -1;
3171                 goto out_unlock;
3172         }
3173         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3174             != MPI3_IOCSTATUS_SUCCESS) {
3175                 ioc_err(mrioc,
3176                     "Issue IOCInit: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3177                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3178                     mrioc->init_cmds.ioc_loginfo);
3179                 retval = -1;
3180                 goto out_unlock;
3181         }
3182
3183         mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3184         writel(mrioc->reply_free_queue_host_index,
3185             &mrioc->sysif_regs->reply_free_host_index);
3186
3187         mrioc->sbq_host_index = mrioc->num_sense_bufs;
3188         writel(mrioc->sbq_host_index,
3189             &mrioc->sysif_regs->sense_buffer_free_host_index);
3190 out_unlock:
3191         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3192         mutex_unlock(&mrioc->init_cmds.mutex);
3193
3194 out:
3195         if (drv_info)
3196                 dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,
3197                     data_dma);
3198
3199         return retval;
3200 }
3201
3202 /**
3203  * mpi3mr_unmask_events - Unmask events in event mask bitmap
3204  * @mrioc: Adapter instance reference
3205  * @event: MPI event ID
3206  *
3207  * Un mask the specific event by resetting the event_mask
3208  * bitmap.
3209  *
3210  * Return: 0 on success, non-zero on failures.
3211  */
3212 static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
3213 {
3214         u32 desired_event;
3215         u8 word;
3216
3217         if (event >= 128)
3218                 return;
3219
3220         desired_event = (1 << (event % 32));
3221         word = event / 32;
3222
3223         mrioc->event_masks[word] &= ~desired_event;
3224 }
3225
3226 /**
3227  * mpi3mr_issue_event_notification - Send event notification
3228  * @mrioc: Adapter instance reference
3229  *
3230  * Issue event notification MPI request through admin queue and
3231  * wait for the completion of it or time out.
3232  *
3233  * Return: 0 on success, non-zero on failures.
3234  */
3235 static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
3236 {
3237         struct mpi3_event_notification_request evtnotify_req;
3238         int retval = 0;
3239         u8 i;
3240
3241         memset(&evtnotify_req, 0, sizeof(evtnotify_req));
3242         mutex_lock(&mrioc->init_cmds.mutex);
3243         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3244                 retval = -1;
3245                 ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
3246                 mutex_unlock(&mrioc->init_cmds.mutex);
3247                 goto out;
3248         }
3249         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3250         mrioc->init_cmds.is_waiting = 1;
3251         mrioc->init_cmds.callback = NULL;
3252         evtnotify_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3253         evtnotify_req.function = MPI3_FUNCTION_EVENT_NOTIFICATION;
3254         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3255                 evtnotify_req.event_masks[i] =
3256                     cpu_to_le32(mrioc->event_masks[i]);
3257         init_completion(&mrioc->init_cmds.done);
3258         retval = mpi3mr_admin_request_post(mrioc, &evtnotify_req,
3259             sizeof(evtnotify_req), 1);
3260         if (retval) {
3261                 ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
3262                 goto out_unlock;
3263         }
3264         wait_for_completion_timeout(&mrioc->init_cmds.done,
3265             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3266         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3267                 ioc_err(mrioc, "event notification timed out\n");
3268                 mpi3mr_check_rh_fault_ioc(mrioc,
3269                     MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
3270                 retval = -1;
3271                 goto out_unlock;
3272         }
3273         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3274             != MPI3_IOCSTATUS_SUCCESS) {
3275                 ioc_err(mrioc,
3276                     "Issue EvtNotify: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3277                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3278                     mrioc->init_cmds.ioc_loginfo);
3279                 retval = -1;
3280                 goto out_unlock;
3281         }
3282
3283 out_unlock:
3284         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3285         mutex_unlock(&mrioc->init_cmds.mutex);
3286 out:
3287         return retval;
3288 }
3289
3290 /**
3291  * mpi3mr_process_event_ack - Process event acknowledgment
3292  * @mrioc: Adapter instance reference
3293  * @event: MPI3 event ID
3294  * @event_ctx: event context
3295  *
3296  * Send event acknowledgment through admin queue and wait for
3297  * it to complete.
3298  *
3299  * Return: 0 on success, non-zero on failures.
3300  */
3301 int mpi3mr_process_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
3302         u32 event_ctx)
3303 {
3304         struct mpi3_event_ack_request evtack_req;
3305         int retval = 0;
3306
3307         memset(&evtack_req, 0, sizeof(evtack_req));
3308         mutex_lock(&mrioc->init_cmds.mutex);
3309         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3310                 retval = -1;
3311                 ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
3312                 mutex_unlock(&mrioc->init_cmds.mutex);
3313                 goto out;
3314         }
3315         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3316         mrioc->init_cmds.is_waiting = 1;
3317         mrioc->init_cmds.callback = NULL;
3318         evtack_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3319         evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
3320         evtack_req.event = event;
3321         evtack_req.event_context = cpu_to_le32(event_ctx);
3322
3323         init_completion(&mrioc->init_cmds.done);
3324         retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
3325             sizeof(evtack_req), 1);
3326         if (retval) {
3327                 ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
3328                 goto out_unlock;
3329         }
3330         wait_for_completion_timeout(&mrioc->init_cmds.done,
3331             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3332         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3333                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
3334                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
3335                         mpi3mr_soft_reset_handler(mrioc,
3336                             MPI3MR_RESET_FROM_EVTACK_TIMEOUT, 1);
3337                 retval = -1;
3338                 goto out_unlock;
3339         }
3340         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3341             != MPI3_IOCSTATUS_SUCCESS) {
3342                 ioc_err(mrioc,
3343                     "Send EvtAck: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3344                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3345                     mrioc->init_cmds.ioc_loginfo);
3346                 retval = -1;
3347                 goto out_unlock;
3348         }
3349
3350 out_unlock:
3351         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3352         mutex_unlock(&mrioc->init_cmds.mutex);
3353 out:
3354         return retval;
3355 }
3356
3357 /**
3358  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
3359  * @mrioc: Adapter instance reference
3360  *
3361  * Allocate chain buffers and set a bitmap to indicate free
3362  * chain buffers. Chain buffers are used to pass the SGE
3363  * information along with MPI3 SCSI IO requests for host I/O.
3364  *
3365  * Return: 0 on success, non-zero on failure
3366  */
3367 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
3368 {
3369         int retval = 0;
3370         u32 sz, i;
3371         u16 num_chains;
3372
3373         if (mrioc->chain_sgl_list)
3374                 return retval;
3375
3376         num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
3377
3378         if (prot_mask & (SHOST_DIX_TYPE0_PROTECTION
3379             | SHOST_DIX_TYPE1_PROTECTION
3380             | SHOST_DIX_TYPE2_PROTECTION
3381             | SHOST_DIX_TYPE3_PROTECTION))
3382                 num_chains += (num_chains / MPI3MR_CHAINBUFDIX_FACTOR);
3383
3384         mrioc->chain_buf_count = num_chains;
3385         sz = sizeof(struct chain_element) * num_chains;
3386         mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);
3387         if (!mrioc->chain_sgl_list)
3388                 goto out_failed;
3389
3390         sz = MPI3MR_PAGE_SIZE_4K;
3391         mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",
3392             &mrioc->pdev->dev, sz, 16, 0);
3393         if (!mrioc->chain_buf_pool) {
3394                 ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
3395                 goto out_failed;
3396         }
3397
3398         for (i = 0; i < num_chains; i++) {
3399                 mrioc->chain_sgl_list[i].addr =
3400                     dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,
3401                     &mrioc->chain_sgl_list[i].dma_addr);
3402
3403                 if (!mrioc->chain_sgl_list[i].addr)
3404                         goto out_failed;
3405         }
3406         mrioc->chain_bitmap = bitmap_zalloc(num_chains, GFP_KERNEL);
3407         if (!mrioc->chain_bitmap)
3408                 goto out_failed;
3409         return retval;
3410 out_failed:
3411         retval = -1;
3412         return retval;
3413 }
3414
3415 /**
3416  * mpi3mr_port_enable_complete - Mark port enable complete
3417  * @mrioc: Adapter instance reference
3418  * @drv_cmd: Internal command tracker
3419  *
3420  * Call back for asynchronous port enable request sets the
3421  * driver command to indicate port enable request is complete.
3422  *
3423  * Return: Nothing
3424  */
3425 static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
3426         struct mpi3mr_drv_cmd *drv_cmd)
3427 {
3428         drv_cmd->callback = NULL;
3429         mrioc->scan_started = 0;
3430         if (drv_cmd->state & MPI3MR_CMD_RESET)
3431                 mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
3432         else
3433                 mrioc->scan_failed = drv_cmd->ioc_status;
3434         drv_cmd->state = MPI3MR_CMD_NOTUSED;
3435 }
3436
3437 /**
3438  * mpi3mr_issue_port_enable - Issue Port Enable
3439  * @mrioc: Adapter instance reference
3440  * @async: Flag to wait for completion or not
3441  *
3442  * Issue Port Enable MPI request through admin queue and if the
3443  * async flag is not set wait for the completion of the port
3444  * enable or time out.
3445  *
3446  * Return: 0 on success, non-zero on failures.
3447  */
3448 int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
3449 {
3450         struct mpi3_port_enable_request pe_req;
3451         int retval = 0;
3452         u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
3453
3454         memset(&pe_req, 0, sizeof(pe_req));
3455         mutex_lock(&mrioc->init_cmds.mutex);
3456         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3457                 retval = -1;
3458                 ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
3459                 mutex_unlock(&mrioc->init_cmds.mutex);
3460                 goto out;
3461         }
3462         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3463         if (async) {
3464                 mrioc->init_cmds.is_waiting = 0;
3465                 mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
3466         } else {
3467                 mrioc->init_cmds.is_waiting = 1;
3468                 mrioc->init_cmds.callback = NULL;
3469                 init_completion(&mrioc->init_cmds.done);
3470         }
3471         pe_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3472         pe_req.function = MPI3_FUNCTION_PORT_ENABLE;
3473
3474         retval = mpi3mr_admin_request_post(mrioc, &pe_req, sizeof(pe_req), 1);
3475         if (retval) {
3476                 ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
3477                 goto out_unlock;
3478         }
3479         if (async) {
3480                 mutex_unlock(&mrioc->init_cmds.mutex);
3481                 goto out;
3482         }
3483
3484         wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));
3485         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3486                 ioc_err(mrioc, "port enable timed out\n");
3487                 retval = -1;
3488                 mpi3mr_check_rh_fault_ioc(mrioc, MPI3MR_RESET_FROM_PE_TIMEOUT);
3489                 goto out_unlock;
3490         }
3491         mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
3492
3493 out_unlock:
3494         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3495         mutex_unlock(&mrioc->init_cmds.mutex);
3496 out:
3497         return retval;
3498 }
3499
3500 /* Protocol type to name mapper structure */
3501 static const struct {
3502         u8 protocol;
3503         char *name;
3504 } mpi3mr_protocols[] = {
3505         { MPI3_IOCFACTS_PROTOCOL_SCSI_INITIATOR, "Initiator" },
3506         { MPI3_IOCFACTS_PROTOCOL_SCSI_TARGET, "Target" },
3507         { MPI3_IOCFACTS_PROTOCOL_NVME, "NVMe attachment" },
3508 };
3509
3510 /* Capability to name mapper structure*/
3511 static const struct {
3512         u32 capability;
3513         char *name;
3514 } mpi3mr_capabilities[] = {
3515         { MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" },
3516         { MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED, "MultiPath" },
3517 };
3518
3519 /**
3520  * mpi3mr_print_ioc_info - Display controller information
3521  * @mrioc: Adapter instance reference
3522  *
3523  * Display controller personalit, capability, supported
3524  * protocols etc.
3525  *
3526  * Return: Nothing
3527  */
3528 static void
3529 mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3530 {
3531         int i = 0, bytes_written = 0;
3532         char personality[16];
3533         char protocol[50] = {0};
3534         char capabilities[100] = {0};
3535         struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3536
3537         switch (mrioc->facts.personality) {
3538         case MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA:
3539                 strncpy(personality, "Enhanced HBA", sizeof(personality));
3540                 break;
3541         case MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR:
3542                 strncpy(personality, "RAID", sizeof(personality));
3543                 break;
3544         default:
3545                 strncpy(personality, "Unknown", sizeof(personality));
3546                 break;
3547         }
3548
3549         ioc_info(mrioc, "Running in %s Personality", personality);
3550
3551         ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3552             fwver->gen_major, fwver->gen_minor, fwver->ph_major,
3553             fwver->ph_minor, fwver->cust_id, fwver->build_num);
3554
3555         for (i = 0; i < ARRAY_SIZE(mpi3mr_protocols); i++) {
3556                 if (mrioc->facts.protocol_flags &
3557                     mpi3mr_protocols[i].protocol) {
3558                         bytes_written += scnprintf(protocol + bytes_written,
3559                                     sizeof(protocol) - bytes_written, "%s%s",
3560                                     bytes_written ? "," : "",
3561                                     mpi3mr_protocols[i].name);
3562                 }
3563         }
3564
3565         bytes_written = 0;
3566         for (i = 0; i < ARRAY_SIZE(mpi3mr_capabilities); i++) {
3567                 if (mrioc->facts.protocol_flags &
3568                     mpi3mr_capabilities[i].capability) {
3569                         bytes_written += scnprintf(capabilities + bytes_written,
3570                                     sizeof(capabilities) - bytes_written, "%s%s",
3571                                     bytes_written ? "," : "",
3572                                     mpi3mr_capabilities[i].name);
3573                 }
3574         }
3575
3576         ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3577                  protocol, capabilities);
3578 }
3579
3580 /**
3581  * mpi3mr_cleanup_resources - Free PCI resources
3582  * @mrioc: Adapter instance reference
3583  *
3584  * Unmap PCI device memory and disable PCI device.
3585  *
3586  * Return: 0 on success and non-zero on failure.
3587  */
3588 void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3589 {
3590         struct pci_dev *pdev = mrioc->pdev;
3591
3592         mpi3mr_cleanup_isr(mrioc);
3593
3594         if (mrioc->sysif_regs) {
3595                 iounmap((void __iomem *)mrioc->sysif_regs);
3596                 mrioc->sysif_regs = NULL;
3597         }
3598
3599         if (pci_is_enabled(pdev)) {
3600                 if (mrioc->bars)
3601                         pci_release_selected_regions(pdev, mrioc->bars);
3602                 pci_disable_device(pdev);
3603         }
3604 }
3605
3606 /**
3607  * mpi3mr_setup_resources - Enable PCI resources
3608  * @mrioc: Adapter instance reference
3609  *
3610  * Enable PCI device memory, MSI-x registers and set DMA mask.
3611  *
3612  * Return: 0 on success and non-zero on failure.
3613  */
3614 int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3615 {
3616         struct pci_dev *pdev = mrioc->pdev;
3617         u32 memap_sz = 0;
3618         int i, retval = 0, capb = 0;
3619         u16 message_control;
3620         u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3621             ((sizeof(dma_addr_t) > 4) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
3622
3623         if (pci_enable_device_mem(pdev)) {
3624                 ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3625                 retval = -ENODEV;
3626                 goto out_failed;
3627         }
3628
3629         capb = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
3630         if (!capb) {
3631                 ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3632                 retval = -ENODEV;
3633                 goto out_failed;
3634         }
3635         mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3636
3637         if (pci_request_selected_regions(pdev, mrioc->bars,
3638             mrioc->driver_name)) {
3639                 ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3640                 retval = -ENODEV;
3641                 goto out_failed;
3642         }
3643
3644         for (i = 0; (i < DEVICE_COUNT_RESOURCE); i++) {
3645                 if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3646                         mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3647                         memap_sz = pci_resource_len(pdev, i);
3648                         mrioc->sysif_regs =
3649                             ioremap(mrioc->sysif_regs_phys, memap_sz);
3650                         break;
3651                 }
3652         }
3653
3654         pci_set_master(pdev);
3655
3656         retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);
3657         if (retval) {
3658                 if (dma_mask != DMA_BIT_MASK(32)) {
3659                         ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3660                         dma_mask = DMA_BIT_MASK(32);
3661                         retval = dma_set_mask_and_coherent(&pdev->dev,
3662                             dma_mask);
3663                 }
3664                 if (retval) {
3665                         mrioc->dma_mask = 0;
3666                         ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3667                         goto out_failed;
3668                 }
3669         }
3670         mrioc->dma_mask = dma_mask;
3671
3672         if (!mrioc->sysif_regs) {
3673                 ioc_err(mrioc,
3674                     "Unable to map adapter memory or resource not found\n");
3675                 retval = -EINVAL;
3676                 goto out_failed;
3677         }
3678
3679         pci_read_config_word(pdev, capb + 2, &message_control);
3680         mrioc->msix_count = (message_control & 0x3FF) + 1;
3681
3682         pci_save_state(pdev);
3683
3684         pci_set_drvdata(pdev, mrioc->shost);
3685
3686         mpi3mr_ioc_disable_intr(mrioc);
3687
3688         ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3689             (unsigned long long)mrioc->sysif_regs_phys,
3690             mrioc->sysif_regs, memap_sz);
3691         ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3692             mrioc->msix_count);
3693
3694         if (!reset_devices && poll_queues > 0)
3695                 mrioc->requested_poll_qcount = min_t(int, poll_queues,
3696                                 mrioc->msix_count - 2);
3697         return retval;
3698
3699 out_failed:
3700         mpi3mr_cleanup_resources(mrioc);
3701         return retval;
3702 }
3703
3704 /**
3705  * mpi3mr_enable_events - Enable required events
3706  * @mrioc: Adapter instance reference
3707  *
3708  * This routine unmasks the events required by the driver by
3709  * sennding appropriate event mask bitmapt through an event
3710  * notification request.
3711  *
3712  * Return: 0 on success and non-zero on failure.
3713  */
3714 static int mpi3mr_enable_events(struct mpi3mr_ioc *mrioc)
3715 {
3716         int retval = 0;
3717         u32  i;
3718
3719         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3720                 mrioc->event_masks[i] = -1;
3721
3722         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3723         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3724         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3725         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3726         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_ADDED);
3727         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3728         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3729         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3730         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3731         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3732         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3733         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PREPARE_FOR_RESET);
3734         mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3735         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3736
3737         retval = mpi3mr_issue_event_notification(mrioc);
3738         if (retval)
3739                 ioc_err(mrioc, "failed to issue event notification %d\n",
3740                     retval);
3741         return retval;
3742 }
3743
3744 /**
3745  * mpi3mr_init_ioc - Initialize the controller
3746  * @mrioc: Adapter instance reference
3747  *
3748  * This the controller initialization routine, executed either
3749  * after soft reset or from pci probe callback.
3750  * Setup the required resources, memory map the controller
3751  * registers, create admin and operational reply queue pairs,
3752  * allocate required memory for reply pool, sense buffer pool,
3753  * issue IOC init request to the firmware, unmask the events and
3754  * issue port enable to discover SAS/SATA/NVMe devies and RAID
3755  * volumes.
3756  *
3757  * Return: 0 on success and non-zero on failure.
3758  */
3759 int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc)
3760 {
3761         int retval = 0;
3762         u8 retry = 0;
3763         struct mpi3_ioc_facts_data facts_data;
3764         u32 sz;
3765
3766 retry_init:
3767         retval = mpi3mr_bring_ioc_ready(mrioc);
3768         if (retval) {
3769                 ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3770                     retval);
3771                 goto out_failed_noretry;
3772         }
3773
3774         retval = mpi3mr_setup_isr(mrioc, 1);
3775         if (retval) {
3776                 ioc_err(mrioc, "Failed to setup ISR error %d\n",
3777                     retval);
3778                 goto out_failed_noretry;
3779         }
3780
3781         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3782         if (retval) {
3783                 ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3784                     retval);
3785                 goto out_failed;
3786         }
3787
3788         mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
3789
3790         mrioc->num_io_throttle_group = mrioc->facts.max_io_throttle_group;
3791         atomic_set(&mrioc->pend_large_data_sz, 0);
3792
3793         if (reset_devices)
3794                 mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
3795                     MPI3MR_HOST_IOS_KDUMP);
3796
3797         if (!(mrioc->facts.ioc_capabilities &
3798             MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED)) {
3799                 mrioc->sas_transport_enabled = 1;
3800                 mrioc->scsi_device_channel = 1;
3801                 mrioc->shost->max_channel = 1;
3802                 mrioc->shost->transportt = mpi3mr_transport_template;
3803         }
3804
3805         mrioc->reply_sz = mrioc->facts.reply_sz;
3806
3807         retval = mpi3mr_check_reset_dma_mask(mrioc);
3808         if (retval) {
3809                 ioc_err(mrioc, "Resetting dma mask failed %d\n",
3810                     retval);
3811                 goto out_failed_noretry;
3812         }
3813
3814         mpi3mr_print_ioc_info(mrioc);
3815
3816         dprint_init(mrioc, "allocating config page buffers\n");
3817         mrioc->cfg_page = dma_alloc_coherent(&mrioc->pdev->dev,
3818             MPI3MR_DEFAULT_CFG_PAGE_SZ, &mrioc->cfg_page_dma, GFP_KERNEL);
3819         if (!mrioc->cfg_page)
3820                 goto out_failed_noretry;
3821
3822         mrioc->cfg_page_sz = MPI3MR_DEFAULT_CFG_PAGE_SZ;
3823
3824         retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3825         if (retval) {
3826                 ioc_err(mrioc,
3827                     "%s :Failed to allocated reply sense buffers %d\n",
3828                     __func__, retval);
3829                 goto out_failed_noretry;
3830         }
3831
3832         retval = mpi3mr_alloc_chain_bufs(mrioc);
3833         if (retval) {
3834                 ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
3835                     retval);
3836                 goto out_failed_noretry;
3837         }
3838
3839         retval = mpi3mr_issue_iocinit(mrioc);
3840         if (retval) {
3841                 ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
3842                     retval);
3843                 goto out_failed;
3844         }
3845
3846         retval = mpi3mr_print_pkg_ver(mrioc);
3847         if (retval) {
3848                 ioc_err(mrioc, "failed to get package version\n");
3849                 goto out_failed;
3850         }
3851
3852         retval = mpi3mr_setup_isr(mrioc, 0);
3853         if (retval) {
3854                 ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
3855                     retval);
3856                 goto out_failed_noretry;
3857         }
3858
3859         retval = mpi3mr_create_op_queues(mrioc);
3860         if (retval) {
3861                 ioc_err(mrioc, "Failed to create OpQueues error %d\n",
3862                     retval);
3863                 goto out_failed;
3864         }
3865
3866         if (!mrioc->pel_seqnum_virt) {
3867                 dprint_init(mrioc, "allocating memory for pel_seqnum_virt\n");
3868                 mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
3869                 mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
3870                     mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
3871                     GFP_KERNEL);
3872                 if (!mrioc->pel_seqnum_virt) {
3873                         retval = -ENOMEM;
3874                         goto out_failed_noretry;
3875                 }
3876         }
3877
3878         if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) {
3879                 dprint_init(mrioc, "allocating memory for throttle groups\n");
3880                 sz = sizeof(struct mpi3mr_throttle_group_info);
3881                 mrioc->throttle_groups = kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL);
3882                 if (!mrioc->throttle_groups)
3883                         goto out_failed_noretry;
3884         }
3885
3886         retval = mpi3mr_enable_events(mrioc);
3887         if (retval) {
3888                 ioc_err(mrioc, "failed to enable events %d\n",
3889                     retval);
3890                 goto out_failed;
3891         }
3892
3893         ioc_info(mrioc, "controller initialization completed successfully\n");
3894         return retval;
3895 out_failed:
3896         if (retry < 2) {
3897                 retry++;
3898                 ioc_warn(mrioc, "retrying controller initialization, retry_count:%d\n",
3899                     retry);
3900                 mpi3mr_memset_buffers(mrioc);
3901                 goto retry_init;
3902         }
3903 out_failed_noretry:
3904         ioc_err(mrioc, "controller initialization failed\n");
3905         mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
3906             MPI3MR_RESET_FROM_CTLR_CLEANUP);
3907         mrioc->unrecoverable = 1;
3908         return retval;
3909 }
3910
3911 /**
3912  * mpi3mr_reinit_ioc - Re-Initialize the controller
3913  * @mrioc: Adapter instance reference
3914  * @is_resume: Called from resume or reset path
3915  *
3916  * This the controller re-initialization routine, executed from
3917  * the soft reset handler or resume callback. Creates
3918  * operational reply queue pairs, allocate required memory for
3919  * reply pool, sense buffer pool, issue IOC init request to the
3920  * firmware, unmask the events and issue port enable to discover
3921  * SAS/SATA/NVMe devices and RAID volumes.
3922  *
3923  * Return: 0 on success and non-zero on failure.
3924  */
3925 int mpi3mr_reinit_ioc(struct mpi3mr_ioc *mrioc, u8 is_resume)
3926 {
3927         int retval = 0;
3928         u8 retry = 0;
3929         struct mpi3_ioc_facts_data facts_data;
3930         u32 pe_timeout, ioc_status;
3931
3932 retry_init:
3933         pe_timeout =
3934             (MPI3MR_PORTENABLE_TIMEOUT / MPI3MR_PORTENABLE_POLL_INTERVAL);
3935
3936         dprint_reset(mrioc, "bringing up the controller to ready state\n");
3937         retval = mpi3mr_bring_ioc_ready(mrioc);
3938         if (retval) {
3939                 ioc_err(mrioc, "failed to bring to ready state\n");
3940                 goto out_failed_noretry;
3941         }
3942
3943         if (is_resume) {
3944                 dprint_reset(mrioc, "setting up single ISR\n");
3945                 retval = mpi3mr_setup_isr(mrioc, 1);
3946                 if (retval) {
3947                         ioc_err(mrioc, "failed to setup ISR\n");
3948                         goto out_failed_noretry;
3949                 }
3950         } else
3951                 mpi3mr_ioc_enable_intr(mrioc);
3952
3953         dprint_reset(mrioc, "getting ioc_facts\n");
3954         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3955         if (retval) {
3956                 ioc_err(mrioc, "failed to get ioc_facts\n");
3957                 goto out_failed;
3958         }
3959
3960         dprint_reset(mrioc, "validating ioc_facts\n");
3961         retval = mpi3mr_revalidate_factsdata(mrioc);
3962         if (retval) {
3963                 ioc_err(mrioc, "failed to revalidate ioc_facts data\n");
3964                 goto out_failed_noretry;
3965         }
3966
3967         mpi3mr_print_ioc_info(mrioc);
3968
3969         dprint_reset(mrioc, "sending ioc_init\n");
3970         retval = mpi3mr_issue_iocinit(mrioc);
3971         if (retval) {
3972                 ioc_err(mrioc, "failed to send ioc_init\n");
3973                 goto out_failed;
3974         }
3975
3976         dprint_reset(mrioc, "getting package version\n");
3977         retval = mpi3mr_print_pkg_ver(mrioc);
3978         if (retval) {
3979                 ioc_err(mrioc, "failed to get package version\n");
3980                 goto out_failed;
3981         }
3982
3983         if (is_resume) {
3984                 dprint_reset(mrioc, "setting up multiple ISR\n");
3985                 retval = mpi3mr_setup_isr(mrioc, 0);
3986                 if (retval) {
3987                         ioc_err(mrioc, "failed to re-setup ISR\n");
3988                         goto out_failed_noretry;
3989                 }
3990         }
3991
3992         dprint_reset(mrioc, "creating operational queue pairs\n");
3993         retval = mpi3mr_create_op_queues(mrioc);
3994         if (retval) {
3995                 ioc_err(mrioc, "failed to create operational queue pairs\n");
3996                 goto out_failed;
3997         }
3998
3999         if (!mrioc->pel_seqnum_virt) {
4000                 dprint_reset(mrioc, "allocating memory for pel_seqnum_virt\n");
4001                 mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
4002                 mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
4003                     mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
4004                     GFP_KERNEL);
4005                 if (!mrioc->pel_seqnum_virt) {
4006                         retval = -ENOMEM;
4007                         goto out_failed_noretry;
4008                 }
4009         }
4010
4011         if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {
4012                 ioc_err(mrioc,
4013                     "cannot create minimum number of operational queues expected:%d created:%d\n",
4014                     mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
4015                 goto out_failed_noretry;
4016         }
4017
4018         dprint_reset(mrioc, "enabling events\n");
4019         retval = mpi3mr_enable_events(mrioc);
4020         if (retval) {
4021                 ioc_err(mrioc, "failed to enable events\n");
4022                 goto out_failed;
4023         }
4024
4025         mrioc->device_refresh_on = 1;
4026         mpi3mr_add_event_wait_for_device_refresh(mrioc);
4027
4028         ioc_info(mrioc, "sending port enable\n");
4029         retval = mpi3mr_issue_port_enable(mrioc, 1);
4030         if (retval) {
4031                 ioc_err(mrioc, "failed to issue port enable\n");
4032                 goto out_failed;
4033         }
4034         do {
4035                 ssleep(MPI3MR_PORTENABLE_POLL_INTERVAL);
4036                 if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED)
4037                         break;
4038                 if (!pci_device_is_present(mrioc->pdev))
4039                         mrioc->unrecoverable = 1;
4040                 if (mrioc->unrecoverable) {
4041                         retval = -1;
4042                         goto out_failed_noretry;
4043                 }
4044                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4045                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
4046                     (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
4047                         mpi3mr_print_fault_info(mrioc);
4048                         mrioc->init_cmds.is_waiting = 0;
4049                         mrioc->init_cmds.callback = NULL;
4050                         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4051                         goto out_failed;
4052                 }
4053         } while (--pe_timeout);
4054
4055         if (!pe_timeout) {
4056                 ioc_err(mrioc, "port enable timed out\n");
4057                 mpi3mr_check_rh_fault_ioc(mrioc,
4058                     MPI3MR_RESET_FROM_PE_TIMEOUT);
4059                 mrioc->init_cmds.is_waiting = 0;
4060                 mrioc->init_cmds.callback = NULL;
4061                 mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4062                 goto out_failed;
4063         } else if (mrioc->scan_failed) {
4064                 ioc_err(mrioc,
4065                     "port enable failed with status=0x%04x\n",
4066                     mrioc->scan_failed);
4067         } else
4068                 ioc_info(mrioc, "port enable completed successfully\n");
4069
4070         ioc_info(mrioc, "controller %s completed successfully\n",
4071             (is_resume)?"resume":"re-initialization");
4072         return retval;
4073 out_failed:
4074         if (retry < 2) {
4075                 retry++;
4076                 ioc_warn(mrioc, "retrying controller %s, retry_count:%d\n",
4077                     (is_resume)?"resume":"re-initialization", retry);
4078                 mpi3mr_memset_buffers(mrioc);
4079                 goto retry_init;
4080         }
4081 out_failed_noretry:
4082         ioc_err(mrioc, "controller %s is failed\n",
4083             (is_resume)?"resume":"re-initialization");
4084         mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
4085             MPI3MR_RESET_FROM_CTLR_CLEANUP);
4086         mrioc->unrecoverable = 1;
4087         return retval;
4088 }
4089
4090 /**
4091  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
4092  *                                      segments
4093  * @mrioc: Adapter instance reference
4094  * @qidx: Operational reply queue index
4095  *
4096  * Return: Nothing.
4097  */
4098 static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4099 {
4100         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
4101         struct segments *segments;
4102         int i, size;
4103
4104         if (!op_reply_q->q_segments)
4105                 return;
4106
4107         size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
4108         segments = op_reply_q->q_segments;
4109         for (i = 0; i < op_reply_q->num_segments; i++)
4110                 memset(segments[i].segment, 0, size);
4111 }
4112
4113 /**
4114  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
4115  *                                      segments
4116  * @mrioc: Adapter instance reference
4117  * @qidx: Operational request queue index
4118  *
4119  * Return: Nothing.
4120  */
4121 static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4122 {
4123         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
4124         struct segments *segments;
4125         int i, size;
4126
4127         if (!op_req_q->q_segments)
4128                 return;
4129
4130         size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
4131         segments = op_req_q->q_segments;
4132         for (i = 0; i < op_req_q->num_segments; i++)
4133                 memset(segments[i].segment, 0, size);
4134 }
4135
4136 /**
4137  * mpi3mr_memset_buffers - memset memory for a controller
4138  * @mrioc: Adapter instance reference
4139  *
4140  * clear all the memory allocated for a controller, typically
4141  * called post reset to reuse the memory allocated during the
4142  * controller init.
4143  *
4144  * Return: Nothing.
4145  */
4146 void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
4147 {
4148         u16 i;
4149         struct mpi3mr_throttle_group_info *tg;
4150
4151         mrioc->change_count = 0;
4152         mrioc->active_poll_qcount = 0;
4153         mrioc->default_qcount = 0;
4154         if (mrioc->admin_req_base)
4155                 memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
4156         if (mrioc->admin_reply_base)
4157                 memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
4158
4159         if (mrioc->init_cmds.reply) {
4160                 memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
4161                 memset(mrioc->bsg_cmds.reply, 0,
4162                     sizeof(*mrioc->bsg_cmds.reply));
4163                 memset(mrioc->host_tm_cmds.reply, 0,
4164                     sizeof(*mrioc->host_tm_cmds.reply));
4165                 memset(mrioc->pel_cmds.reply, 0,
4166                     sizeof(*mrioc->pel_cmds.reply));
4167                 memset(mrioc->pel_abort_cmd.reply, 0,
4168                     sizeof(*mrioc->pel_abort_cmd.reply));
4169                 memset(mrioc->transport_cmds.reply, 0,
4170                     sizeof(*mrioc->transport_cmds.reply));
4171                 for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
4172                         memset(mrioc->dev_rmhs_cmds[i].reply, 0,
4173                             sizeof(*mrioc->dev_rmhs_cmds[i].reply));
4174                 for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++)
4175                         memset(mrioc->evtack_cmds[i].reply, 0,
4176                             sizeof(*mrioc->evtack_cmds[i].reply));
4177                 bitmap_clear(mrioc->removepend_bitmap, 0,
4178                              mrioc->dev_handle_bitmap_bits);
4179                 bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);
4180                 bitmap_clear(mrioc->evtack_cmds_bitmap, 0,
4181                              MPI3MR_NUM_EVTACKCMD);
4182         }
4183
4184         for (i = 0; i < mrioc->num_queues; i++) {
4185                 mrioc->op_reply_qinfo[i].qid = 0;
4186                 mrioc->op_reply_qinfo[i].ci = 0;
4187                 mrioc->op_reply_qinfo[i].num_replies = 0;
4188                 mrioc->op_reply_qinfo[i].ephase = 0;
4189                 atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
4190                 atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);
4191                 mpi3mr_memset_op_reply_q_buffers(mrioc, i);
4192
4193                 mrioc->req_qinfo[i].ci = 0;
4194                 mrioc->req_qinfo[i].pi = 0;
4195                 mrioc->req_qinfo[i].num_requests = 0;
4196                 mrioc->req_qinfo[i].qid = 0;
4197                 mrioc->req_qinfo[i].reply_qid = 0;
4198                 spin_lock_init(&mrioc->req_qinfo[i].q_lock);
4199                 mpi3mr_memset_op_req_q_buffers(mrioc, i);
4200         }
4201
4202         atomic_set(&mrioc->pend_large_data_sz, 0);
4203         if (mrioc->throttle_groups) {
4204                 tg = mrioc->throttle_groups;
4205                 for (i = 0; i < mrioc->num_io_throttle_group; i++, tg++) {
4206                         tg->id = 0;
4207                         tg->fw_qd = 0;
4208                         tg->modified_qd = 0;
4209                         tg->io_divert = 0;
4210                         tg->need_qd_reduction = 0;
4211                         tg->high = 0;
4212                         tg->low = 0;
4213                         tg->qd_reduction = 0;
4214                         atomic_set(&tg->pend_large_data_sz, 0);
4215                 }
4216         }
4217 }
4218
4219 /**
4220  * mpi3mr_free_mem - Free memory allocated for a controller
4221  * @mrioc: Adapter instance reference
4222  *
4223  * Free all the memory allocated for a controller.
4224  *
4225  * Return: Nothing.
4226  */
4227 void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
4228 {
4229         u16 i;
4230         struct mpi3mr_intr_info *intr_info;
4231
4232         mpi3mr_free_enclosure_list(mrioc);
4233
4234         if (mrioc->sense_buf_pool) {
4235                 if (mrioc->sense_buf)
4236                         dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,
4237                             mrioc->sense_buf_dma);
4238                 dma_pool_destroy(mrioc->sense_buf_pool);
4239                 mrioc->sense_buf = NULL;
4240                 mrioc->sense_buf_pool = NULL;
4241         }
4242         if (mrioc->sense_buf_q_pool) {
4243                 if (mrioc->sense_buf_q)
4244                         dma_pool_free(mrioc->sense_buf_q_pool,
4245                             mrioc->sense_buf_q, mrioc->sense_buf_q_dma);
4246                 dma_pool_destroy(mrioc->sense_buf_q_pool);
4247                 mrioc->sense_buf_q = NULL;
4248                 mrioc->sense_buf_q_pool = NULL;
4249         }
4250
4251         if (mrioc->reply_buf_pool) {
4252                 if (mrioc->reply_buf)
4253                         dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,
4254                             mrioc->reply_buf_dma);
4255                 dma_pool_destroy(mrioc->reply_buf_pool);
4256                 mrioc->reply_buf = NULL;
4257                 mrioc->reply_buf_pool = NULL;
4258         }
4259         if (mrioc->reply_free_q_pool) {
4260                 if (mrioc->reply_free_q)
4261                         dma_pool_free(mrioc->reply_free_q_pool,
4262                             mrioc->reply_free_q, mrioc->reply_free_q_dma);
4263                 dma_pool_destroy(mrioc->reply_free_q_pool);
4264                 mrioc->reply_free_q = NULL;
4265                 mrioc->reply_free_q_pool = NULL;
4266         }
4267
4268         for (i = 0; i < mrioc->num_op_req_q; i++)
4269                 mpi3mr_free_op_req_q_segments(mrioc, i);
4270
4271         for (i = 0; i < mrioc->num_op_reply_q; i++)
4272                 mpi3mr_free_op_reply_q_segments(mrioc, i);
4273
4274         for (i = 0; i < mrioc->intr_info_count; i++) {
4275                 intr_info = mrioc->intr_info + i;
4276                 intr_info->op_reply_q = NULL;
4277         }
4278
4279         kfree(mrioc->req_qinfo);
4280         mrioc->req_qinfo = NULL;
4281         mrioc->num_op_req_q = 0;
4282
4283         kfree(mrioc->op_reply_qinfo);
4284         mrioc->op_reply_qinfo = NULL;
4285         mrioc->num_op_reply_q = 0;
4286
4287         kfree(mrioc->init_cmds.reply);
4288         mrioc->init_cmds.reply = NULL;
4289
4290         kfree(mrioc->bsg_cmds.reply);
4291         mrioc->bsg_cmds.reply = NULL;
4292
4293         kfree(mrioc->host_tm_cmds.reply);
4294         mrioc->host_tm_cmds.reply = NULL;
4295
4296         kfree(mrioc->pel_cmds.reply);
4297         mrioc->pel_cmds.reply = NULL;
4298
4299         kfree(mrioc->pel_abort_cmd.reply);
4300         mrioc->pel_abort_cmd.reply = NULL;
4301
4302         for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4303                 kfree(mrioc->evtack_cmds[i].reply);
4304                 mrioc->evtack_cmds[i].reply = NULL;
4305         }
4306
4307         bitmap_free(mrioc->removepend_bitmap);
4308         mrioc->removepend_bitmap = NULL;
4309
4310         bitmap_free(mrioc->devrem_bitmap);
4311         mrioc->devrem_bitmap = NULL;
4312
4313         bitmap_free(mrioc->evtack_cmds_bitmap);
4314         mrioc->evtack_cmds_bitmap = NULL;
4315
4316         bitmap_free(mrioc->chain_bitmap);
4317         mrioc->chain_bitmap = NULL;
4318
4319         kfree(mrioc->transport_cmds.reply);
4320         mrioc->transport_cmds.reply = NULL;
4321
4322         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4323                 kfree(mrioc->dev_rmhs_cmds[i].reply);
4324                 mrioc->dev_rmhs_cmds[i].reply = NULL;
4325         }
4326
4327         if (mrioc->chain_buf_pool) {
4328                 for (i = 0; i < mrioc->chain_buf_count; i++) {
4329                         if (mrioc->chain_sgl_list[i].addr) {
4330                                 dma_pool_free(mrioc->chain_buf_pool,
4331                                     mrioc->chain_sgl_list[i].addr,
4332                                     mrioc->chain_sgl_list[i].dma_addr);
4333                                 mrioc->chain_sgl_list[i].addr = NULL;
4334                         }
4335                 }
4336                 dma_pool_destroy(mrioc->chain_buf_pool);
4337                 mrioc->chain_buf_pool = NULL;
4338         }
4339
4340         kfree(mrioc->chain_sgl_list);
4341         mrioc->chain_sgl_list = NULL;
4342
4343         if (mrioc->admin_reply_base) {
4344                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
4345                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
4346                 mrioc->admin_reply_base = NULL;
4347         }
4348         if (mrioc->admin_req_base) {
4349                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
4350                     mrioc->admin_req_base, mrioc->admin_req_dma);
4351                 mrioc->admin_req_base = NULL;
4352         }
4353
4354         if (mrioc->pel_seqnum_virt) {
4355                 dma_free_coherent(&mrioc->pdev->dev, mrioc->pel_seqnum_sz,
4356                     mrioc->pel_seqnum_virt, mrioc->pel_seqnum_dma);
4357                 mrioc->pel_seqnum_virt = NULL;
4358         }
4359
4360         kfree(mrioc->logdata_buf);
4361         mrioc->logdata_buf = NULL;
4362
4363 }
4364
4365 /**
4366  * mpi3mr_issue_ioc_shutdown - shutdown controller
4367  * @mrioc: Adapter instance reference
4368  *
4369  * Send shutodwn notification to the controller and wait for the
4370  * shutdown_timeout for it to be completed.
4371  *
4372  * Return: Nothing.
4373  */
4374 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
4375 {
4376         u32 ioc_config, ioc_status;
4377         u8 retval = 1;
4378         u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
4379
4380         ioc_info(mrioc, "Issuing shutdown Notification\n");
4381         if (mrioc->unrecoverable) {
4382                 ioc_warn(mrioc,
4383                     "IOC is unrecoverable shutdown is not issued\n");
4384                 return;
4385         }
4386         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4387         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4388             == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS) {
4389                 ioc_info(mrioc, "shutdown already in progress\n");
4390                 return;
4391         }
4392
4393         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4394         ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
4395         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
4396
4397         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
4398
4399         if (mrioc->facts.shutdown_timeout)
4400                 timeout = mrioc->facts.shutdown_timeout * 10;
4401
4402         do {
4403                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4404                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4405                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_COMPLETE) {
4406                         retval = 0;
4407                         break;
4408                 }
4409                 msleep(100);
4410         } while (--timeout);
4411
4412         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4413         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4414
4415         if (retval) {
4416                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4417                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS)
4418                         ioc_warn(mrioc,
4419                             "shutdown still in progress after timeout\n");
4420         }
4421
4422         ioc_info(mrioc,
4423             "Base IOC Sts/Config after %s shutdown is (0x%x)/(0x%x)\n",
4424             (!retval) ? "successful" : "failed", ioc_status,
4425             ioc_config);
4426 }
4427
4428 /**
4429  * mpi3mr_cleanup_ioc - Cleanup controller
4430  * @mrioc: Adapter instance reference
4431  *
4432  * controller cleanup handler, Message unit reset or soft reset
4433  * and shutdown notification is issued to the controller.
4434  *
4435  * Return: Nothing.
4436  */
4437 void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc)
4438 {
4439         enum mpi3mr_iocstate ioc_state;
4440
4441         dprint_exit(mrioc, "cleaning up the controller\n");
4442         mpi3mr_ioc_disable_intr(mrioc);
4443
4444         ioc_state = mpi3mr_get_iocstate(mrioc);
4445
4446         if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
4447             (ioc_state == MRIOC_STATE_READY)) {
4448                 if (mpi3mr_issue_and_process_mur(mrioc,
4449                     MPI3MR_RESET_FROM_CTLR_CLEANUP))
4450                         mpi3mr_issue_reset(mrioc,
4451                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
4452                             MPI3MR_RESET_FROM_MUR_FAILURE);
4453                 mpi3mr_issue_ioc_shutdown(mrioc);
4454         }
4455         dprint_exit(mrioc, "controller cleanup completed\n");
4456 }
4457
4458 /**
4459  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
4460  * @mrioc: Adapter instance reference
4461  * @cmdptr: Internal command tracker
4462  *
4463  * Complete an internal driver commands with state indicating it
4464  * is completed due to reset.
4465  *
4466  * Return: Nothing.
4467  */
4468 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
4469         struct mpi3mr_drv_cmd *cmdptr)
4470 {
4471         if (cmdptr->state & MPI3MR_CMD_PENDING) {
4472                 cmdptr->state |= MPI3MR_CMD_RESET;
4473                 cmdptr->state &= ~MPI3MR_CMD_PENDING;
4474                 if (cmdptr->is_waiting) {
4475                         complete(&cmdptr->done);
4476                         cmdptr->is_waiting = 0;
4477                 } else if (cmdptr->callback)
4478                         cmdptr->callback(mrioc, cmdptr);
4479         }
4480 }
4481
4482 /**
4483  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
4484  * @mrioc: Adapter instance reference
4485  *
4486  * Flush all internal driver commands post reset
4487  *
4488  * Return: Nothing.
4489  */
4490 void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
4491 {
4492         struct mpi3mr_drv_cmd *cmdptr;
4493         u8 i;
4494
4495         cmdptr = &mrioc->init_cmds;
4496         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4497
4498         cmdptr = &mrioc->cfg_cmds;
4499         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4500
4501         cmdptr = &mrioc->bsg_cmds;
4502         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4503         cmdptr = &mrioc->host_tm_cmds;
4504         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4505
4506         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4507                 cmdptr = &mrioc->dev_rmhs_cmds[i];
4508                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4509         }
4510
4511         for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4512                 cmdptr = &mrioc->evtack_cmds[i];
4513                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4514         }
4515
4516         cmdptr = &mrioc->pel_cmds;
4517         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4518
4519         cmdptr = &mrioc->pel_abort_cmd;
4520         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4521
4522         cmdptr = &mrioc->transport_cmds;
4523         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4524 }
4525
4526 /**
4527  * mpi3mr_pel_wait_post - Issue PEL Wait
4528  * @mrioc: Adapter instance reference
4529  * @drv_cmd: Internal command tracker
4530  *
4531  * Issue PEL Wait MPI request through admin queue and return.
4532  *
4533  * Return: Nothing.
4534  */
4535 static void mpi3mr_pel_wait_post(struct mpi3mr_ioc *mrioc,
4536         struct mpi3mr_drv_cmd *drv_cmd)
4537 {
4538         struct mpi3_pel_req_action_wait pel_wait;
4539
4540         mrioc->pel_abort_requested = false;
4541
4542         memset(&pel_wait, 0, sizeof(pel_wait));
4543         drv_cmd->state = MPI3MR_CMD_PENDING;
4544         drv_cmd->is_waiting = 0;
4545         drv_cmd->callback = mpi3mr_pel_wait_complete;
4546         drv_cmd->ioc_status = 0;
4547         drv_cmd->ioc_loginfo = 0;
4548         pel_wait.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4549         pel_wait.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4550         pel_wait.action = MPI3_PEL_ACTION_WAIT;
4551         pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);
4552         pel_wait.locale = cpu_to_le16(mrioc->pel_locale);
4553         pel_wait.class = cpu_to_le16(mrioc->pel_class);
4554         pel_wait.wait_time = MPI3_PEL_WAITTIME_INFINITE_WAIT;
4555         dprint_bsg_info(mrioc, "sending pel_wait seqnum(%d), class(%d), locale(0x%08x)\n",
4556             mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);
4557
4558         if (mpi3mr_admin_request_post(mrioc, &pel_wait, sizeof(pel_wait), 0)) {
4559                 dprint_bsg_err(mrioc,
4560                             "Issuing PELWait: Admin post failed\n");
4561                 drv_cmd->state = MPI3MR_CMD_NOTUSED;
4562                 drv_cmd->callback = NULL;
4563                 drv_cmd->retry_count = 0;
4564                 mrioc->pel_enabled = false;
4565         }
4566 }
4567
4568 /**
4569  * mpi3mr_pel_get_seqnum_post - Issue PEL Get Sequence number
4570  * @mrioc: Adapter instance reference
4571  * @drv_cmd: Internal command tracker
4572  *
4573  * Issue PEL get sequence number MPI request through admin queue
4574  * and return.
4575  *
4576  * Return: 0 on success, non-zero on failure.
4577  */
4578 int mpi3mr_pel_get_seqnum_post(struct mpi3mr_ioc *mrioc,
4579         struct mpi3mr_drv_cmd *drv_cmd)
4580 {
4581         struct mpi3_pel_req_action_get_sequence_numbers pel_getseq_req;
4582         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
4583         int retval = 0;
4584
4585         memset(&pel_getseq_req, 0, sizeof(pel_getseq_req));
4586         mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;
4587         mrioc->pel_cmds.is_waiting = 0;
4588         mrioc->pel_cmds.ioc_status = 0;
4589         mrioc->pel_cmds.ioc_loginfo = 0;
4590         mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;
4591         pel_getseq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4592         pel_getseq_req.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4593         pel_getseq_req.action = MPI3_PEL_ACTION_GET_SEQNUM;
4594         mpi3mr_add_sg_single(&pel_getseq_req.sgl, sgl_flags,
4595             mrioc->pel_seqnum_sz, mrioc->pel_seqnum_dma);
4596
4597         retval = mpi3mr_admin_request_post(mrioc, &pel_getseq_req,
4598                         sizeof(pel_getseq_req), 0);
4599         if (retval) {
4600                 if (drv_cmd) {
4601                         drv_cmd->state = MPI3MR_CMD_NOTUSED;
4602                         drv_cmd->callback = NULL;
4603                         drv_cmd->retry_count = 0;
4604                 }
4605                 mrioc->pel_enabled = false;
4606         }
4607
4608         return retval;
4609 }
4610
4611 /**
4612  * mpi3mr_pel_wait_complete - PELWait Completion callback
4613  * @mrioc: Adapter instance reference
4614  * @drv_cmd: Internal command tracker
4615  *
4616  * This is a callback handler for the PELWait request and
4617  * firmware completes a PELWait request when it is aborted or a
4618  * new PEL entry is available. This sends AEN to the application
4619  * and if the PELwait completion is not due to PELAbort then
4620  * this will send a request for new PEL Sequence number
4621  *
4622  * Return: Nothing.
4623  */
4624 static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
4625         struct mpi3mr_drv_cmd *drv_cmd)
4626 {
4627         struct mpi3_pel_reply *pel_reply = NULL;
4628         u16 ioc_status, pe_log_status;
4629         bool do_retry = false;
4630
4631         if (drv_cmd->state & MPI3MR_CMD_RESET)
4632                 goto cleanup_drv_cmd;
4633
4634         ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4635         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4636                 ioc_err(mrioc, "%s: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
4637                         __func__, ioc_status, drv_cmd->ioc_loginfo);
4638                 dprint_bsg_err(mrioc,
4639                     "pel_wait: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4640                     ioc_status, drv_cmd->ioc_loginfo);
4641                 do_retry = true;
4642         }
4643
4644         if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4645                 pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4646
4647         if (!pel_reply) {
4648                 dprint_bsg_err(mrioc,
4649                     "pel_wait: failed due to no reply\n");
4650                 goto out_failed;
4651         }
4652
4653         pe_log_status = le16_to_cpu(pel_reply->pe_log_status);
4654         if ((pe_log_status != MPI3_PEL_STATUS_SUCCESS) &&
4655             (pe_log_status != MPI3_PEL_STATUS_ABORTED)) {
4656                 ioc_err(mrioc, "%s: Failed pe_log_status(0x%04x)\n",
4657                         __func__, pe_log_status);
4658                 dprint_bsg_err(mrioc,
4659                     "pel_wait: failed due to pel_log_status(0x%04x)\n",
4660                     pe_log_status);
4661                 do_retry = true;
4662         }
4663
4664         if (do_retry) {
4665                 if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4666                         drv_cmd->retry_count++;
4667                         dprint_bsg_err(mrioc, "pel_wait: retrying(%d)\n",
4668                             drv_cmd->retry_count);
4669                         mpi3mr_pel_wait_post(mrioc, drv_cmd);
4670                         return;
4671                 }
4672                 dprint_bsg_err(mrioc,
4673                     "pel_wait: failed after all retries(%d)\n",
4674                     drv_cmd->retry_count);
4675                 goto out_failed;
4676         }
4677         atomic64_inc(&event_counter);
4678         if (!mrioc->pel_abort_requested) {
4679                 mrioc->pel_cmds.retry_count = 0;
4680                 mpi3mr_pel_get_seqnum_post(mrioc, &mrioc->pel_cmds);
4681         }
4682
4683         return;
4684 out_failed:
4685         mrioc->pel_enabled = false;
4686 cleanup_drv_cmd:
4687         drv_cmd->state = MPI3MR_CMD_NOTUSED;
4688         drv_cmd->callback = NULL;
4689         drv_cmd->retry_count = 0;
4690 }
4691
4692 /**
4693  * mpi3mr_pel_get_seqnum_complete - PELGetSeqNum Completion callback
4694  * @mrioc: Adapter instance reference
4695  * @drv_cmd: Internal command tracker
4696  *
4697  * This is a callback handler for the PEL get sequence number
4698  * request and a new PEL wait request will be issued to the
4699  * firmware from this
4700  *
4701  * Return: Nothing.
4702  */
4703 void mpi3mr_pel_get_seqnum_complete(struct mpi3mr_ioc *mrioc,
4704         struct mpi3mr_drv_cmd *drv_cmd)
4705 {
4706         struct mpi3_pel_reply *pel_reply = NULL;
4707         struct mpi3_pel_seq *pel_seqnum_virt;
4708         u16 ioc_status;
4709         bool do_retry = false;
4710
4711         pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;
4712
4713         if (drv_cmd->state & MPI3MR_CMD_RESET)
4714                 goto cleanup_drv_cmd;
4715
4716         ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4717         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4718                 dprint_bsg_err(mrioc,
4719                     "pel_get_seqnum: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4720                     ioc_status, drv_cmd->ioc_loginfo);
4721                 do_retry = true;
4722         }
4723
4724         if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4725                 pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4726         if (!pel_reply) {
4727                 dprint_bsg_err(mrioc,
4728                     "pel_get_seqnum: failed due to no reply\n");
4729                 goto out_failed;
4730         }
4731
4732         if (le16_to_cpu(pel_reply->pe_log_status) != MPI3_PEL_STATUS_SUCCESS) {
4733                 dprint_bsg_err(mrioc,
4734                     "pel_get_seqnum: failed due to pel_log_status(0x%04x)\n",
4735                     le16_to_cpu(pel_reply->pe_log_status));
4736                 do_retry = true;
4737         }
4738
4739         if (do_retry) {
4740                 if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4741                         drv_cmd->retry_count++;
4742                         dprint_bsg_err(mrioc,
4743                             "pel_get_seqnum: retrying(%d)\n",
4744                             drv_cmd->retry_count);
4745                         mpi3mr_pel_get_seqnum_post(mrioc, drv_cmd);
4746                         return;
4747                 }
4748
4749                 dprint_bsg_err(mrioc,
4750                     "pel_get_seqnum: failed after all retries(%d)\n",
4751                     drv_cmd->retry_count);
4752                 goto out_failed;
4753         }
4754         mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;
4755         drv_cmd->retry_count = 0;
4756         mpi3mr_pel_wait_post(mrioc, drv_cmd);
4757
4758         return;
4759 out_failed:
4760         mrioc->pel_enabled = false;
4761 cleanup_drv_cmd:
4762         drv_cmd->state = MPI3MR_CMD_NOTUSED;
4763         drv_cmd->callback = NULL;
4764         drv_cmd->retry_count = 0;
4765 }
4766
4767 /**
4768  * mpi3mr_soft_reset_handler - Reset the controller
4769  * @mrioc: Adapter instance reference
4770  * @reset_reason: Reset reason code
4771  * @snapdump: Flag to generate snapdump in firmware or not
4772  *
4773  * This is an handler for recovering controller by issuing soft
4774  * reset are diag fault reset.  This is a blocking function and
4775  * when one reset is executed if any other resets they will be
4776  * blocked. All BSG requests will be blocked during the reset. If
4777  * controller reset is successful then the controller will be
4778  * reinitalized, otherwise the controller will be marked as not
4779  * recoverable
4780  *
4781  * In snapdump bit is set, the controller is issued with diag
4782  * fault reset so that the firmware can create a snap dump and
4783  * post that the firmware will result in F000 fault and the
4784  * driver will issue soft reset to recover from that.
4785  *
4786  * Return: 0 on success, non-zero on failure.
4787  */
4788 int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
4789         u32 reset_reason, u8 snapdump)
4790 {
4791         int retval = 0, i;
4792         unsigned long flags;
4793         u32 host_diagnostic, timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
4794
4795         /* Block the reset handler until diag save in progress*/
4796         dprint_reset(mrioc,
4797             "soft_reset_handler: check and block on diagsave_timeout(%d)\n",
4798             mrioc->diagsave_timeout);
4799         while (mrioc->diagsave_timeout)
4800                 ssleep(1);
4801         /*
4802          * Block new resets until the currently executing one is finished and
4803          * return the status of the existing reset for all blocked resets
4804          */
4805         dprint_reset(mrioc, "soft_reset_handler: acquiring reset_mutex\n");
4806         if (!mutex_trylock(&mrioc->reset_mutex)) {
4807                 ioc_info(mrioc,
4808                     "controller reset triggered by %s is blocked due to another reset in progress\n",
4809                     mpi3mr_reset_rc_name(reset_reason));
4810                 do {
4811                         ssleep(1);
4812                 } while (mrioc->reset_in_progress == 1);
4813                 ioc_info(mrioc,
4814                     "returning previous reset result(%d) for the reset triggered by %s\n",
4815                     mrioc->prev_reset_result,
4816                     mpi3mr_reset_rc_name(reset_reason));
4817                 return mrioc->prev_reset_result;
4818         }
4819         ioc_info(mrioc, "controller reset is triggered by %s\n",
4820             mpi3mr_reset_rc_name(reset_reason));
4821
4822         mrioc->device_refresh_on = 0;
4823         mrioc->reset_in_progress = 1;
4824         mrioc->stop_bsgs = 1;
4825         mrioc->prev_reset_result = -1;
4826
4827         if ((!snapdump) && (reset_reason != MPI3MR_RESET_FROM_FAULT_WATCH) &&
4828             (reset_reason != MPI3MR_RESET_FROM_FIRMWARE) &&
4829             (reset_reason != MPI3MR_RESET_FROM_CIACTIV_FAULT)) {
4830                 for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
4831                         mrioc->event_masks[i] = -1;
4832
4833                 dprint_reset(mrioc, "soft_reset_handler: masking events\n");
4834                 mpi3mr_issue_event_notification(mrioc);
4835         }
4836
4837         mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
4838
4839         mpi3mr_ioc_disable_intr(mrioc);
4840
4841         if (snapdump) {
4842                 mpi3mr_set_diagsave(mrioc);
4843                 retval = mpi3mr_issue_reset(mrioc,
4844                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
4845                 if (!retval) {
4846                         do {
4847                                 host_diagnostic =
4848                                     readl(&mrioc->sysif_regs->host_diagnostic);
4849                                 if (!(host_diagnostic &
4850                                     MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
4851                                         break;
4852                                 msleep(100);
4853                         } while (--timeout);
4854                 }
4855         }
4856
4857         retval = mpi3mr_issue_reset(mrioc,
4858             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, reset_reason);
4859         if (retval) {
4860                 ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
4861                 goto out;
4862         }
4863         if (mrioc->num_io_throttle_group !=
4864             mrioc->facts.max_io_throttle_group) {
4865                 ioc_err(mrioc,
4866                     "max io throttle group doesn't match old(%d), new(%d)\n",
4867                     mrioc->num_io_throttle_group,
4868                     mrioc->facts.max_io_throttle_group);
4869                 retval = -EPERM;
4870                 goto out;
4871         }
4872
4873         mpi3mr_flush_delayed_cmd_lists(mrioc);
4874         mpi3mr_flush_drv_cmds(mrioc);
4875         bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);
4876         bitmap_clear(mrioc->removepend_bitmap, 0,
4877                      mrioc->dev_handle_bitmap_bits);
4878         bitmap_clear(mrioc->evtack_cmds_bitmap, 0, MPI3MR_NUM_EVTACKCMD);
4879         mpi3mr_flush_host_io(mrioc);
4880         mpi3mr_cleanup_fwevt_list(mrioc);
4881         mpi3mr_invalidate_devhandles(mrioc);
4882         mpi3mr_free_enclosure_list(mrioc);
4883
4884         if (mrioc->prepare_for_reset) {
4885                 mrioc->prepare_for_reset = 0;
4886                 mrioc->prepare_for_reset_timeout_counter = 0;
4887         }
4888         mpi3mr_memset_buffers(mrioc);
4889         retval = mpi3mr_reinit_ioc(mrioc, 0);
4890         if (retval) {
4891                 pr_err(IOCNAME "reinit after soft reset failed: reason %d\n",
4892                     mrioc->name, reset_reason);
4893                 goto out;
4894         }
4895         ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME);
4896
4897 out:
4898         if (!retval) {
4899                 mrioc->diagsave_timeout = 0;
4900                 mrioc->reset_in_progress = 0;
4901                 mrioc->pel_abort_requested = 0;
4902                 if (mrioc->pel_enabled) {
4903                         mrioc->pel_cmds.retry_count = 0;
4904                         mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds);
4905                 }
4906
4907                 mrioc->device_refresh_on = 0;
4908
4909                 mrioc->ts_update_counter = 0;
4910                 spin_lock_irqsave(&mrioc->watchdog_lock, flags);
4911                 if (mrioc->watchdog_work_q)
4912                         queue_delayed_work(mrioc->watchdog_work_q,
4913                             &mrioc->watchdog_work,
4914                             msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
4915                 spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
4916                 mrioc->stop_bsgs = 0;
4917                 if (mrioc->pel_enabled)
4918                         atomic64_inc(&event_counter);
4919         } else {
4920                 mpi3mr_issue_reset(mrioc,
4921                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
4922                 mrioc->device_refresh_on = 0;
4923                 mrioc->unrecoverable = 1;
4924                 mrioc->reset_in_progress = 0;
4925                 retval = -1;
4926                 mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
4927         }
4928         mrioc->prev_reset_result = retval;
4929         mutex_unlock(&mrioc->reset_mutex);
4930         ioc_info(mrioc, "controller reset is %s\n",
4931             ((retval == 0) ? "successful" : "failed"));
4932         return retval;
4933 }
4934
4935
4936 /**
4937  * mpi3mr_free_config_dma_memory - free memory for config page
4938  * @mrioc: Adapter instance reference
4939  * @mem_desc: memory descriptor structure
4940  *
4941  * Check whether the size of the buffer specified by the memory
4942  * descriptor is greater than the default page size if so then
4943  * free the memory pointed by the descriptor.
4944  *
4945  * Return: Nothing.
4946  */
4947 static void mpi3mr_free_config_dma_memory(struct mpi3mr_ioc *mrioc,
4948         struct dma_memory_desc *mem_desc)
4949 {
4950         if ((mem_desc->size > mrioc->cfg_page_sz) && mem_desc->addr) {
4951                 dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,
4952                     mem_desc->addr, mem_desc->dma_addr);
4953                 mem_desc->addr = NULL;
4954         }
4955 }
4956
4957 /**
4958  * mpi3mr_alloc_config_dma_memory - Alloc memory for config page
4959  * @mrioc: Adapter instance reference
4960  * @mem_desc: Memory descriptor to hold dma memory info
4961  *
4962  * This function allocates new dmaable memory or provides the
4963  * default config page dmaable memory based on the memory size
4964  * described by the descriptor.
4965  *
4966  * Return: 0 on success, non-zero on failure.
4967  */
4968 static int mpi3mr_alloc_config_dma_memory(struct mpi3mr_ioc *mrioc,
4969         struct dma_memory_desc *mem_desc)
4970 {
4971         if (mem_desc->size > mrioc->cfg_page_sz) {
4972                 mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,
4973                     mem_desc->size, &mem_desc->dma_addr, GFP_KERNEL);
4974                 if (!mem_desc->addr)
4975                         return -ENOMEM;
4976         } else {
4977                 mem_desc->addr = mrioc->cfg_page;
4978                 mem_desc->dma_addr = mrioc->cfg_page_dma;
4979                 memset(mem_desc->addr, 0, mrioc->cfg_page_sz);
4980         }
4981         return 0;
4982 }
4983
4984 /**
4985  * mpi3mr_post_cfg_req - Issue config requests and wait
4986  * @mrioc: Adapter instance reference
4987  * @cfg_req: Configuration request
4988  * @timeout: Timeout in seconds
4989  * @ioc_status: Pointer to return ioc status
4990  *
4991  * A generic function for posting MPI3 configuration request to
4992  * the firmware. This blocks for the completion of request for
4993  * timeout seconds and if the request times out this function
4994  * faults the controller with proper reason code.
4995  *
4996  * On successful completion of the request this function returns
4997  * appropriate ioc status from the firmware back to the caller.
4998  *
4999  * Return: 0 on success, non-zero on failure.
5000  */
5001 static int mpi3mr_post_cfg_req(struct mpi3mr_ioc *mrioc,
5002         struct mpi3_config_request *cfg_req, int timeout, u16 *ioc_status)
5003 {
5004         int retval = 0;
5005
5006         mutex_lock(&mrioc->cfg_cmds.mutex);
5007         if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) {
5008                 retval = -1;
5009                 ioc_err(mrioc, "sending config request failed due to command in use\n");
5010                 mutex_unlock(&mrioc->cfg_cmds.mutex);
5011                 goto out;
5012         }
5013         mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING;
5014         mrioc->cfg_cmds.is_waiting = 1;
5015         mrioc->cfg_cmds.callback = NULL;
5016         mrioc->cfg_cmds.ioc_status = 0;
5017         mrioc->cfg_cmds.ioc_loginfo = 0;
5018
5019         cfg_req->host_tag = cpu_to_le16(MPI3MR_HOSTTAG_CFG_CMDS);
5020         cfg_req->function = MPI3_FUNCTION_CONFIG;
5021
5022         init_completion(&mrioc->cfg_cmds.done);
5023         dprint_cfg_info(mrioc, "posting config request\n");
5024         if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5025                 dprint_dump(cfg_req, sizeof(struct mpi3_config_request),
5026                     "mpi3_cfg_req");
5027         retval = mpi3mr_admin_request_post(mrioc, cfg_req, sizeof(*cfg_req), 1);
5028         if (retval) {
5029                 ioc_err(mrioc, "posting config request failed\n");
5030                 goto out_unlock;
5031         }
5032         wait_for_completion_timeout(&mrioc->cfg_cmds.done, (timeout * HZ));
5033         if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) {
5034                 mpi3mr_check_rh_fault_ioc(mrioc,
5035                     MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT);
5036                 ioc_err(mrioc, "config request timed out\n");
5037                 retval = -1;
5038                 goto out_unlock;
5039         }
5040         *ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
5041         if ((*ioc_status) != MPI3_IOCSTATUS_SUCCESS)
5042                 dprint_cfg_err(mrioc,
5043                     "cfg_page request returned with ioc_status(0x%04x), log_info(0x%08x)\n",
5044                     *ioc_status, mrioc->cfg_cmds.ioc_loginfo);
5045
5046 out_unlock:
5047         mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED;
5048         mutex_unlock(&mrioc->cfg_cmds.mutex);
5049
5050 out:
5051         return retval;
5052 }
5053
5054 /**
5055  * mpi3mr_process_cfg_req - config page request processor
5056  * @mrioc: Adapter instance reference
5057  * @cfg_req: Configuration request
5058  * @cfg_hdr: Configuration page header
5059  * @timeout: Timeout in seconds
5060  * @ioc_status: Pointer to return ioc status
5061  * @cfg_buf: Memory pointer to copy config page or header
5062  * @cfg_buf_sz: Size of the memory to get config page or header
5063  *
5064  * This is handler for config page read, write and config page
5065  * header read operations.
5066  *
5067  * This function expects the cfg_req to be populated with page
5068  * type, page number, action for the header read and with page
5069  * address for all other operations.
5070  *
5071  * The cfg_hdr can be passed as null for reading required header
5072  * details for read/write pages the cfg_hdr should point valid
5073  * configuration page header.
5074  *
5075  * This allocates dmaable memory based on the size of the config
5076  * buffer and set the SGE of the cfg_req.
5077  *
5078  * For write actions, the config page data has to be passed in
5079  * the cfg_buf and size of the data has to be mentioned in the
5080  * cfg_buf_sz.
5081  *
5082  * For read/header actions, on successful completion of the
5083  * request with successful ioc_status the data will be copied
5084  * into the cfg_buf limited to a minimum of actual page size and
5085  * cfg_buf_sz
5086  *
5087  *
5088  * Return: 0 on success, non-zero on failure.
5089  */
5090 static int mpi3mr_process_cfg_req(struct mpi3mr_ioc *mrioc,
5091         struct mpi3_config_request *cfg_req,
5092         struct mpi3_config_page_header *cfg_hdr, int timeout, u16 *ioc_status,
5093         void *cfg_buf, u32 cfg_buf_sz)
5094 {
5095         struct dma_memory_desc mem_desc;
5096         int retval = -1;
5097         u8 invalid_action = 0;
5098         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
5099
5100         memset(&mem_desc, 0, sizeof(struct dma_memory_desc));
5101
5102         if (cfg_req->action == MPI3_CONFIG_ACTION_PAGE_HEADER)
5103                 mem_desc.size = sizeof(struct mpi3_config_page_header);
5104         else {
5105                 if (!cfg_hdr) {
5106                         ioc_err(mrioc, "null config header passed for config action(%d), page_type(0x%02x), page_num(%d)\n",
5107                             cfg_req->action, cfg_req->page_type,
5108                             cfg_req->page_number);
5109                         goto out;
5110                 }
5111                 switch (cfg_hdr->page_attribute & MPI3_CONFIG_PAGEATTR_MASK) {
5112                 case MPI3_CONFIG_PAGEATTR_READ_ONLY:
5113                         if (cfg_req->action
5114                             != MPI3_CONFIG_ACTION_READ_CURRENT)
5115                                 invalid_action = 1;
5116                         break;
5117                 case MPI3_CONFIG_PAGEATTR_CHANGEABLE:
5118                         if ((cfg_req->action ==
5119                              MPI3_CONFIG_ACTION_READ_PERSISTENT) ||
5120                             (cfg_req->action ==
5121                              MPI3_CONFIG_ACTION_WRITE_PERSISTENT))
5122                                 invalid_action = 1;
5123                         break;
5124                 case MPI3_CONFIG_PAGEATTR_PERSISTENT:
5125                 default:
5126                         break;
5127                 }
5128                 if (invalid_action) {
5129                         ioc_err(mrioc,
5130                             "config action(%d) is not allowed for page_type(0x%02x), page_num(%d) with page_attribute(0x%02x)\n",
5131                             cfg_req->action, cfg_req->page_type,
5132                             cfg_req->page_number, cfg_hdr->page_attribute);
5133                         goto out;
5134                 }
5135                 mem_desc.size = le16_to_cpu(cfg_hdr->page_length) * 4;
5136                 cfg_req->page_length = cfg_hdr->page_length;
5137                 cfg_req->page_version = cfg_hdr->page_version;
5138         }
5139         if (mpi3mr_alloc_config_dma_memory(mrioc, &mem_desc))
5140                 goto out;
5141
5142         mpi3mr_add_sg_single(&cfg_req->sgl, sgl_flags, mem_desc.size,
5143             mem_desc.dma_addr);
5144
5145         if ((cfg_req->action == MPI3_CONFIG_ACTION_WRITE_PERSISTENT) ||
5146             (cfg_req->action == MPI3_CONFIG_ACTION_WRITE_CURRENT)) {
5147                 memcpy(mem_desc.addr, cfg_buf, min_t(u16, mem_desc.size,
5148                     cfg_buf_sz));
5149                 dprint_cfg_info(mrioc, "config buffer to be written\n");
5150                 if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5151                         dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf");
5152         }
5153
5154         if (mpi3mr_post_cfg_req(mrioc, cfg_req, timeout, ioc_status))
5155                 goto out;
5156
5157         retval = 0;
5158         if ((*ioc_status == MPI3_IOCSTATUS_SUCCESS) &&
5159             (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_PERSISTENT) &&
5160             (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_CURRENT)) {
5161                 memcpy(cfg_buf, mem_desc.addr, min_t(u16, mem_desc.size,
5162                     cfg_buf_sz));
5163                 dprint_cfg_info(mrioc, "config buffer read\n");
5164                 if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5165                         dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf");
5166         }
5167
5168 out:
5169         mpi3mr_free_config_dma_memory(mrioc, &mem_desc);
5170         return retval;
5171 }
5172
5173 /**
5174  * mpi3mr_cfg_get_dev_pg0 - Read current device page0
5175  * @mrioc: Adapter instance reference
5176  * @ioc_status: Pointer to return ioc status
5177  * @dev_pg0: Pointer to return device page 0
5178  * @pg_sz: Size of the memory allocated to the page pointer
5179  * @form: The form to be used for addressing the page
5180  * @form_spec: Form specific information like device handle
5181  *
5182  * This is handler for config page read for a specific device
5183  * page0. The ioc_status has the controller returned ioc_status.
5184  * This routine doesn't check ioc_status to decide whether the
5185  * page read is success or not and it is the callers
5186  * responsibility.
5187  *
5188  * Return: 0 on success, non-zero on failure.
5189  */
5190 int mpi3mr_cfg_get_dev_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5191         struct mpi3_device_page0 *dev_pg0, u16 pg_sz, u32 form, u32 form_spec)
5192 {
5193         struct mpi3_config_page_header cfg_hdr;
5194         struct mpi3_config_request cfg_req;
5195         u32 page_address;
5196
5197         memset(dev_pg0, 0, pg_sz);
5198         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5199         memset(&cfg_req, 0, sizeof(cfg_req));
5200
5201         cfg_req.function = MPI3_FUNCTION_CONFIG;
5202         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5203         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DEVICE;
5204         cfg_req.page_number = 0;
5205         cfg_req.page_address = 0;
5206
5207         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5208             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5209                 ioc_err(mrioc, "device page0 header read failed\n");
5210                 goto out_failed;
5211         }
5212         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5213                 ioc_err(mrioc, "device page0 header read failed with ioc_status(0x%04x)\n",
5214                     *ioc_status);
5215                 goto out_failed;
5216         }
5217         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5218         page_address = ((form & MPI3_DEVICE_PGAD_FORM_MASK) |
5219             (form_spec & MPI3_DEVICE_PGAD_HANDLE_MASK));
5220         cfg_req.page_address = cpu_to_le32(page_address);
5221         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5222             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, dev_pg0, pg_sz)) {
5223                 ioc_err(mrioc, "device page0 read failed\n");
5224                 goto out_failed;
5225         }
5226         return 0;
5227 out_failed:
5228         return -1;
5229 }
5230
5231
5232 /**
5233  * mpi3mr_cfg_get_sas_phy_pg0 - Read current SAS Phy page0
5234  * @mrioc: Adapter instance reference
5235  * @ioc_status: Pointer to return ioc status
5236  * @phy_pg0: Pointer to return SAS Phy page 0
5237  * @pg_sz: Size of the memory allocated to the page pointer
5238  * @form: The form to be used for addressing the page
5239  * @form_spec: Form specific information like phy number
5240  *
5241  * This is handler for config page read for a specific SAS Phy
5242  * page0. The ioc_status has the controller returned ioc_status.
5243  * This routine doesn't check ioc_status to decide whether the
5244  * page read is success or not and it is the callers
5245  * responsibility.
5246  *
5247  * Return: 0 on success, non-zero on failure.
5248  */
5249 int mpi3mr_cfg_get_sas_phy_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5250         struct mpi3_sas_phy_page0 *phy_pg0, u16 pg_sz, u32 form,
5251         u32 form_spec)
5252 {
5253         struct mpi3_config_page_header cfg_hdr;
5254         struct mpi3_config_request cfg_req;
5255         u32 page_address;
5256
5257         memset(phy_pg0, 0, pg_sz);
5258         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5259         memset(&cfg_req, 0, sizeof(cfg_req));
5260
5261         cfg_req.function = MPI3_FUNCTION_CONFIG;
5262         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5263         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY;
5264         cfg_req.page_number = 0;
5265         cfg_req.page_address = 0;
5266
5267         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5268             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5269                 ioc_err(mrioc, "sas phy page0 header read failed\n");
5270                 goto out_failed;
5271         }
5272         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5273                 ioc_err(mrioc, "sas phy page0 header read failed with ioc_status(0x%04x)\n",
5274                     *ioc_status);
5275                 goto out_failed;
5276         }
5277         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5278         page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) |
5279             (form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK));
5280         cfg_req.page_address = cpu_to_le32(page_address);
5281         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5282             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, phy_pg0, pg_sz)) {
5283                 ioc_err(mrioc, "sas phy page0 read failed\n");
5284                 goto out_failed;
5285         }
5286         return 0;
5287 out_failed:
5288         return -1;
5289 }
5290
5291 /**
5292  * mpi3mr_cfg_get_sas_phy_pg1 - Read current SAS Phy page1
5293  * @mrioc: Adapter instance reference
5294  * @ioc_status: Pointer to return ioc status
5295  * @phy_pg1: Pointer to return SAS Phy page 1
5296  * @pg_sz: Size of the memory allocated to the page pointer
5297  * @form: The form to be used for addressing the page
5298  * @form_spec: Form specific information like phy number
5299  *
5300  * This is handler for config page read for a specific SAS Phy
5301  * page1. The ioc_status has the controller returned ioc_status.
5302  * This routine doesn't check ioc_status to decide whether the
5303  * page read is success or not and it is the callers
5304  * responsibility.
5305  *
5306  * Return: 0 on success, non-zero on failure.
5307  */
5308 int mpi3mr_cfg_get_sas_phy_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5309         struct mpi3_sas_phy_page1 *phy_pg1, u16 pg_sz, u32 form,
5310         u32 form_spec)
5311 {
5312         struct mpi3_config_page_header cfg_hdr;
5313         struct mpi3_config_request cfg_req;
5314         u32 page_address;
5315
5316         memset(phy_pg1, 0, pg_sz);
5317         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5318         memset(&cfg_req, 0, sizeof(cfg_req));
5319
5320         cfg_req.function = MPI3_FUNCTION_CONFIG;
5321         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5322         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY;
5323         cfg_req.page_number = 1;
5324         cfg_req.page_address = 0;
5325
5326         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5327             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5328                 ioc_err(mrioc, "sas phy page1 header read failed\n");
5329                 goto out_failed;
5330         }
5331         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5332                 ioc_err(mrioc, "sas phy page1 header read failed with ioc_status(0x%04x)\n",
5333                     *ioc_status);
5334                 goto out_failed;
5335         }
5336         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5337         page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) |
5338             (form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK));
5339         cfg_req.page_address = cpu_to_le32(page_address);
5340         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5341             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, phy_pg1, pg_sz)) {
5342                 ioc_err(mrioc, "sas phy page1 read failed\n");
5343                 goto out_failed;
5344         }
5345         return 0;
5346 out_failed:
5347         return -1;
5348 }
5349
5350
5351 /**
5352  * mpi3mr_cfg_get_sas_exp_pg0 - Read current SAS Expander page0
5353  * @mrioc: Adapter instance reference
5354  * @ioc_status: Pointer to return ioc status
5355  * @exp_pg0: Pointer to return SAS Expander page 0
5356  * @pg_sz: Size of the memory allocated to the page pointer
5357  * @form: The form to be used for addressing the page
5358  * @form_spec: Form specific information like device handle
5359  *
5360  * This is handler for config page read for a specific SAS
5361  * Expander page0. The ioc_status has the controller returned
5362  * ioc_status. This routine doesn't check ioc_status to decide
5363  * whether the page read is success or not and it is the callers
5364  * responsibility.
5365  *
5366  * Return: 0 on success, non-zero on failure.
5367  */
5368 int mpi3mr_cfg_get_sas_exp_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5369         struct mpi3_sas_expander_page0 *exp_pg0, u16 pg_sz, u32 form,
5370         u32 form_spec)
5371 {
5372         struct mpi3_config_page_header cfg_hdr;
5373         struct mpi3_config_request cfg_req;
5374         u32 page_address;
5375
5376         memset(exp_pg0, 0, pg_sz);
5377         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5378         memset(&cfg_req, 0, sizeof(cfg_req));
5379
5380         cfg_req.function = MPI3_FUNCTION_CONFIG;
5381         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5382         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER;
5383         cfg_req.page_number = 0;
5384         cfg_req.page_address = 0;
5385
5386         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5387             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5388                 ioc_err(mrioc, "expander page0 header read failed\n");
5389                 goto out_failed;
5390         }
5391         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5392                 ioc_err(mrioc, "expander page0 header read failed with ioc_status(0x%04x)\n",
5393                     *ioc_status);
5394                 goto out_failed;
5395         }
5396         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5397         page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) |
5398             (form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK |
5399             MPI3_SAS_EXPAND_PGAD_HANDLE_MASK)));
5400         cfg_req.page_address = cpu_to_le32(page_address);
5401         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5402             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, exp_pg0, pg_sz)) {
5403                 ioc_err(mrioc, "expander page0 read failed\n");
5404                 goto out_failed;
5405         }
5406         return 0;
5407 out_failed:
5408         return -1;
5409 }
5410
5411 /**
5412  * mpi3mr_cfg_get_sas_exp_pg1 - Read current SAS Expander page1
5413  * @mrioc: Adapter instance reference
5414  * @ioc_status: Pointer to return ioc status
5415  * @exp_pg1: Pointer to return SAS Expander page 1
5416  * @pg_sz: Size of the memory allocated to the page pointer
5417  * @form: The form to be used for addressing the page
5418  * @form_spec: Form specific information like phy number
5419  *
5420  * This is handler for config page read for a specific SAS
5421  * Expander page1. The ioc_status has the controller returned
5422  * ioc_status. This routine doesn't check ioc_status to decide
5423  * whether the page read is success or not and it is the callers
5424  * responsibility.
5425  *
5426  * Return: 0 on success, non-zero on failure.
5427  */
5428 int mpi3mr_cfg_get_sas_exp_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5429         struct mpi3_sas_expander_page1 *exp_pg1, u16 pg_sz, u32 form,
5430         u32 form_spec)
5431 {
5432         struct mpi3_config_page_header cfg_hdr;
5433         struct mpi3_config_request cfg_req;
5434         u32 page_address;
5435
5436         memset(exp_pg1, 0, pg_sz);
5437         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5438         memset(&cfg_req, 0, sizeof(cfg_req));
5439
5440         cfg_req.function = MPI3_FUNCTION_CONFIG;
5441         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5442         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER;
5443         cfg_req.page_number = 1;
5444         cfg_req.page_address = 0;
5445
5446         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5447             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5448                 ioc_err(mrioc, "expander page1 header read failed\n");
5449                 goto out_failed;
5450         }
5451         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5452                 ioc_err(mrioc, "expander page1 header read failed with ioc_status(0x%04x)\n",
5453                     *ioc_status);
5454                 goto out_failed;
5455         }
5456         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5457         page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) |
5458             (form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK |
5459             MPI3_SAS_EXPAND_PGAD_HANDLE_MASK)));
5460         cfg_req.page_address = cpu_to_le32(page_address);
5461         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5462             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, exp_pg1, pg_sz)) {
5463                 ioc_err(mrioc, "expander page1 read failed\n");
5464                 goto out_failed;
5465         }
5466         return 0;
5467 out_failed:
5468         return -1;
5469 }
5470
5471 /**
5472  * mpi3mr_cfg_get_enclosure_pg0 - Read current Enclosure page0
5473  * @mrioc: Adapter instance reference
5474  * @ioc_status: Pointer to return ioc status
5475  * @encl_pg0: Pointer to return Enclosure page 0
5476  * @pg_sz: Size of the memory allocated to the page pointer
5477  * @form: The form to be used for addressing the page
5478  * @form_spec: Form specific information like device handle
5479  *
5480  * This is handler for config page read for a specific Enclosure
5481  * page0. The ioc_status has the controller returned ioc_status.
5482  * This routine doesn't check ioc_status to decide whether the
5483  * page read is success or not and it is the callers
5484  * responsibility.
5485  *
5486  * Return: 0 on success, non-zero on failure.
5487  */
5488 int mpi3mr_cfg_get_enclosure_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5489         struct mpi3_enclosure_page0 *encl_pg0, u16 pg_sz, u32 form,
5490         u32 form_spec)
5491 {
5492         struct mpi3_config_page_header cfg_hdr;
5493         struct mpi3_config_request cfg_req;
5494         u32 page_address;
5495
5496         memset(encl_pg0, 0, pg_sz);
5497         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5498         memset(&cfg_req, 0, sizeof(cfg_req));
5499
5500         cfg_req.function = MPI3_FUNCTION_CONFIG;
5501         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5502         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_ENCLOSURE;
5503         cfg_req.page_number = 0;
5504         cfg_req.page_address = 0;
5505
5506         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5507             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5508                 ioc_err(mrioc, "enclosure page0 header read failed\n");
5509                 goto out_failed;
5510         }
5511         if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5512                 ioc_err(mrioc, "enclosure page0 header read failed with ioc_status(0x%04x)\n",
5513                     *ioc_status);
5514                 goto out_failed;
5515         }
5516         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5517         page_address = ((form & MPI3_ENCLOS_PGAD_FORM_MASK) |
5518             (form_spec & MPI3_ENCLOS_PGAD_HANDLE_MASK));
5519         cfg_req.page_address = cpu_to_le32(page_address);
5520         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5521             MPI3MR_INTADMCMD_TIMEOUT, ioc_status, encl_pg0, pg_sz)) {
5522                 ioc_err(mrioc, "enclosure page0 read failed\n");
5523                 goto out_failed;
5524         }
5525         return 0;
5526 out_failed:
5527         return -1;
5528 }
5529
5530
5531 /**
5532  * mpi3mr_cfg_get_sas_io_unit_pg0 - Read current SASIOUnit page0
5533  * @mrioc: Adapter instance reference
5534  * @sas_io_unit_pg0: Pointer to return SAS IO Unit page 0
5535  * @pg_sz: Size of the memory allocated to the page pointer
5536  *
5537  * This is handler for config page read for the SAS IO Unit
5538  * page0. This routine checks ioc_status to decide whether the
5539  * page read is success or not.
5540  *
5541  * Return: 0 on success, non-zero on failure.
5542  */
5543 int mpi3mr_cfg_get_sas_io_unit_pg0(struct mpi3mr_ioc *mrioc,
5544         struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0, u16 pg_sz)
5545 {
5546         struct mpi3_config_page_header cfg_hdr;
5547         struct mpi3_config_request cfg_req;
5548         u16 ioc_status = 0;
5549
5550         memset(sas_io_unit_pg0, 0, pg_sz);
5551         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5552         memset(&cfg_req, 0, sizeof(cfg_req));
5553
5554         cfg_req.function = MPI3_FUNCTION_CONFIG;
5555         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5556         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5557         cfg_req.page_number = 0;
5558         cfg_req.page_address = 0;
5559
5560         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5561             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5562                 ioc_err(mrioc, "sas io unit page0 header read failed\n");
5563                 goto out_failed;
5564         }
5565         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5566                 ioc_err(mrioc, "sas io unit page0 header read failed with ioc_status(0x%04x)\n",
5567                     ioc_status);
5568                 goto out_failed;
5569         }
5570         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5571
5572         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5573             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg0, pg_sz)) {
5574                 ioc_err(mrioc, "sas io unit page0 read failed\n");
5575                 goto out_failed;
5576         }
5577         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5578                 ioc_err(mrioc, "sas io unit page0 read failed with ioc_status(0x%04x)\n",
5579                     ioc_status);
5580                 goto out_failed;
5581         }
5582         return 0;
5583 out_failed:
5584         return -1;
5585 }
5586
5587 /**
5588  * mpi3mr_cfg_get_sas_io_unit_pg1 - Read current SASIOUnit page1
5589  * @mrioc: Adapter instance reference
5590  * @sas_io_unit_pg1: Pointer to return SAS IO Unit page 1
5591  * @pg_sz: Size of the memory allocated to the page pointer
5592  *
5593  * This is handler for config page read for the SAS IO Unit
5594  * page1. This routine checks ioc_status to decide whether the
5595  * page read is success or not.
5596  *
5597  * Return: 0 on success, non-zero on failure.
5598  */
5599 int mpi3mr_cfg_get_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5600         struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz)
5601 {
5602         struct mpi3_config_page_header cfg_hdr;
5603         struct mpi3_config_request cfg_req;
5604         u16 ioc_status = 0;
5605
5606         memset(sas_io_unit_pg1, 0, pg_sz);
5607         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5608         memset(&cfg_req, 0, sizeof(cfg_req));
5609
5610         cfg_req.function = MPI3_FUNCTION_CONFIG;
5611         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5612         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5613         cfg_req.page_number = 1;
5614         cfg_req.page_address = 0;
5615
5616         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5617             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5618                 ioc_err(mrioc, "sas io unit page1 header read failed\n");
5619                 goto out_failed;
5620         }
5621         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5622                 ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5623                     ioc_status);
5624                 goto out_failed;
5625         }
5626         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5627
5628         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5629             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) {
5630                 ioc_err(mrioc, "sas io unit page1 read failed\n");
5631                 goto out_failed;
5632         }
5633         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5634                 ioc_err(mrioc, "sas io unit page1 read failed with ioc_status(0x%04x)\n",
5635                     ioc_status);
5636                 goto out_failed;
5637         }
5638         return 0;
5639 out_failed:
5640         return -1;
5641 }
5642
5643 /**
5644  * mpi3mr_cfg_set_sas_io_unit_pg1 - Write SASIOUnit page1
5645  * @mrioc: Adapter instance reference
5646  * @sas_io_unit_pg1: Pointer to the SAS IO Unit page 1 to write
5647  * @pg_sz: Size of the memory allocated to the page pointer
5648  *
5649  * This is handler for config page write for the SAS IO Unit
5650  * page1. This routine checks ioc_status to decide whether the
5651  * page read is success or not. This will modify both current
5652  * and persistent page.
5653  *
5654  * Return: 0 on success, non-zero on failure.
5655  */
5656 int mpi3mr_cfg_set_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5657         struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz)
5658 {
5659         struct mpi3_config_page_header cfg_hdr;
5660         struct mpi3_config_request cfg_req;
5661         u16 ioc_status = 0;
5662
5663         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5664         memset(&cfg_req, 0, sizeof(cfg_req));
5665
5666         cfg_req.function = MPI3_FUNCTION_CONFIG;
5667         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5668         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5669         cfg_req.page_number = 1;
5670         cfg_req.page_address = 0;
5671
5672         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5673             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5674                 ioc_err(mrioc, "sas io unit page1 header read failed\n");
5675                 goto out_failed;
5676         }
5677         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5678                 ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5679                     ioc_status);
5680                 goto out_failed;
5681         }
5682         cfg_req.action = MPI3_CONFIG_ACTION_WRITE_CURRENT;
5683
5684         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5685             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) {
5686                 ioc_err(mrioc, "sas io unit page1 write current failed\n");
5687                 goto out_failed;
5688         }
5689         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5690                 ioc_err(mrioc, "sas io unit page1 write current failed with ioc_status(0x%04x)\n",
5691                     ioc_status);
5692                 goto out_failed;
5693         }
5694
5695         cfg_req.action = MPI3_CONFIG_ACTION_WRITE_PERSISTENT;
5696
5697         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5698             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) {
5699                 ioc_err(mrioc, "sas io unit page1 write persistent failed\n");
5700                 goto out_failed;
5701         }
5702         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5703                 ioc_err(mrioc, "sas io unit page1 write persistent failed with ioc_status(0x%04x)\n",
5704                     ioc_status);
5705                 goto out_failed;
5706         }
5707         return 0;
5708 out_failed:
5709         return -1;
5710 }
5711
5712 /**
5713  * mpi3mr_cfg_get_driver_pg1 - Read current Driver page1
5714  * @mrioc: Adapter instance reference
5715  * @driver_pg1: Pointer to return Driver page 1
5716  * @pg_sz: Size of the memory allocated to the page pointer
5717  *
5718  * This is handler for config page read for the Driver page1.
5719  * This routine checks ioc_status to decide whether the page
5720  * read is success or not.
5721  *
5722  * Return: 0 on success, non-zero on failure.
5723  */
5724 int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_ioc *mrioc,
5725         struct mpi3_driver_page1 *driver_pg1, u16 pg_sz)
5726 {
5727         struct mpi3_config_page_header cfg_hdr;
5728         struct mpi3_config_request cfg_req;
5729         u16 ioc_status = 0;
5730
5731         memset(driver_pg1, 0, pg_sz);
5732         memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5733         memset(&cfg_req, 0, sizeof(cfg_req));
5734
5735         cfg_req.function = MPI3_FUNCTION_CONFIG;
5736         cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5737         cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DRIVER;
5738         cfg_req.page_number = 1;
5739         cfg_req.page_address = 0;
5740
5741         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5742             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5743                 ioc_err(mrioc, "driver page1 header read failed\n");
5744                 goto out_failed;
5745         }
5746         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5747                 ioc_err(mrioc, "driver page1 header read failed with ioc_status(0x%04x)\n",
5748                     ioc_status);
5749                 goto out_failed;
5750         }
5751         cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5752
5753         if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5754             MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, driver_pg1, pg_sz)) {
5755                 ioc_err(mrioc, "driver page1 read failed\n");
5756                 goto out_failed;
5757         }
5758         if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5759                 ioc_err(mrioc, "driver page1 read failed with ioc_status(0x%04x)\n",
5760                     ioc_status);
5761                 goto out_failed;
5762         }
5763         return 0;
5764 out_failed:
5765         return -1;
5766 }