drivers/block/skd_main.c

   1 /* Copyright 2012 STEC, Inc.
   2  *
   3  * This file is licensed under the terms of the 3-clause
   4  * BSD License (http://opensource.org/licenses/BSD-3-Clause)
   5  * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
   6  * at your option. Both licenses are also available in the LICENSE file
   7  * distributed with this project. This file may not be copied, modified,
   8  * or distributed except in accordance with those terms.
   9  * Gordoni Waidhofer <gwaidhofer@stec-inc.com>
  10  * Initial Driver Design!
  11  * Thomas Swann <tswann@stec-inc.com>
  12  * Interrupt handling.
  13  * Ramprasad Chinthekindi <rchinthekindi@stec-inc.com>
  14  * biomode implementation.
  15  * Akhil Bhansali <abhansali@stec-inc.com>
  16  * Added support for DISCARD / FLUSH and FUA.
  17  */
  18
  19 #include <linux/kernel.h>
  20 #include <linux/module.h>
  21 #include <linux/init.h>
  22 #include <linux/pci.h>
  23 #include <linux/slab.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/blkdev.h>
  26 #include <linux/sched.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/compiler.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/bitops.h>
  31 #include <linux/delay.h>
  32 #include <linux/time.h>
  33 #include <linux/hdreg.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/completion.h>
  36 #include <linux/scatterlist.h>
  37 #include <linux/version.h>
  38 #include <linux/err.h>
  39 #include <linux/aer.h>
  40 #include <linux/ctype.h>
  41 #include <linux/wait.h>
  42 #include <linux/uio.h>
  43 #include <scsi/scsi.h>
  44 #include <scsi/sg.h>
  45 #include <linux/io.h>
  46 #include <linux/uaccess.h>
  47 #include <asm/unaligned.h>
  48
  49 #include "skd_s1120.h"
  50
  51 static int skd_dbg_level;
  52 static int skd_isr_comp_limit = 4;
  53
  54 enum {
  55         STEC_LINK_2_5GTS = 0,
  56         STEC_LINK_5GTS = 1,
  57         STEC_LINK_8GTS = 2,
  58         STEC_LINK_UNKNOWN = 0xFF
  59 };
  60
  61 enum {
  62         SKD_FLUSH_INITIALIZER,
  63         SKD_FLUSH_ZERO_SIZE_FIRST,
  64         SKD_FLUSH_DATA_SECOND,
  65 };
  66
  67 #define SKD_ASSERT(expr) \
  68         do { \
  69                 if (unlikely(!(expr))) { \
  70                         pr_err("Assertion failed! %s,%s,%s,line=%d\n",  \
  71                                # expr, __FILE__, __func__, __LINE__); \
  72                 } \
  73         } while (0)
  74
  75 #define DRV_NAME "skd"
  76 #define DRV_VERSION "2.2.1"
  77 #define DRV_BUILD_ID "0260"
  78 #define PFX DRV_NAME ": "
  79 #define DRV_BIN_VERSION 0x100
  80 #define DRV_VER_COMPL   "2.2.1." DRV_BUILD_ID
  81
  82 MODULE_AUTHOR("bug-reports: support@stec-inc.com");
  83 MODULE_LICENSE("Dual BSD/GPL");
  84
  85 MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver (b" DRV_BUILD_ID ")");
  86 MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
  87
  88 #define PCI_VENDOR_ID_STEC      0x1B39
  89 #define PCI_DEVICE_ID_S1120     0x0001
  90
  91 #define SKD_FUA_NV              (1 << 1)
  92 #define SKD_MINORS_PER_DEVICE   16
  93
  94 #define SKD_MAX_QUEUE_DEPTH     200u
  95
  96 #define SKD_PAUSE_TIMEOUT       (5 * 1000)
  97
  98 #define SKD_N_FITMSG_BYTES      (512u)
  99
 100 #define SKD_N_SPECIAL_CONTEXT   32u
 101 #define SKD_N_SPECIAL_FITMSG_BYTES      (128u)
 102
 103 /* SG elements are 32 bytes, so we can make this 4096 and still be under the
 104  * 128KB limit.  That allows 4096*4K = 16M xfer size
 105  */
 106 #define SKD_N_SG_PER_REQ_DEFAULT 256u
 107 #define SKD_N_SG_PER_SPECIAL    256u
 108
 109 #define SKD_N_COMPLETION_ENTRY  256u
 110 #define SKD_N_READ_CAP_BYTES    (8u)
 111
 112 #define SKD_N_INTERNAL_BYTES    (512u)
 113
 114 /* 5 bits of uniqifier, 0xF800 */
 115 #define SKD_ID_INCR             (0x400)
 116 #define SKD_ID_TABLE_MASK       (3u << 8u)
 117 #define  SKD_ID_RW_REQUEST      (0u << 8u)
 118 #define  SKD_ID_INTERNAL        (1u << 8u)
 119 #define  SKD_ID_SPECIAL_REQUEST (2u << 8u)
 120 #define  SKD_ID_FIT_MSG         (3u << 8u)
 121 #define SKD_ID_SLOT_MASK        0x00FFu
 122 #define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
 123
 124 #define SKD_N_TIMEOUT_SLOT      4u
 125 #define SKD_TIMEOUT_SLOT_MASK   3u
 126
 127 #define SKD_N_MAX_SECTORS 2048u
 128
 129 #define SKD_MAX_RETRIES 2u
 130
 131 #define SKD_TIMER_SECONDS(seconds) (seconds)
 132 #define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
 133
 134 #define INQ_STD_NBYTES 36
 135
 136 enum skd_drvr_state {
 137         SKD_DRVR_STATE_LOAD,
 138         SKD_DRVR_STATE_IDLE,
 139         SKD_DRVR_STATE_BUSY,
 140         SKD_DRVR_STATE_STARTING,
 141         SKD_DRVR_STATE_ONLINE,
 142         SKD_DRVR_STATE_PAUSING,
 143         SKD_DRVR_STATE_PAUSED,
 144         SKD_DRVR_STATE_DRAINING_TIMEOUT,
 145         SKD_DRVR_STATE_RESTARTING,
 146         SKD_DRVR_STATE_RESUMING,
 147         SKD_DRVR_STATE_STOPPING,
 148         SKD_DRVR_STATE_FAULT,
 149         SKD_DRVR_STATE_DISAPPEARED,
 150         SKD_DRVR_STATE_PROTOCOL_MISMATCH,
 151         SKD_DRVR_STATE_BUSY_ERASE,
 152         SKD_DRVR_STATE_BUSY_SANITIZE,
 153         SKD_DRVR_STATE_BUSY_IMMINENT,
 154         SKD_DRVR_STATE_WAIT_BOOT,
 155         SKD_DRVR_STATE_SYNCING,
 156 };
 157
 158 #define SKD_WAIT_BOOT_TIMO      SKD_TIMER_SECONDS(90u)
 159 #define SKD_STARTING_TIMO       SKD_TIMER_SECONDS(8u)
 160 #define SKD_RESTARTING_TIMO     SKD_TIMER_MINUTES(4u)
 161 #define SKD_DRAINING_TIMO       SKD_TIMER_SECONDS(6u)
 162 #define SKD_BUSY_TIMO           SKD_TIMER_MINUTES(20u)
 163 #define SKD_STARTED_BUSY_TIMO   SKD_TIMER_SECONDS(60u)
 164 #define SKD_START_WAIT_SECONDS  90u
 165
 166 enum skd_req_state {
 167         SKD_REQ_STATE_IDLE,
 168         SKD_REQ_STATE_SETUP,
 169         SKD_REQ_STATE_BUSY,
 170         SKD_REQ_STATE_COMPLETED,
 171         SKD_REQ_STATE_TIMEOUT,
 172         SKD_REQ_STATE_ABORTED,
 173 };
 174
 175 enum skd_fit_msg_state {
 176         SKD_MSG_STATE_IDLE,
 177         SKD_MSG_STATE_BUSY,
 178 };
 179
 180 enum skd_check_status_action {
 181         SKD_CHECK_STATUS_REPORT_GOOD,
 182         SKD_CHECK_STATUS_REPORT_SMART_ALERT,
 183         SKD_CHECK_STATUS_REQUEUE_REQUEST,
 184         SKD_CHECK_STATUS_REPORT_ERROR,
 185         SKD_CHECK_STATUS_BUSY_IMMINENT,
 186 };
 187
 188 struct skd_fitmsg_context {
 189         enum skd_fit_msg_state state;
 190
 191         struct skd_fitmsg_context *next;
 192
 193         u32 id;
 194         u16 outstanding;
 195
 196         u32 length;
 197         u32 offset;
 198
 199         u8 *msg_buf;
 200         dma_addr_t mb_dma_address;
 201 };
 202
 203 struct skd_request_context {
 204         enum skd_req_state state;
 205
 206         struct skd_request_context *next;
 207
 208         u16 id;
 209         u32 fitmsg_id;
 210
 211         struct request *req;
 212         u8 flush_cmd;
 213
 214         u32 timeout_stamp;
 215         u8 sg_data_dir;
 216         struct scatterlist *sg;
 217         u32 n_sg;
 218         u32 sg_byte_count;
 219
 220         struct fit_sg_descriptor *sksg_list;
 221         dma_addr_t sksg_dma_address;
 222
 223         struct fit_completion_entry_v1 completion;
 224
 225         struct fit_comp_error_info err_info;
 226
 227 };
 228 #define SKD_DATA_DIR_HOST_TO_CARD       1
 229 #define SKD_DATA_DIR_CARD_TO_HOST       2
 230
 231 struct skd_special_context {
 232         struct skd_request_context req;
 233
 234         u8 orphaned;
 235
 236         void *data_buf;
 237         dma_addr_t db_dma_address;
 238
 239         u8 *msg_buf;
 240         dma_addr_t mb_dma_address;
 241 };
 242
 243 struct skd_sg_io {
 244         fmode_t mode;
 245         void __user *argp;
 246
 247         struct sg_io_hdr sg;
 248
 249         u8 cdb[16];
 250
 251         u32 dxfer_len;
 252         u32 iovcnt;
 253         struct sg_iovec *iov;
 254         struct sg_iovec no_iov_iov;
 255
 256         struct skd_special_context *skspcl;
 257 };
 258
 259 typedef enum skd_irq_type {
 260         SKD_IRQ_LEGACY,
 261         SKD_IRQ_MSI,
 262         SKD_IRQ_MSIX
 263 } skd_irq_type_t;
 264
 265 #define SKD_MAX_BARS                    2
 266
 267 struct skd_device {
 268         volatile void __iomem *mem_map[SKD_MAX_BARS];
 269         resource_size_t mem_phys[SKD_MAX_BARS];
 270         u32 mem_size[SKD_MAX_BARS];
 271
 272         struct skd_msix_entry *msix_entries;
 273
 274         struct pci_dev *pdev;
 275         int pcie_error_reporting_is_enabled;
 276
 277         spinlock_t lock;
 278         struct gendisk *disk;
 279         struct request_queue *queue;
 280         struct device *class_dev;
 281         int gendisk_on;
 282         int sync_done;
 283
 284         atomic_t device_count;
 285         u32 devno;
 286         u32 major;
 287         char name[32];
 288         char isr_name[30];
 289
 290         enum skd_drvr_state state;
 291         u32 drive_state;
 292
 293         u32 in_flight;
 294         u32 cur_max_queue_depth;
 295         u32 queue_low_water_mark;
 296         u32 dev_max_queue_depth;
 297
 298         u32 num_fitmsg_context;
 299         u32 num_req_context;
 300
 301         u32 timeout_slot[SKD_N_TIMEOUT_SLOT];
 302         u32 timeout_stamp;
 303         struct skd_fitmsg_context *skmsg_free_list;
 304         struct skd_fitmsg_context *skmsg_table;
 305
 306         struct skd_request_context *skreq_free_list;
 307         struct skd_request_context *skreq_table;
 308
 309         struct skd_special_context *skspcl_free_list;
 310         struct skd_special_context *skspcl_table;
 311
 312         struct skd_special_context internal_skspcl;
 313         u32 read_cap_blocksize;
 314         u32 read_cap_last_lba;
 315         int read_cap_is_valid;
 316         int inquiry_is_valid;
 317         u8 inq_serial_num[13];  /*12 chars plus null term */
 318         u8 id_str[80];          /* holds a composite name (pci + sernum) */
 319
 320         u8 skcomp_cycle;
 321         u32 skcomp_ix;
 322         struct fit_completion_entry_v1 *skcomp_table;
 323         struct fit_comp_error_info *skerr_table;
 324         dma_addr_t cq_dma_address;
 325
 326         wait_queue_head_t waitq;
 327
 328         struct timer_list timer;
 329         u32 timer_countdown;
 330         u32 timer_substate;
 331
 332         int n_special;
 333         int sgs_per_request;
 334         u32 last_mtd;
 335
 336         u32 proto_ver;
 337
 338         int dbg_level;
 339         u32 connect_time_stamp;
 340         int connect_retries;
 341 #define SKD_MAX_CONNECT_RETRIES 16
 342         u32 drive_jiffies;
 343
 344         u32 timo_slot;
 345
 346
 347         struct work_struct completion_worker;
 348 };
 349
 350 #define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
 351 #define SKD_READL(DEV, OFF)      skd_reg_read32(DEV, OFF)
 352 #define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
 353
 354 static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
 355 {
 356         u32 val;
 357
 358         if (likely(skdev->dbg_level < 2))
 359                 return readl(skdev->mem_map[1] + offset);
 360         else {
 361                 barrier();
 362                 val = readl(skdev->mem_map[1] + offset);
 363                 barrier();
 364                 pr_debug("%s:%s:%d offset %x = %x\n",
 365                          skdev->name, __func__, __LINE__, offset, val);
 366                 return val;
 367         }
 368
 369 }
 370
 371 static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
 372                                    u32 offset)
 373 {
 374         if (likely(skdev->dbg_level < 2)) {
 375                 writel(val, skdev->mem_map[1] + offset);
 376                 barrier();
 377         } else {
 378                 barrier();
 379                 writel(val, skdev->mem_map[1] + offset);
 380                 barrier();
 381                 pr_debug("%s:%s:%d offset %x = %x\n",
 382                          skdev->name, __func__, __LINE__, offset, val);
 383         }
 384 }
 385
 386 static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
 387                                    u32 offset)
 388 {
 389         if (likely(skdev->dbg_level < 2)) {
 390                 writeq(val, skdev->mem_map[1] + offset);
 391                 barrier();
 392         } else {
 393                 barrier();
 394                 writeq(val, skdev->mem_map[1] + offset);
 395                 barrier();
 396                 pr_debug("%s:%s:%d offset %x = %016llx\n",
 397                          skdev->name, __func__, __LINE__, offset, val);
 398         }
 399 }
 400
 401
 402 #define SKD_IRQ_DEFAULT SKD_IRQ_MSI
 403 static int skd_isr_type = SKD_IRQ_DEFAULT;
 404
 405 module_param(skd_isr_type, int, 0444);
 406 MODULE_PARM_DESC(skd_isr_type, "Interrupt type capability."
 407                  " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
 408
 409 #define SKD_MAX_REQ_PER_MSG_DEFAULT 1
 410 static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
 411
 412 module_param(skd_max_req_per_msg, int, 0444);
 413 MODULE_PARM_DESC(skd_max_req_per_msg,
 414                  "Maximum SCSI requests packed in a single message."
 415                  " (1-14, default==1)");
 416
 417 #define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
 418 #define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
 419 static int skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
 420
 421 module_param(skd_max_queue_depth, int, 0444);
 422 MODULE_PARM_DESC(skd_max_queue_depth,
 423                  "Maximum SCSI requests issued to s1120."
 424                  " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR ")");
 425
 426 static int skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
 427 module_param(skd_sgs_per_request, int, 0444);
 428 MODULE_PARM_DESC(skd_sgs_per_request,
 429                  "Maximum SG elements per block request."
 430                  " (1-4096, default==256)");
 431
 432 static int skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
 433 module_param(skd_max_pass_thru, int, 0444);
 434 MODULE_PARM_DESC(skd_max_pass_thru,
 435                  "Maximum SCSI pass-thru at a time." " (1-50, default==32)");
 436
 437 module_param(skd_dbg_level, int, 0444);
 438 MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
 439
 440 module_param(skd_isr_comp_limit, int, 0444);
 441 MODULE_PARM_DESC(skd_isr_comp_limit, "s1120 isr comp limit (0=none) default=4");
 442
 443 /* Major device number dynamically assigned. */
 444 static u32 skd_major;
 445
 446 static void skd_destruct(struct skd_device *skdev);
 447 static const struct block_device_operations skd_blockdev_ops;
 448 static void skd_send_fitmsg(struct skd_device *skdev,
 449                             struct skd_fitmsg_context *skmsg);
 450 static void skd_send_special_fitmsg(struct skd_device *skdev,
 451                                     struct skd_special_context *skspcl);
 452 static void skd_request_fn(struct request_queue *rq);
 453 static void skd_end_request(struct skd_device *skdev,
 454                             struct skd_request_context *skreq, int error);
 455 static int skd_preop_sg_list(struct skd_device *skdev,
 456                              struct skd_request_context *skreq);
 457 static void skd_postop_sg_list(struct skd_device *skdev,
 458                                struct skd_request_context *skreq);
 459
 460 static void skd_restart_device(struct skd_device *skdev);
 461 static int skd_quiesce_dev(struct skd_device *skdev);
 462 static int skd_unquiesce_dev(struct skd_device *skdev);
 463 static void skd_release_special(struct skd_device *skdev,
 464                                 struct skd_special_context *skspcl);
 465 static void skd_disable_interrupts(struct skd_device *skdev);
 466 static void skd_isr_fwstate(struct skd_device *skdev);
 467 static void skd_recover_requests(struct skd_device *skdev, int requeue);
 468 static void skd_soft_reset(struct skd_device *skdev);
 469
 470 static const char *skd_name(struct skd_device *skdev);
 471 const char *skd_drive_state_to_str(int state);
 472 const char *skd_skdev_state_to_str(enum skd_drvr_state state);
 473 static void skd_log_skdev(struct skd_device *skdev, const char *event);
 474 static void skd_log_skmsg(struct skd_device *skdev,
 475                           struct skd_fitmsg_context *skmsg, const char *event);
 476 static void skd_log_skreq(struct skd_device *skdev,
 477                           struct skd_request_context *skreq, const char *event);
 478
 479 /*
 480  *****************************************************************************
 481  * READ/WRITE REQUESTS
 482  *****************************************************************************
 483  */
 484 static void skd_fail_all_pending(struct skd_device *skdev)
 485 {
 486         struct request_queue *q = skdev->queue;
 487         struct request *req;
 488
 489         for (;; ) {
 490                 req = blk_peek_request(q);
 491                 if (req == NULL)
 492                         break;
 493                 blk_start_request(req);
 494                 __blk_end_request_all(req, -EIO);
 495         }
 496 }
 497
 498 static void
 499 skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
 500                 int data_dir, unsigned lba,
 501                 unsigned count)
 502 {
 503         if (data_dir == READ)
 504                 scsi_req->cdb[0] = 0x28;
 505         else
 506                 scsi_req->cdb[0] = 0x2a;
 507
 508         scsi_req->cdb[1] = 0;
 509         scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
 510         scsi_req->cdb[3] = (lba & 0xff0000) >> 16;
 511         scsi_req->cdb[4] = (lba & 0xff00) >> 8;
 512         scsi_req->cdb[5] = (lba & 0xff);
 513         scsi_req->cdb[6] = 0;
 514         scsi_req->cdb[7] = (count & 0xff00) >> 8;
 515         scsi_req->cdb[8] = count & 0xff;
 516         scsi_req->cdb[9] = 0;
 517 }
 518
 519 static void
 520 skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
 521                             struct skd_request_context *skreq)
 522 {
 523         skreq->flush_cmd = 1;
 524
 525         scsi_req->cdb[0] = 0x35;
 526         scsi_req->cdb[1] = 0;
 527         scsi_req->cdb[2] = 0;
 528         scsi_req->cdb[3] = 0;
 529         scsi_req->cdb[4] = 0;
 530         scsi_req->cdb[5] = 0;
 531         scsi_req->cdb[6] = 0;
 532         scsi_req->cdb[7] = 0;
 533         scsi_req->cdb[8] = 0;
 534         scsi_req->cdb[9] = 0;
 535 }
 536
 537 static void skd_request_fn_not_online(struct request_queue *q);
 538
 539 static void skd_request_fn(struct request_queue *q)
 540 {
 541         struct skd_device *skdev = q->queuedata;
 542         struct skd_fitmsg_context *skmsg = NULL;
 543         struct fit_msg_hdr *fmh = NULL;
 544         struct skd_request_context *skreq;
 545         struct request *req = NULL;
 546         struct skd_scsi_request *scsi_req;
 547         unsigned long io_flags;
 548         int error;
 549         u32 lba;
 550         u32 count;
 551         int data_dir;
 552         u32 be_lba;
 553         u32 be_count;
 554         u64 be_dmaa;
 555         u64 cmdctxt;
 556         u32 timo_slot;
 557         void *cmd_ptr;
 558         int flush, fua;
 559
 560         if (skdev->state != SKD_DRVR_STATE_ONLINE) {
 561                 skd_request_fn_not_online(q);
 562                 return;
 563         }
 564
 565         if (blk_queue_stopped(skdev->queue)) {
 566                 if (skdev->skmsg_free_list == NULL ||
 567                     skdev->skreq_free_list == NULL ||
 568                     skdev->in_flight >= skdev->queue_low_water_mark)
 569                         /* There is still some kind of shortage */
 570                         return;
 571
 572                 queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
 573         }
 574
 575         /*
 576          * Stop conditions:
 577          *  - There are no more native requests
 578          *  - There are already the maximum number of requests in progress
 579          *  - There are no more skd_request_context entries
 580          *  - There are no more FIT msg buffers
 581          */
 582         for (;; ) {
 583
 584                 flush = fua = 0;
 585
 586                 req = blk_peek_request(q);
 587
 588                 /* Are there any native requests to start? */
 589                 if (req == NULL)
 590                         break;
 591
 592                 lba = (u32)blk_rq_pos(req);
 593                 count = blk_rq_sectors(req);
 594                 data_dir = rq_data_dir(req);
 595                 io_flags = req->cmd_flags;
 596
 597                 if (req_op(req) == REQ_OP_FLUSH)
 598                         flush++;
 599
 600                 if (io_flags & REQ_FUA)
 601                         fua++;
 602
 603                 pr_debug("%s:%s:%d new req=%p lba=%u(0x%x) "
 604                          "count=%u(0x%x) dir=%d\n",
 605                          skdev->name, __func__, __LINE__,
 606                          req, lba, lba, count, count, data_dir);
 607
 608                 /* At this point we know there is a request */
 609
 610                 /* Are too many requets already in progress? */
 611                 if (skdev->in_flight >= skdev->cur_max_queue_depth) {
 612                         pr_debug("%s:%s:%d qdepth %d, limit %d\n",
 613                                  skdev->name, __func__, __LINE__,
 614                                  skdev->in_flight, skdev->cur_max_queue_depth);
 615                         break;
 616                 }
 617
 618                 /* Is a skd_request_context available? */
 619                 skreq = skdev->skreq_free_list;
 620                 if (skreq == NULL) {
 621                         pr_debug("%s:%s:%d Out of req=%p\n",
 622                                  skdev->name, __func__, __LINE__, q);
 623                         break;
 624                 }
 625                 SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
 626                 SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
 627
 628                 /* Now we check to see if we can get a fit msg */
 629                 if (skmsg == NULL) {
 630                         if (skdev->skmsg_free_list == NULL) {
 631                                 pr_debug("%s:%s:%d Out of msg\n",
 632                                          skdev->name, __func__, __LINE__);
 633                                 break;
 634                         }
 635                 }
 636
 637                 skreq->flush_cmd = 0;
 638                 skreq->n_sg = 0;
 639                 skreq->sg_byte_count = 0;
 640
 641                 /*
 642                  * OK to now dequeue request from q.
 643                  *
 644                  * At this point we are comitted to either start or reject
 645                  * the native request. Note that skd_request_context is
 646                  * available but is still at the head of the free list.
 647                  */
 648                 blk_start_request(req);
 649                 skreq->req = req;
 650                 skreq->fitmsg_id = 0;
 651
 652                 /* Either a FIT msg is in progress or we have to start one. */
 653                 if (skmsg == NULL) {
 654                         /* Are there any FIT msg buffers available? */
 655                         skmsg = skdev->skmsg_free_list;
 656                         if (skmsg == NULL) {
 657                                 pr_debug("%s:%s:%d Out of msg skdev=%p\n",
 658                                          skdev->name, __func__, __LINE__,
 659                                          skdev);
 660                                 break;
 661                         }
 662                         SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
 663                         SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
 664
 665                         skdev->skmsg_free_list = skmsg->next;
 666
 667                         skmsg->state = SKD_MSG_STATE_BUSY;
 668                         skmsg->id += SKD_ID_INCR;
 669
 670                         /* Initialize the FIT msg header */
 671                         fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
 672                         memset(fmh, 0, sizeof(*fmh));
 673                         fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
 674                         skmsg->length = sizeof(*fmh);
 675                 }
 676
 677                 skreq->fitmsg_id = skmsg->id;
 678
 679                 /*
 680                  * Note that a FIT msg may have just been started
 681                  * but contains no SoFIT requests yet.
 682                  */
 683
 684                 /*
 685                  * Transcode the request, checking as we go. The outcome of
 686                  * the transcoding is represented by the error variable.
 687                  */
 688                 cmd_ptr = &skmsg->msg_buf[skmsg->length];
 689                 memset(cmd_ptr, 0, 32);
 690
 691                 be_lba = cpu_to_be32(lba);
 692                 be_count = cpu_to_be32(count);
 693                 be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
 694                 cmdctxt = skreq->id + SKD_ID_INCR;
 695
 696                 scsi_req = cmd_ptr;
 697                 scsi_req->hdr.tag = cmdctxt;
 698                 scsi_req->hdr.sg_list_dma_address = be_dmaa;
 699
 700                 if (data_dir == READ)
 701                         skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
 702                 else
 703                         skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
 704
 705                 if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
 706                         skd_prep_zerosize_flush_cdb(scsi_req, skreq);
 707                         SKD_ASSERT(skreq->flush_cmd == 1);
 708
 709                 } else {
 710                         skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
 711                 }
 712
 713                 if (fua)
 714                         scsi_req->cdb[1] |= SKD_FUA_NV;
 715
 716                 if (!req->bio)
 717                         goto skip_sg;
 718
 719                 error = skd_preop_sg_list(skdev, skreq);
 720
 721                 if (error != 0) {
 722                         /*
 723                          * Complete the native request with error.
 724                          * Note that the request context is still at the
 725                          * head of the free list, and that the SoFIT request
 726                          * was encoded into the FIT msg buffer but the FIT
 727                          * msg length has not been updated. In short, the
 728                          * only resource that has been allocated but might
 729                          * not be used is that the FIT msg could be empty.
 730                          */
 731                         pr_debug("%s:%s:%d error Out\n",
 732                                  skdev->name, __func__, __LINE__);
 733                         skd_end_request(skdev, skreq, error);
 734                         continue;
 735                 }
 736
 737 skip_sg:
 738                 scsi_req->hdr.sg_list_len_bytes =
 739                         cpu_to_be32(skreq->sg_byte_count);
 740
 741                 /* Complete resource allocations. */
 742                 skdev->skreq_free_list = skreq->next;
 743                 skreq->state = SKD_REQ_STATE_BUSY;
 744                 skreq->id += SKD_ID_INCR;
 745
 746                 skmsg->length += sizeof(struct skd_scsi_request);
 747                 fmh->num_protocol_cmds_coalesced++;
 748
 749                 /*
 750                  * Update the active request counts.
 751                  * Capture the timeout timestamp.
 752                  */
 753                 skreq->timeout_stamp = skdev->timeout_stamp;
 754                 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
 755                 skdev->timeout_slot[timo_slot]++;
 756                 skdev->in_flight++;
 757                 pr_debug("%s:%s:%d req=0x%x busy=%d\n",
 758                          skdev->name, __func__, __LINE__,
 759                          skreq->id, skdev->in_flight);
 760
 761                 /*
 762                  * If the FIT msg buffer is full send it.
 763                  */
 764                 if (skmsg->length >= SKD_N_FITMSG_BYTES ||
 765                     fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
 766                         skd_send_fitmsg(skdev, skmsg);
 767                         skmsg = NULL;
 768                         fmh = NULL;
 769                 }
 770         }
 771
 772         /*
 773          * Is a FIT msg in progress? If it is empty put the buffer back
 774          * on the free list. If it is non-empty send what we got.
 775          * This minimizes latency when there are fewer requests than
 776          * what fits in a FIT msg.
 777          */
 778         if (skmsg != NULL) {
 779                 /* Bigger than just a FIT msg header? */
 780                 if (skmsg->length > sizeof(struct fit_msg_hdr)) {
 781                         pr_debug("%s:%s:%d sending msg=%p, len %d\n",
 782                                  skdev->name, __func__, __LINE__,
 783                                  skmsg, skmsg->length);
 784                         skd_send_fitmsg(skdev, skmsg);
 785                 } else {
 786                         /*
 787                          * The FIT msg is empty. It means we got started
 788                          * on the msg, but the requests were rejected.
 789                          */
 790                         skmsg->state = SKD_MSG_STATE_IDLE;
 791                         skmsg->id += SKD_ID_INCR;
 792                         skmsg->next = skdev->skmsg_free_list;
 793                         skdev->skmsg_free_list = skmsg;
 794                 }
 795                 skmsg = NULL;
 796                 fmh = NULL;
 797         }
 798
 799         /*
 800          * If req is non-NULL it means there is something to do but
 801          * we are out of a resource.
 802          */
 803         if (req)
 804                 blk_stop_queue(skdev->queue);
 805 }
 806
 807 static void skd_end_request(struct skd_device *skdev,
 808                             struct skd_request_context *skreq, int error)
 809 {
 810         if (unlikely(error)) {
 811                 struct request *req = skreq->req;
 812                 char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
 813                 u32 lba = (u32)blk_rq_pos(req);
 814                 u32 count = blk_rq_sectors(req);
 815
 816                 pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
 817                        skd_name(skdev), cmd, lba, count, skreq->id);
 818         } else
 819                 pr_debug("%s:%s:%d id=0x%x error=%d\n",
 820                          skdev->name, __func__, __LINE__, skreq->id, error);
 821
 822         __blk_end_request_all(skreq->req, error);
 823 }
 824
 825 static int skd_preop_sg_list(struct skd_device *skdev,
 826                              struct skd_request_context *skreq)
 827 {
 828         struct request *req = skreq->req;
 829         int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
 830         int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
 831         struct scatterlist *sg = &skreq->sg[0];
 832         int n_sg;
 833         int i;
 834
 835         skreq->sg_byte_count = 0;
 836
 837         /* SKD_ASSERT(skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD ||
 838                    skreq->sg_data_dir == SKD_DATA_DIR_CARD_TO_HOST); */
 839
 840         n_sg = blk_rq_map_sg(skdev->queue, req, sg);
 841         if (n_sg <= 0)
 842                 return -EINVAL;
 843
 844         /*
 845          * Map scatterlist to PCI bus addresses.
 846          * Note PCI might change the number of entries.
 847          */
 848         n_sg = pci_map_sg(skdev->pdev, sg, n_sg, pci_dir);
 849         if (n_sg <= 0)
 850                 return -EINVAL;
 851
 852         SKD_ASSERT(n_sg <= skdev->sgs_per_request);
 853
 854         skreq->n_sg = n_sg;
 855
 856         for (i = 0; i < n_sg; i++) {
 857                 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
 858                 u32 cnt = sg_dma_len(&sg[i]);
 859                 uint64_t dma_addr = sg_dma_address(&sg[i]);
 860
 861                 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
 862                 sgd->byte_count = cnt;
 863                 skreq->sg_byte_count += cnt;
 864                 sgd->host_side_addr = dma_addr;
 865                 sgd->dev_side_addr = 0;
 866         }
 867
 868         skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
 869         skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
 870
 871         if (unlikely(skdev->dbg_level > 1)) {
 872                 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
 873                          skdev->name, __func__, __LINE__,
 874                          skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
 875                 for (i = 0; i < n_sg; i++) {
 876                         struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
 877                         pr_debug("%s:%s:%d   sg[%d] count=%u ctrl=0x%x "
 878                                  "addr=0x%llx next=0x%llx\n",
 879                                  skdev->name, __func__, __LINE__,
 880                                  i, sgd->byte_count, sgd->control,
 881                                  sgd->host_side_addr, sgd->next_desc_ptr);
 882                 }
 883         }
 884
 885         return 0;
 886 }
 887
 888 static void skd_postop_sg_list(struct skd_device *skdev,
 889                                struct skd_request_context *skreq)
 890 {
 891         int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
 892         int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
 893
 894         /*
 895          * restore the next ptr for next IO request so we
 896          * don't have to set it every time.
 897          */
 898         skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
 899                 skreq->sksg_dma_address +
 900                 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
 901         pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, pci_dir);
 902 }
 903
 904 static void skd_request_fn_not_online(struct request_queue *q)
 905 {
 906         struct skd_device *skdev = q->queuedata;
 907         int error;
 908
 909         SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
 910
 911         skd_log_skdev(skdev, "req_not_online");
 912         switch (skdev->state) {
 913         case SKD_DRVR_STATE_PAUSING:
 914         case SKD_DRVR_STATE_PAUSED:
 915         case SKD_DRVR_STATE_STARTING:
 916         case SKD_DRVR_STATE_RESTARTING:
 917         case SKD_DRVR_STATE_WAIT_BOOT:
 918         /* In case of starting, we haven't started the queue,
 919          * so we can't get here... but requests are
 920          * possibly hanging out waiting for us because we
 921          * reported the dev/skd0 already.  They'll wait
 922          * forever if connect doesn't complete.
 923          * What to do??? delay dev/skd0 ??
 924          */
 925         case SKD_DRVR_STATE_BUSY:
 926         case SKD_DRVR_STATE_BUSY_IMMINENT:
 927         case SKD_DRVR_STATE_BUSY_ERASE:
 928         case SKD_DRVR_STATE_DRAINING_TIMEOUT:
 929                 return;
 930
 931         case SKD_DRVR_STATE_BUSY_SANITIZE:
 932         case SKD_DRVR_STATE_STOPPING:
 933         case SKD_DRVR_STATE_SYNCING:
 934         case SKD_DRVR_STATE_FAULT:
 935         case SKD_DRVR_STATE_DISAPPEARED:
 936         default:
 937                 error = -EIO;
 938                 break;
 939         }
 940
 941         /* If we get here, terminate all pending block requeusts
 942          * with EIO and any scsi pass thru with appropriate sense
 943          */
 944
 945         skd_fail_all_pending(skdev);
 946 }
 947
 948 /*
 949  *****************************************************************************
 950  * TIMER
 951  *****************************************************************************
 952  */
 953
 954 static void skd_timer_tick_not_online(struct skd_device *skdev);
 955
 956 static void skd_timer_tick(ulong arg)
 957 {
 958         struct skd_device *skdev = (struct skd_device *)arg;
 959
 960         u32 timo_slot;
 961         u32 overdue_timestamp;
 962         unsigned long reqflags;
 963         u32 state;
 964
 965         if (skdev->state == SKD_DRVR_STATE_FAULT)
 966                 /* The driver has declared fault, and we want it to
 967                  * stay that way until driver is reloaded.
 968                  */
 969                 return;
 970
 971         spin_lock_irqsave(&skdev->lock, reqflags);
 972
 973         state = SKD_READL(skdev, FIT_STATUS);
 974         state &= FIT_SR_DRIVE_STATE_MASK;
 975         if (state != skdev->drive_state)
 976                 skd_isr_fwstate(skdev);
 977
 978         if (skdev->state != SKD_DRVR_STATE_ONLINE) {
 979                 skd_timer_tick_not_online(skdev);
 980                 goto timer_func_out;
 981         }
 982         skdev->timeout_stamp++;
 983         timo_slot = skdev->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
 984
 985         /*
 986          * All requests that happened during the previous use of
 987          * this slot should be done by now. The previous use was
 988          * over 7 seconds ago.
 989          */
 990         if (skdev->timeout_slot[timo_slot] == 0)
 991                 goto timer_func_out;
 992
 993         /* Something is overdue */
 994         overdue_timestamp = skdev->timeout_stamp - SKD_N_TIMEOUT_SLOT;
 995
 996         pr_debug("%s:%s:%d found %d timeouts, draining busy=%d\n",
 997                  skdev->name, __func__, __LINE__,
 998                  skdev->timeout_slot[timo_slot], skdev->in_flight);
 999         pr_err("(%s): Overdue IOs (%d), busy %d\n",
1000                skd_name(skdev), skdev->timeout_slot[timo_slot],
1001                skdev->in_flight);
1002
1003         skdev->timer_countdown = SKD_DRAINING_TIMO;
1004         skdev->state = SKD_DRVR_STATE_DRAINING_TIMEOUT;
1005         skdev->timo_slot = timo_slot;
1006         blk_stop_queue(skdev->queue);
1007
1008 timer_func_out:
1009         mod_timer(&skdev->timer, (jiffies + HZ));
1010
1011         spin_unlock_irqrestore(&skdev->lock, reqflags);
1012 }
1013
1014 static void skd_timer_tick_not_online(struct skd_device *skdev)
1015 {
1016         switch (skdev->state) {
1017         case SKD_DRVR_STATE_IDLE:
1018         case SKD_DRVR_STATE_LOAD:
1019                 break;
1020         case SKD_DRVR_STATE_BUSY_SANITIZE:
1021                 pr_debug("%s:%s:%d drive busy sanitize[%x], driver[%x]\n",
1022                          skdev->name, __func__, __LINE__,
1023                          skdev->drive_state, skdev->state);
1024                 /* If we've been in sanitize for 3 seconds, we figure we're not
1025                  * going to get anymore completions, so recover requests now
1026                  */
1027                 if (skdev->timer_countdown > 0) {
1028                         skdev->timer_countdown--;
1029                         return;
1030                 }
1031                 skd_recover_requests(skdev, 0);
1032                 break;
1033
1034         case SKD_DRVR_STATE_BUSY:
1035         case SKD_DRVR_STATE_BUSY_IMMINENT:
1036         case SKD_DRVR_STATE_BUSY_ERASE:
1037                 pr_debug("%s:%s:%d busy[%x], countdown=%d\n",
1038                          skdev->name, __func__, __LINE__,
1039                          skdev->state, skdev->timer_countdown);
1040                 if (skdev->timer_countdown > 0) {
1041                         skdev->timer_countdown--;
1042                         return;
1043                 }
1044                 pr_debug("%s:%s:%d busy[%x], timedout=%d, restarting device.",
1045                          skdev->name, __func__, __LINE__,
1046                          skdev->state, skdev->timer_countdown);
1047                 skd_restart_device(skdev);
1048                 break;
1049
1050         case SKD_DRVR_STATE_WAIT_BOOT:
1051         case SKD_DRVR_STATE_STARTING:
1052                 if (skdev->timer_countdown > 0) {
1053                         skdev->timer_countdown--;
1054                         return;
1055                 }
1056                 /* For now, we fault the drive.  Could attempt resets to
1057                  * revcover at some point. */
1058                 skdev->state = SKD_DRVR_STATE_FAULT;
1059
1060                 pr_err("(%s): DriveFault Connect Timeout (%x)\n",
1061                        skd_name(skdev), skdev->drive_state);
1062
1063                 /*start the queue so we can respond with error to requests */
1064                 /* wakeup anyone waiting for startup complete */
1065                 blk_start_queue(skdev->queue);
1066                 skdev->gendisk_on = -1;
1067                 wake_up_interruptible(&skdev->waitq);
1068                 break;
1069
1070         case SKD_DRVR_STATE_ONLINE:
1071                 /* shouldn't get here. */
1072                 break;
1073
1074         case SKD_DRVR_STATE_PAUSING:
1075         case SKD_DRVR_STATE_PAUSED:
1076                 break;
1077
1078         case SKD_DRVR_STATE_DRAINING_TIMEOUT:
1079                 pr_debug("%s:%s:%d "
1080                          "draining busy [%d] tick[%d] qdb[%d] tmls[%d]\n",
1081                          skdev->name, __func__, __LINE__,
1082                          skdev->timo_slot,
1083                          skdev->timer_countdown,
1084                          skdev->in_flight,
1085                          skdev->timeout_slot[skdev->timo_slot]);
1086                 /* if the slot has cleared we can let the I/O continue */
1087                 if (skdev->timeout_slot[skdev->timo_slot] == 0) {
1088                         pr_debug("%s:%s:%d Slot drained, starting queue.\n",
1089                                  skdev->name, __func__, __LINE__);
1090                         skdev->state = SKD_DRVR_STATE_ONLINE;
1091                         blk_start_queue(skdev->queue);
1092                         return;
1093                 }
1094                 if (skdev->timer_countdown > 0) {
1095                         skdev->timer_countdown--;
1096                         return;
1097                 }
1098                 skd_restart_device(skdev);
1099                 break;
1100
1101         case SKD_DRVR_STATE_RESTARTING:
1102                 if (skdev->timer_countdown > 0) {
1103                         skdev->timer_countdown--;
1104                         return;
1105                 }
1106                 /* For now, we fault the drive. Could attempt resets to
1107                  * revcover at some point. */
1108                 skdev->state = SKD_DRVR_STATE_FAULT;
1109                 pr_err("(%s): DriveFault Reconnect Timeout (%x)\n",
1110                        skd_name(skdev), skdev->drive_state);
1111
1112                 /*
1113                  * Recovering does two things:
1114                  * 1. completes IO with error
1115                  * 2. reclaims dma resources
1116                  * When is it safe to recover requests?
1117                  * - if the drive state is faulted
1118                  * - if the state is still soft reset after out timeout
1119                  * - if the drive registers are dead (state = FF)
1120                  * If it is "unsafe", we still need to recover, so we will
1121                  * disable pci bus mastering and disable our interrupts.
1122                  */
1123
1124                 if ((skdev->drive_state == FIT_SR_DRIVE_SOFT_RESET) ||
1125                     (skdev->drive_state == FIT_SR_DRIVE_FAULT) ||
1126                     (skdev->drive_state == FIT_SR_DRIVE_STATE_MASK))
1127                         /* It never came out of soft reset. Try to
1128                          * recover the requests and then let them
1129                          * fail. This is to mitigate hung processes. */
1130                         skd_recover_requests(skdev, 0);
1131                 else {
1132                         pr_err("(%s): Disable BusMaster (%x)\n",
1133                                skd_name(skdev), skdev->drive_state);
1134                         pci_disable_device(skdev->pdev);
1135                         skd_disable_interrupts(skdev);
1136                         skd_recover_requests(skdev, 0);
1137                 }
1138
1139                 /*start the queue so we can respond with error to requests */
1140                 /* wakeup anyone waiting for startup complete */
1141                 blk_start_queue(skdev->queue);
1142                 skdev->gendisk_on = -1;
1143                 wake_up_interruptible(&skdev->waitq);
1144                 break;
1145
1146         case SKD_DRVR_STATE_RESUMING:
1147         case SKD_DRVR_STATE_STOPPING:
1148         case SKD_DRVR_STATE_SYNCING:
1149         case SKD_DRVR_STATE_FAULT:
1150         case SKD_DRVR_STATE_DISAPPEARED:
1151         default:
1152                 break;
1153         }
1154 }
1155
1156 static int skd_start_timer(struct skd_device *skdev)
1157 {
1158         int rc;
1159
1160         init_timer(&skdev->timer);
1161         setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
1162
1163         rc = mod_timer(&skdev->timer, (jiffies + HZ));
1164         if (rc)
1165                 pr_err("%s: failed to start timer %d\n",
1166                        __func__, rc);
1167         return rc;
1168 }
1169
1170 static void skd_kill_timer(struct skd_device *skdev)
1171 {
1172         del_timer_sync(&skdev->timer);
1173 }
1174
1175 /*
1176  *****************************************************************************
1177  * IOCTL
1178  *****************************************************************************
1179  */
1180 static int skd_ioctl_sg_io(struct skd_device *skdev,
1181                            fmode_t mode, void __user *argp);
1182 static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1183                                         struct skd_sg_io *sksgio);
1184 static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1185                                    struct skd_sg_io *sksgio);
1186 static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1187                                     struct skd_sg_io *sksgio);
1188 static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1189                                  struct skd_sg_io *sksgio, int dxfer_dir);
1190 static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1191                                  struct skd_sg_io *sksgio);
1192 static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio);
1193 static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1194                                     struct skd_sg_io *sksgio);
1195 static int skd_sg_io_put_status(struct skd_device *skdev,
1196                                 struct skd_sg_io *sksgio);
1197
1198 static void skd_complete_special(struct skd_device *skdev,
1199                                  volatile struct fit_completion_entry_v1
1200                                  *skcomp,
1201                                  volatile struct fit_comp_error_info *skerr,
1202                                  struct skd_special_context *skspcl);
1203
1204 static int skd_bdev_ioctl(struct block_device *bdev, fmode_t mode,
1205                           uint cmd_in, ulong arg)
1206 {
1207         int rc = 0;
1208         struct gendisk *disk = bdev->bd_disk;
1209         struct skd_device *skdev = disk->private_data;
1210         void __user *p = (void *)arg;
1211
1212         pr_debug("%s:%s:%d %s: CMD[%s] ioctl  mode 0x%x, cmd 0x%x arg %0lx\n",
1213                  skdev->name, __func__, __LINE__,
1214                  disk->disk_name, current->comm, mode, cmd_in, arg);
1215
1216         if (!capable(CAP_SYS_ADMIN))
1217                 return -EPERM;
1218
1219         switch (cmd_in) {
1220         case SG_SET_TIMEOUT:
1221         case SG_GET_TIMEOUT:
1222         case SG_GET_VERSION_NUM:
1223                 rc = scsi_cmd_ioctl(disk->queue, disk, mode, cmd_in, p);
1224                 break;
1225         case SG_IO:
1226                 rc = skd_ioctl_sg_io(skdev, mode, p);
1227                 break;
1228
1229         default:
1230                 rc = -ENOTTY;
1231                 break;
1232         }
1233
1234         pr_debug("%s:%s:%d %s:  completion rc %d\n",
1235                  skdev->name, __func__, __LINE__, disk->disk_name, rc);
1236         return rc;
1237 }
1238
1239 static int skd_ioctl_sg_io(struct skd_device *skdev, fmode_t mode,
1240                            void __user *argp)
1241 {
1242         int rc;
1243         struct skd_sg_io sksgio;
1244
1245         memset(&sksgio, 0, sizeof(sksgio));
1246         sksgio.mode = mode;
1247         sksgio.argp = argp;
1248         sksgio.iov = &sksgio.no_iov_iov;
1249
1250         switch (skdev->state) {
1251         case SKD_DRVR_STATE_ONLINE:
1252         case SKD_DRVR_STATE_BUSY_IMMINENT:
1253                 break;
1254
1255         default:
1256                 pr_debug("%s:%s:%d drive not online\n",
1257                          skdev->name, __func__, __LINE__);
1258                 rc = -ENXIO;
1259                 goto out;
1260         }
1261
1262         rc = skd_sg_io_get_and_check_args(skdev, &sksgio);
1263         if (rc)
1264                 goto out;
1265
1266         rc = skd_sg_io_obtain_skspcl(skdev, &sksgio);
1267         if (rc)
1268                 goto out;
1269
1270         rc = skd_sg_io_prep_buffering(skdev, &sksgio);
1271         if (rc)
1272                 goto out;
1273
1274         rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_TO_DEV);
1275         if (rc)
1276                 goto out;
1277
1278         rc = skd_sg_io_send_fitmsg(skdev, &sksgio);
1279         if (rc)
1280                 goto out;
1281
1282         rc = skd_sg_io_await(skdev, &sksgio);
1283         if (rc)
1284                 goto out;
1285
1286         rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_FROM_DEV);
1287         if (rc)
1288                 goto out;
1289
1290         rc = skd_sg_io_put_status(skdev, &sksgio);
1291         if (rc)
1292                 goto out;
1293
1294         rc = 0;
1295
1296 out:
1297         skd_sg_io_release_skspcl(skdev, &sksgio);
1298
1299         if (sksgio.iov != NULL && sksgio.iov != &sksgio.no_iov_iov)
1300                 kfree(sksgio.iov);
1301         return rc;
1302 }
1303
1304 static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1305                                         struct skd_sg_io *sksgio)
1306 {
1307         struct sg_io_hdr *sgp = &sksgio->sg;
1308         int i, acc;
1309
1310         if (!access_ok(VERIFY_WRITE, sksgio->argp, sizeof(sg_io_hdr_t))) {
1311                 pr_debug("%s:%s:%d access sg failed %p\n",
1312                          skdev->name, __func__, __LINE__, sksgio->argp);
1313                 return -EFAULT;
1314         }
1315
1316         if (__copy_from_user(sgp, sksgio->argp, sizeof(sg_io_hdr_t))) {
1317                 pr_debug("%s:%s:%d copy_from_user sg failed %p\n",
1318                          skdev->name, __func__, __LINE__, sksgio->argp);
1319                 return -EFAULT;
1320         }
1321
1322         if (sgp->interface_id != SG_INTERFACE_ID_ORIG) {
1323                 pr_debug("%s:%s:%d interface_id invalid 0x%x\n",
1324                          skdev->name, __func__, __LINE__, sgp->interface_id);
1325                 return -EINVAL;
1326         }
1327
1328         if (sgp->cmd_len > sizeof(sksgio->cdb)) {
1329                 pr_debug("%s:%s:%d cmd_len invalid %d\n",
1330                          skdev->name, __func__, __LINE__, sgp->cmd_len);
1331                 return -EINVAL;
1332         }
1333
1334         if (sgp->iovec_count > 256) {
1335                 pr_debug("%s:%s:%d iovec_count invalid %d\n",
1336                          skdev->name, __func__, __LINE__, sgp->iovec_count);
1337                 return -EINVAL;
1338         }
1339
1340         if (sgp->dxfer_len > (PAGE_SIZE * SKD_N_SG_PER_SPECIAL)) {
1341                 pr_debug("%s:%s:%d dxfer_len invalid %d\n",
1342                          skdev->name, __func__, __LINE__, sgp->dxfer_len);
1343                 return -EINVAL;
1344         }
1345
1346         switch (sgp->dxfer_direction) {
1347         case SG_DXFER_NONE:
1348                 acc = -1;
1349                 break;
1350
1351         case SG_DXFER_TO_DEV:
1352                 acc = VERIFY_READ;
1353                 break;
1354
1355         case SG_DXFER_FROM_DEV:
1356         case SG_DXFER_TO_FROM_DEV:
1357                 acc = VERIFY_WRITE;
1358                 break;
1359
1360         default:
1361                 pr_debug("%s:%s:%d dxfer_dir invalid %d\n",
1362                          skdev->name, __func__, __LINE__, sgp->dxfer_direction);
1363                 return -EINVAL;
1364         }
1365
1366         if (copy_from_user(sksgio->cdb, sgp->cmdp, sgp->cmd_len)) {
1367                 pr_debug("%s:%s:%d copy_from_user cmdp failed %p\n",
1368                          skdev->name, __func__, __LINE__, sgp->cmdp);
1369                 return -EFAULT;
1370         }
1371
1372         if (sgp->mx_sb_len != 0) {
1373                 if (!access_ok(VERIFY_WRITE, sgp->sbp, sgp->mx_sb_len)) {
1374                         pr_debug("%s:%s:%d access sbp failed %p\n",
1375                                  skdev->name, __func__, __LINE__, sgp->sbp);
1376                         return -EFAULT;
1377                 }
1378         }
1379
1380         if (sgp->iovec_count == 0) {
1381                 sksgio->iov[0].iov_base = sgp->dxferp;
1382                 sksgio->iov[0].iov_len = sgp->dxfer_len;
1383                 sksgio->iovcnt = 1;
1384                 sksgio->dxfer_len = sgp->dxfer_len;
1385         } else {
1386                 struct sg_iovec *iov;
1387                 uint nbytes = sizeof(*iov) * sgp->iovec_count;
1388                 size_t iov_data_len;
1389
1390                 iov = kmalloc(nbytes, GFP_KERNEL);
1391                 if (iov == NULL) {
1392                         pr_debug("%s:%s:%d alloc iovec failed %d\n",
1393                                  skdev->name, __func__, __LINE__,
1394                                  sgp->iovec_count);
1395                         return -ENOMEM;
1396                 }
1397                 sksgio->iov = iov;
1398                 sksgio->iovcnt = sgp->iovec_count;
1399
1400                 if (copy_from_user(iov, sgp->dxferp, nbytes)) {
1401                         pr_debug("%s:%s:%d copy_from_user iovec failed %p\n",
1402                                  skdev->name, __func__, __LINE__, sgp->dxferp);
1403                         return -EFAULT;
1404                 }
1405
1406                 /*
1407                  * Sum up the vecs, making sure they don't overflow
1408                  */
1409                 iov_data_len = 0;
1410                 for (i = 0; i < sgp->iovec_count; i++) {
1411                         if (iov_data_len + iov[i].iov_len < iov_data_len)
1412                                 return -EINVAL;
1413                         iov_data_len += iov[i].iov_len;
1414                 }
1415
1416                 /* SG_IO howto says that the shorter of the two wins */
1417                 if (sgp->dxfer_len < iov_data_len) {
1418                         sksgio->iovcnt = iov_shorten((struct iovec *)iov,
1419                                                      sgp->iovec_count,
1420                                                      sgp->dxfer_len);
1421                         sksgio->dxfer_len = sgp->dxfer_len;
1422                 } else
1423                         sksgio->dxfer_len = iov_data_len;
1424         }
1425
1426         if (sgp->dxfer_direction != SG_DXFER_NONE) {
1427                 struct sg_iovec *iov = sksgio->iov;
1428                 for (i = 0; i < sksgio->iovcnt; i++, iov++) {
1429                         if (!access_ok(acc, iov->iov_base, iov->iov_len)) {
1430                                 pr_debug("%s:%s:%d access data failed %p/%d\n",
1431                                          skdev->name, __func__, __LINE__,
1432                                          iov->iov_base, (int)iov->iov_len);
1433                                 return -EFAULT;
1434                         }
1435                 }
1436         }
1437
1438         return 0;
1439 }
1440
1441 static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1442                                    struct skd_sg_io *sksgio)
1443 {
1444         struct skd_special_context *skspcl = NULL;
1445         int rc;
1446
1447         for (;;) {
1448                 ulong flags;
1449
1450                 spin_lock_irqsave(&skdev->lock, flags);
1451                 skspcl = skdev->skspcl_free_list;
1452                 if (skspcl != NULL) {
1453                         skdev->skspcl_free_list =
1454                                 (struct skd_special_context *)skspcl->req.next;
1455                         skspcl->req.id += SKD_ID_INCR;
1456                         skspcl->req.state = SKD_REQ_STATE_SETUP;
1457                         skspcl->orphaned = 0;
1458                         skspcl->req.n_sg = 0;
1459                 }
1460                 spin_unlock_irqrestore(&skdev->lock, flags);
1461
1462                 if (skspcl != NULL) {
1463                         rc = 0;
1464                         break;
1465                 }
1466
1467                 pr_debug("%s:%s:%d blocking\n",
1468                          skdev->name, __func__, __LINE__);
1469
1470                 rc = wait_event_interruptible_timeout(
1471                                 skdev->waitq,
1472                                 (skdev->skspcl_free_list != NULL),
1473                                 msecs_to_jiffies(sksgio->sg.timeout));
1474
1475                 pr_debug("%s:%s:%d unblocking, rc=%d\n",
1476                          skdev->name, __func__, __LINE__, rc);
1477
1478                 if (rc <= 0) {
1479                         if (rc == 0)
1480                                 rc = -ETIMEDOUT;
1481                         else
1482                                 rc = -EINTR;
1483                         break;
1484                 }
1485                 /*
1486                  * If we get here rc > 0 meaning the timeout to
1487                  * wait_event_interruptible_timeout() had time left, hence the
1488                  * sought event -- non-empty free list -- happened.
1489                  * Retry the allocation.
1490                  */
1491         }
1492         sksgio->skspcl = skspcl;
1493
1494         return rc;
1495 }
1496
1497 static int skd_skreq_prep_buffering(struct skd_device *skdev,
1498                                     struct skd_request_context *skreq,
1499                                     u32 dxfer_len)
1500 {
1501         u32 resid = dxfer_len;
1502
1503         /*
1504          * The DMA engine must have aligned addresses and byte counts.
1505          */
1506         resid += (-resid) & 3;
1507         skreq->sg_byte_count = resid;
1508
1509         skreq->n_sg = 0;
1510
1511         while (resid > 0) {
1512                 u32 nbytes = PAGE_SIZE;
1513                 u32 ix = skreq->n_sg;
1514                 struct scatterlist *sg = &skreq->sg[ix];
1515                 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1516                 struct page *page;
1517
1518                 if (nbytes > resid)
1519                         nbytes = resid;
1520
1521                 page = alloc_page(GFP_KERNEL);
1522                 if (page == NULL)
1523                         return -ENOMEM;
1524
1525                 sg_set_page(sg, page, nbytes, 0);
1526
1527                 /* TODO: This should be going through a pci_???()
1528                  * routine to do proper mapping. */
1529                 sksg->control = FIT_SGD_CONTROL_NOT_LAST;
1530                 sksg->byte_count = nbytes;
1531
1532                 sksg->host_side_addr = sg_phys(sg);
1533
1534                 sksg->dev_side_addr = 0;
1535                 sksg->next_desc_ptr = skreq->sksg_dma_address +
1536                                       (ix + 1) * sizeof(*sksg);
1537
1538                 skreq->n_sg++;
1539                 resid -= nbytes;
1540         }
1541
1542         if (skreq->n_sg > 0) {
1543                 u32 ix = skreq->n_sg - 1;
1544                 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1545
1546                 sksg->control = FIT_SGD_CONTROL_LAST;
1547                 sksg->next_desc_ptr = 0;
1548         }
1549
1550         if (unlikely(skdev->dbg_level > 1)) {
1551                 u32 i;
1552
1553                 pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
1554                          skdev->name, __func__, __LINE__,
1555                          skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
1556                 for (i = 0; i < skreq->n_sg; i++) {
1557                         struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1558
1559                         pr_debug("%s:%s:%d   sg[%d] count=%u ctrl=0x%x "
1560                                  "addr=0x%llx next=0x%llx\n",
1561                                  skdev->name, __func__, __LINE__,
1562                                  i, sgd->byte_count, sgd->control,
1563                                  sgd->host_side_addr, sgd->next_desc_ptr);
1564                 }
1565         }
1566
1567         return 0;
1568 }
1569
1570 static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1571                                     struct skd_sg_io *sksgio)
1572 {
1573         struct skd_special_context *skspcl = sksgio->skspcl;
1574         struct skd_request_context *skreq = &skspcl->req;
1575         u32 dxfer_len = sksgio->dxfer_len;
1576         int rc;
1577
1578         rc = skd_skreq_prep_buffering(skdev, skreq, dxfer_len);
1579         /*
1580          * Eventually, errors or not, skd_release_special() is called
1581          * to recover allocations including partial allocations.
1582          */
1583         return rc;
1584 }
1585
1586 static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1587                                  struct skd_sg_io *sksgio, int dxfer_dir)
1588 {
1589         struct skd_special_context *skspcl = sksgio->skspcl;
1590         u32 iov_ix = 0;
1591         struct sg_iovec curiov;
1592         u32 sksg_ix = 0;
1593         u8 *bufp = NULL;
1594         u32 buf_len = 0;
1595         u32 resid = sksgio->dxfer_len;
1596         int rc;
1597
1598         curiov.iov_len = 0;
1599         curiov.iov_base = NULL;
1600
1601         if (dxfer_dir != sksgio->sg.dxfer_direction) {
1602                 if (dxfer_dir != SG_DXFER_TO_DEV ||
1603                     sksgio->sg.dxfer_direction != SG_DXFER_TO_FROM_DEV)
1604                         return 0;
1605         }
1606
1607         while (resid > 0) {
1608                 u32 nbytes = PAGE_SIZE;
1609
1610                 if (curiov.iov_len == 0) {
1611                         curiov = sksgio->iov[iov_ix++];
1612                         continue;
1613                 }
1614
1615                 if (buf_len == 0) {
1616                         struct page *page;
1617                         page = sg_page(&skspcl->req.sg[sksg_ix++]);
1618                         bufp = page_address(page);
1619                         buf_len = PAGE_SIZE;
1620                 }
1621
1622                 nbytes = min_t(u32, nbytes, resid);
1623                 nbytes = min_t(u32, nbytes, curiov.iov_len);
1624                 nbytes = min_t(u32, nbytes, buf_len);
1625
1626                 if (dxfer_dir == SG_DXFER_TO_DEV)
1627                         rc = __copy_from_user(bufp, curiov.iov_base, nbytes);
1628                 else
1629                         rc = __copy_to_user(curiov.iov_base, bufp, nbytes);
1630
1631                 if (rc)
1632                         return -EFAULT;
1633
1634                 resid -= nbytes;
1635                 curiov.iov_len -= nbytes;
1636                 curiov.iov_base += nbytes;
1637                 buf_len -= nbytes;
1638         }
1639
1640         return 0;
1641 }
1642
1643 static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1644                                  struct skd_sg_io *sksgio)
1645 {
1646         struct skd_special_context *skspcl = sksgio->skspcl;
1647         struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
1648         struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
1649
1650         memset(skspcl->msg_buf, 0, SKD_N_SPECIAL_FITMSG_BYTES);
1651
1652         /* Initialize the FIT msg header */
1653         fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1654         fmh->num_protocol_cmds_coalesced = 1;
1655
1656         /* Initialize the SCSI request */
1657         if (sksgio->sg.dxfer_direction != SG_DXFER_NONE)
1658                 scsi_req->hdr.sg_list_dma_address =
1659                         cpu_to_be64(skspcl->req.sksg_dma_address);
1660         scsi_req->hdr.tag = skspcl->req.id;
1661         scsi_req->hdr.sg_list_len_bytes =
1662                 cpu_to_be32(skspcl->req.sg_byte_count);
1663         memcpy(scsi_req->cdb, sksgio->cdb, sizeof(scsi_req->cdb));
1664
1665         skspcl->req.state = SKD_REQ_STATE_BUSY;
1666         skd_send_special_fitmsg(skdev, skspcl);
1667
1668         return 0;
1669 }
1670
1671 static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio)
1672 {
1673         unsigned long flags;
1674         int rc;
1675
1676         rc = wait_event_interruptible_timeout(skdev->waitq,
1677                                               (sksgio->skspcl->req.state !=
1678                                                SKD_REQ_STATE_BUSY),
1679                                               msecs_to_jiffies(sksgio->sg.
1680                                                                timeout));
1681
1682         spin_lock_irqsave(&skdev->lock, flags);
1683
1684         if (sksgio->skspcl->req.state == SKD_REQ_STATE_ABORTED) {
1685                 pr_debug("%s:%s:%d skspcl %p aborted\n",
1686                          skdev->name, __func__, __LINE__, sksgio->skspcl);
1687
1688                 /* Build check cond, sense and let command finish. */
1689                 /* For a timeout, we must fabricate completion and sense
1690                  * data to complete the command */
1691                 sksgio->skspcl->req.completion.status =
1692                         SAM_STAT_CHECK_CONDITION;
1693
1694                 memset(&sksgio->skspcl->req.err_info, 0,
1695                        sizeof(sksgio->skspcl->req.err_info));
1696                 sksgio->skspcl->req.err_info.type = 0x70;
1697                 sksgio->skspcl->req.err_info.key = ABORTED_COMMAND;
1698                 sksgio->skspcl->req.err_info.code = 0x44;
1699                 sksgio->skspcl->req.err_info.qual = 0;
1700                 rc = 0;
1701         } else if (sksgio->skspcl->req.state != SKD_REQ_STATE_BUSY)
1702                 /* No longer on the adapter. We finish. */
1703                 rc = 0;
1704         else {
1705                 /* Something's gone wrong. Still busy. Timeout or
1706                  * user interrupted (control-C). Mark as an orphan
1707                  * so it will be disposed when completed. */
1708                 sksgio->skspcl->orphaned = 1;
1709                 sksgio->skspcl = NULL;
1710                 if (rc == 0) {
1711                         pr_debug("%s:%s:%d timed out %p (%u ms)\n",
1712                                  skdev->name, __func__, __LINE__,
1713                                  sksgio, sksgio->sg.timeout);
1714                         rc = -ETIMEDOUT;
1715                 } else {
1716                         pr_debug("%s:%s:%d cntlc %p\n",
1717                                  skdev->name, __func__, __LINE__, sksgio);
1718                         rc = -EINTR;
1719                 }
1720         }
1721
1722         spin_unlock_irqrestore(&skdev->lock, flags);
1723
1724         return rc;
1725 }
1726
1727 static int skd_sg_io_put_status(struct skd_device *skdev,
1728                                 struct skd_sg_io *sksgio)
1729 {
1730         struct sg_io_hdr *sgp = &sksgio->sg;
1731         struct skd_special_context *skspcl = sksgio->skspcl;
1732         int resid = 0;
1733
1734         u32 nb = be32_to_cpu(skspcl->req.completion.num_returned_bytes);
1735
1736         sgp->status = skspcl->req.completion.status;
1737         resid = sksgio->dxfer_len - nb;
1738
1739         sgp->masked_status = sgp->status & STATUS_MASK;
1740         sgp->msg_status = 0;
1741         sgp->host_status = 0;
1742         sgp->driver_status = 0;
1743         sgp->resid = resid;
1744         if (sgp->masked_status || sgp->host_status || sgp->driver_status)
1745                 sgp->info |= SG_INFO_CHECK;
1746
1747         pr_debug("%s:%s:%d status %x masked %x resid 0x%x\n",
1748                  skdev->name, __func__, __LINE__,
1749                  sgp->status, sgp->masked_status, sgp->resid);
1750
1751         if (sgp->masked_status == SAM_STAT_CHECK_CONDITION) {
1752                 if (sgp->mx_sb_len > 0) {
1753                         struct fit_comp_error_info *ei = &skspcl->req.err_info;
1754                         u32 nbytes = sizeof(*ei);
1755
1756                         nbytes = min_t(u32, nbytes, sgp->mx_sb_len);
1757
1758                         sgp->sb_len_wr = nbytes;
1759
1760                         if (__copy_to_user(sgp->sbp, ei, nbytes)) {
1761                                 pr_debug("%s:%s:%d copy_to_user sense failed %p\n",
1762                                          skdev->name, __func__, __LINE__,
1763                                          sgp->sbp);
1764                                 return -EFAULT;
1765                         }
1766                 }
1767         }
1768
1769         if (__copy_to_user(sksgio->argp, sgp, sizeof(sg_io_hdr_t))) {
1770                 pr_debug("%s:%s:%d copy_to_user sg failed %p\n",
1771                          skdev->name, __func__, __LINE__, sksgio->argp);
1772                 return -EFAULT;
1773         }
1774
1775         return 0;
1776 }
1777
1778 static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1779                                     struct skd_sg_io *sksgio)
1780 {
1781         struct skd_special_context *skspcl = sksgio->skspcl;
1782
1783         if (skspcl != NULL) {
1784                 ulong flags;
1785
1786                 sksgio->skspcl = NULL;
1787
1788                 spin_lock_irqsave(&skdev->lock, flags);
1789                 skd_release_special(skdev, skspcl);
1790                 spin_unlock_irqrestore(&skdev->lock, flags);
1791         }
1792
1793         return 0;
1794 }
1795
1796 /*
1797  *****************************************************************************
1798  * INTERNAL REQUESTS -- generated by driver itself
1799  *****************************************************************************
1800  */
1801
1802 static int skd_format_internal_skspcl(struct skd_device *skdev)
1803 {
1804         struct skd_special_context *skspcl = &skdev->internal_skspcl;
1805         struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1806         struct fit_msg_hdr *fmh;
1807         uint64_t dma_address;
1808         struct skd_scsi_request *scsi;
1809
1810         fmh = (struct fit_msg_hdr *)&skspcl->msg_buf[0];
1811         fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
1812         fmh->num_protocol_cmds_coalesced = 1;
1813
1814         scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1815         memset(scsi, 0, sizeof(*scsi));
1816         dma_address = skspcl->req.sksg_dma_address;
1817         scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
1818         sgd->control = FIT_SGD_CONTROL_LAST;
1819         sgd->byte_count = 0;
1820         sgd->host_side_addr = skspcl->db_dma_address;
1821         sgd->dev_side_addr = 0;
1822         sgd->next_desc_ptr = 0LL;
1823
1824         return 1;
1825 }
1826
1827 #define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
1828
1829 static void skd_send_internal_skspcl(struct skd_device *skdev,
1830                                      struct skd_special_context *skspcl,
1831                                      u8 opcode)
1832 {
1833         struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
1834         struct skd_scsi_request *scsi;
1835         unsigned char *buf = skspcl->data_buf;
1836         int i;
1837
1838         if (skspcl->req.state != SKD_REQ_STATE_IDLE)
1839                 /*
1840                  * A refresh is already in progress.
1841                  * Just wait for it to finish.
1842                  */
1843                 return;
1844
1845         SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
1846         skspcl->req.state = SKD_REQ_STATE_BUSY;
1847         skspcl->req.id += SKD_ID_INCR;
1848
1849         scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
1850         scsi->hdr.tag = skspcl->req.id;
1851
1852         memset(scsi->cdb, 0, sizeof(scsi->cdb));
1853
1854         switch (opcode) {
1855         case TEST_UNIT_READY:
1856                 scsi->cdb[0] = TEST_UNIT_READY;
1857                 sgd->byte_count = 0;
1858                 scsi->hdr.sg_list_len_bytes = 0;
1859                 break;
1860
1861         case READ_CAPACITY:
1862                 scsi->cdb[0] = READ_CAPACITY;
1863                 sgd->byte_count = SKD_N_READ_CAP_BYTES;
1864                 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1865                 break;
1866
1867         case INQUIRY:
1868                 scsi->cdb[0] = INQUIRY;
1869                 scsi->cdb[1] = 0x01;    /* evpd */
1870                 scsi->cdb[2] = 0x80;    /* serial number page */
1871                 scsi->cdb[4] = 0x10;
1872                 sgd->byte_count = 16;
1873                 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1874                 break;
1875
1876         case SYNCHRONIZE_CACHE:
1877                 scsi->cdb[0] = SYNCHRONIZE_CACHE;
1878                 sgd->byte_count = 0;
1879                 scsi->hdr.sg_list_len_bytes = 0;
1880                 break;
1881
1882         case WRITE_BUFFER:
1883                 scsi->cdb[0] = WRITE_BUFFER;
1884                 scsi->cdb[1] = 0x02;
1885                 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1886                 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1887                 sgd->byte_count = WR_BUF_SIZE;
1888                 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1889                 /* fill incrementing byte pattern */
1890                 for (i = 0; i < sgd->byte_count; i++)
1891                         buf[i] = i & 0xFF;
1892                 break;
1893
1894         case READ_BUFFER:
1895                 scsi->cdb[0] = READ_BUFFER;
1896                 scsi->cdb[1] = 0x02;
1897                 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
1898                 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
1899                 sgd->byte_count = WR_BUF_SIZE;
1900                 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
1901                 memset(skspcl->data_buf, 0, sgd->byte_count);
1902                 break;
1903
1904         default:
1905                 SKD_ASSERT("Don't know what to send");
1906                 return;
1907
1908         }
1909         skd_send_special_fitmsg(skdev, skspcl);
1910 }
1911
1912 static void skd_refresh_device_data(struct skd_device *skdev)
1913 {
1914         struct skd_special_context *skspcl = &skdev->internal_skspcl;
1915
1916         skd_send_internal_skspcl(skdev, skspcl, TEST_UNIT_READY);
1917 }
1918
1919 static int skd_chk_read_buf(struct skd_device *skdev,
1920                             struct skd_special_context *skspcl)
1921 {
1922         unsigned char *buf = skspcl->data_buf;
1923         int i;
1924
1925         /* check for incrementing byte pattern */
1926         for (i = 0; i < WR_BUF_SIZE; i++)
1927                 if (buf[i] != (i & 0xFF))
1928                         return 1;
1929
1930         return 0;
1931 }
1932
1933 static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
1934                                  u8 code, u8 qual, u8 fruc)
1935 {
1936         /* If the check condition is of special interest, log a message */
1937         if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
1938             && (code == 0x04) && (qual == 0x06)) {
1939                 pr_err("(%s): *** LOST_WRITE_DATA ERROR *** key/asc/"
1940                        "ascq/fruc %02x/%02x/%02x/%02x\n",
1941                        skd_name(skdev), key, code, qual, fruc);
1942         }
1943 }
1944
1945 static void skd_complete_internal(struct skd_device *skdev,
1946                                   volatile struct fit_completion_entry_v1
1947                                   *skcomp,
1948                                   volatile struct fit_comp_error_info *skerr,
1949                                   struct skd_special_context *skspcl)
1950 {
1951         u8 *buf = skspcl->data_buf;
1952         u8 status;
1953         int i;
1954         struct skd_scsi_request *scsi =
1955                 (struct skd_scsi_request *)&skspcl->msg_buf[64];
1956
1957         SKD_ASSERT(skspcl == &skdev->internal_skspcl);
1958
1959         pr_debug("%s:%s:%d complete internal %x\n",
1960                  skdev->name, __func__, __LINE__, scsi->cdb[0]);
1961
1962         skspcl->req.completion = *skcomp;
1963         skspcl->req.state = SKD_REQ_STATE_IDLE;
1964         skspcl->req.id += SKD_ID_INCR;
1965
1966         status = skspcl->req.completion.status;
1967
1968         skd_log_check_status(skdev, status, skerr->key, skerr->code,
1969                              skerr->qual, skerr->fruc);
1970
1971         switch (scsi->cdb[0]) {
1972         case TEST_UNIT_READY:
1973                 if (status == SAM_STAT_GOOD)
1974                         skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
1975                 else if ((status == SAM_STAT_CHECK_CONDITION) &&
1976                          (skerr->key == MEDIUM_ERROR))
1977                         skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
1978                 else {
1979                         if (skdev->state == SKD_DRVR_STATE_STOPPING) {
1980                                 pr_debug("%s:%s:%d TUR failed, don't send anymore state 0x%x\n",
1981                                          skdev->name, __func__, __LINE__,
1982                                          skdev->state);
1983                                 return;
1984                         }
1985                         pr_debug("%s:%s:%d **** TUR failed, retry skerr\n",
1986                                  skdev->name, __func__, __LINE__);
1987                         skd_send_internal_skspcl(skdev, skspcl, 0x00);
1988                 }
1989                 break;
1990
1991         case WRITE_BUFFER:
1992                 if (status == SAM_STAT_GOOD)
1993                         skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
1994                 else {
1995                         if (skdev->state == SKD_DRVR_STATE_STOPPING) {
1996                                 pr_debug("%s:%s:%d write buffer failed, don't send anymore state 0x%x\n",
1997                                          skdev->name, __func__, __LINE__,
1998                                          skdev->state);
1999                                 return;
2000                         }
2001                         pr_debug("%s:%s:%d **** write buffer failed, retry skerr\n",
2002                                  skdev->name, __func__, __LINE__);
2003                         skd_send_internal_skspcl(skdev, skspcl, 0x00);
2004                 }
2005                 break;
2006
2007         case READ_BUFFER:
2008                 if (status == SAM_STAT_GOOD) {
2009                         if (skd_chk_read_buf(skdev, skspcl) == 0)
2010                                 skd_send_internal_skspcl(skdev, skspcl,
2011                                                          READ_CAPACITY);
2012                         else {
2013                                 pr_err(
2014                                        "(%s):*** W/R Buffer mismatch %d ***\n",
2015                                        skd_name(skdev), skdev->connect_retries);
2016                                 if (skdev->connect_retries <
2017                                     SKD_MAX_CONNECT_RETRIES) {
2018                                         skdev->connect_retries++;
2019                                         skd_soft_reset(skdev);
2020                                 } else {
2021                                         pr_err(
2022                                                "(%s): W/R Buffer Connect Error\n",
2023                                                skd_name(skdev));
2024                                         return;
2025                                 }
2026                         }
2027
2028                 } else {
2029                         if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2030                                 pr_debug("%s:%s:%d "
2031                                          "read buffer failed, don't send anymore state 0x%x\n",
2032                                          skdev->name, __func__, __LINE__,
2033                                          skdev->state);
2034                                 return;
2035                         }
2036                         pr_debug("%s:%s:%d "
2037                                  "**** read buffer failed, retry skerr\n",
2038                                  skdev->name, __func__, __LINE__);
2039                         skd_send_internal_skspcl(skdev, skspcl, 0x00);
2040                 }
2041                 break;
2042
2043         case READ_CAPACITY:
2044                 skdev->read_cap_is_valid = 0;
2045                 if (status == SAM_STAT_GOOD) {
2046                         skdev->read_cap_last_lba =
2047                                 (buf[0] << 24) | (buf[1] << 16) |
2048                                 (buf[2] << 8) | buf[3];
2049                         skdev->read_cap_blocksize =
2050                                 (buf[4] << 24) | (buf[5] << 16) |
2051                                 (buf[6] << 8) | buf[7];
2052
2053                         pr_debug("%s:%s:%d last lba %d, bs %d\n",
2054                                  skdev->name, __func__, __LINE__,
2055                                  skdev->read_cap_last_lba,
2056                                  skdev->read_cap_blocksize);
2057
2058                         set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2059
2060                         skdev->read_cap_is_valid = 1;
2061
2062                         skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2063                 } else if ((status == SAM_STAT_CHECK_CONDITION) &&
2064                            (skerr->key == MEDIUM_ERROR)) {
2065                         skdev->read_cap_last_lba = ~0;
2066                         set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2067                         pr_debug("%s:%s:%d "
2068                                  "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n",
2069                                  skdev->name, __func__, __LINE__);
2070                         skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2071                 } else {
2072                         pr_debug("%s:%s:%d **** READCAP failed, retry TUR\n",
2073                                  skdev->name, __func__, __LINE__);
2074                         skd_send_internal_skspcl(skdev, skspcl,
2075                                                  TEST_UNIT_READY);
2076                 }
2077                 break;
2078
2079         case INQUIRY:
2080                 skdev->inquiry_is_valid = 0;
2081                 if (status == SAM_STAT_GOOD) {
2082                         skdev->inquiry_is_valid = 1;
2083
2084                         for (i = 0; i < 12; i++)
2085                                 skdev->inq_serial_num[i] = buf[i + 4];
2086                         skdev->inq_serial_num[12] = 0;
2087                 }
2088
2089                 if (skd_unquiesce_dev(skdev) < 0)
2090                         pr_debug("%s:%s:%d **** failed, to ONLINE device\n",
2091                                  skdev->name, __func__, __LINE__);
2092                  /* connection is complete */
2093                 skdev->connect_retries = 0;
2094                 break;
2095
2096         case SYNCHRONIZE_CACHE:
2097                 if (status == SAM_STAT_GOOD)
2098                         skdev->sync_done = 1;
2099                 else
2100                         skdev->sync_done = -1;
2101                 wake_up_interruptible(&skdev->waitq);
2102                 break;
2103
2104         default:
2105                 SKD_ASSERT("we didn't send this");
2106         }
2107 }
2108
2109 /*
2110  *****************************************************************************
2111  * FIT MESSAGES
2112  *****************************************************************************
2113  */
2114
2115 static void skd_send_fitmsg(struct skd_device *skdev,
2116                             struct skd_fitmsg_context *skmsg)
2117 {
2118         u64 qcmd;
2119         struct fit_msg_hdr *fmh;
2120
2121         pr_debug("%s:%s:%d dma address 0x%llx, busy=%d\n",
2122                  skdev->name, __func__, __LINE__,
2123                  skmsg->mb_dma_address, skdev->in_flight);
2124         pr_debug("%s:%s:%d msg_buf 0x%p, offset %x\n",
2125                  skdev->name, __func__, __LINE__,
2126                  skmsg->msg_buf, skmsg->offset);
2127
2128         qcmd = skmsg->mb_dma_address;
2129         qcmd |= FIT_QCMD_QID_NORMAL;
2130
2131         fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
2132         skmsg->outstanding = fmh->num_protocol_cmds_coalesced;
2133
2134         if (unlikely(skdev->dbg_level > 1)) {
2135                 u8 *bp = (u8 *)skmsg->msg_buf;
2136                 int i;
2137                 for (i = 0; i < skmsg->length; i += 8) {
2138                         pr_debug("%s:%s:%d msg[%2d] %8ph\n",
2139                                  skdev->name, __func__, __LINE__, i, &bp[i]);
2140                         if (i == 0)
2141                                 i = 64 - 8;
2142                 }
2143         }
2144
2145         if (skmsg->length > 256)
2146                 qcmd |= FIT_QCMD_MSGSIZE_512;
2147         else if (skmsg->length > 128)
2148                 qcmd |= FIT_QCMD_MSGSIZE_256;
2149         else if (skmsg->length > 64)
2150                 qcmd |= FIT_QCMD_MSGSIZE_128;
2151         else
2152                 /*
2153                  * This makes no sense because the FIT msg header is
2154                  * 64 bytes. If the msg is only 64 bytes long it has
2155                  * no payload.
2156                  */
2157                 qcmd |= FIT_QCMD_MSGSIZE_64;
2158
2159         SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2160 }
2161
2162 static void skd_send_special_fitmsg(struct skd_device *skdev,
2163                                     struct skd_special_context *skspcl)
2164 {
2165         u64 qcmd;
2166
2167         if (unlikely(skdev->dbg_level > 1)) {
2168                 u8 *bp = (u8 *)skspcl->msg_buf;
2169                 int i;
2170
2171                 for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
2172                         pr_debug("%s:%s:%d  spcl[%2d] %8ph\n",
2173                                  skdev->name, __func__, __LINE__, i, &bp[i]);
2174                         if (i == 0)
2175                                 i = 64 - 8;
2176                 }
2177
2178                 pr_debug("%s:%s:%d skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
2179                          skdev->name, __func__, __LINE__,
2180                          skspcl, skspcl->req.id, skspcl->req.sksg_list,
2181                          skspcl->req.sksg_dma_address);
2182                 for (i = 0; i < skspcl->req.n_sg; i++) {
2183                         struct fit_sg_descriptor *sgd =
2184                                 &skspcl->req.sksg_list[i];
2185
2186                         pr_debug("%s:%s:%d   sg[%d] count=%u ctrl=0x%x "
2187                                  "addr=0x%llx next=0x%llx\n",
2188                                  skdev->name, __func__, __LINE__,
2189                                  i, sgd->byte_count, sgd->control,
2190                                  sgd->host_side_addr, sgd->next_desc_ptr);
2191                 }
2192         }
2193
2194         /*
2195          * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
2196          * and one 64-byte SSDI command.
2197          */
2198         qcmd = skspcl->mb_dma_address;
2199         qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
2200
2201         SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2202 }
2203
2204 /*
2205  *****************************************************************************
2206  * COMPLETION QUEUE
2207  *****************************************************************************
2208  */
2209
2210 static void skd_complete_other(struct skd_device *skdev,
2211                                volatile struct fit_completion_entry_v1 *skcomp,
2212                                volatile struct fit_comp_error_info *skerr);
2213
2214 struct sns_info {
2215         u8 type;
2216         u8 stat;
2217         u8 key;
2218         u8 asc;
2219         u8 ascq;
2220         u8 mask;
2221         enum skd_check_status_action action;
2222 };
2223
2224 static struct sns_info skd_chkstat_table[] = {
2225         /* Good */
2226         { 0x70, 0x02, RECOVERED_ERROR, 0,    0,    0x1c,
2227           SKD_CHECK_STATUS_REPORT_GOOD },
2228
2229         /* Smart alerts */
2230         { 0x70, 0x02, NO_SENSE,        0x0B, 0x00, 0x1E,        /* warnings */
2231           SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2232         { 0x70, 0x02, NO_SENSE,        0x5D, 0x00, 0x1E,        /* thresholds */
2233           SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2234         { 0x70, 0x02, RECOVERED_ERROR, 0x0B, 0x01, 0x1F,        /* temperature over trigger */
2235           SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2236
2237         /* Retry (with limits) */
2238         { 0x70, 0x02, 0x0B,            0,    0,    0x1C,        /* This one is for DMA ERROR */
2239           SKD_CHECK_STATUS_REQUEUE_REQUEST },
2240         { 0x70, 0x02, 0x06,            0x0B, 0x00, 0x1E,        /* warnings */
2241           SKD_CHECK_STATUS_REQUEUE_REQUEST },
2242         { 0x70, 0x02, 0x06,            0x5D, 0x00, 0x1E,        /* thresholds */
2243           SKD_CHECK_STATUS_REQUEUE_REQUEST },
2244         { 0x70, 0x02, 0x06,            0x80, 0x30, 0x1F,        /* backup power */
2245           SKD_CHECK_STATUS_REQUEUE_REQUEST },
2246
2247         /* Busy (or about to be) */
2248         { 0x70, 0x02, 0x06,            0x3f, 0x01, 0x1F, /* fw changed */
2249           SKD_CHECK_STATUS_BUSY_IMMINENT },
2250 };
2251
2252 /*
2253  * Look up status and sense data to decide how to handle the error
2254  * from the device.
2255  * mask says which fields must match e.g., mask=0x18 means check
2256  * type and stat, ignore key, asc, ascq.
2257  */
2258
2259 static enum skd_check_status_action
2260 skd_check_status(struct skd_device *skdev,
2261                  u8 cmp_status, volatile struct fit_comp_error_info *skerr)
2262 {
2263         int i, n;
2264
2265         pr_err("(%s): key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
2266                skd_name(skdev), skerr->key, skerr->code, skerr->qual,
2267                skerr->fruc);
2268
2269         pr_debug("%s:%s:%d stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
2270                  skdev->name, __func__, __LINE__, skerr->type, cmp_status,
2271                  skerr->key, skerr->code, skerr->qual, skerr->fruc);
2272
2273         /* Does the info match an entry in the good category? */
2274         n = sizeof(skd_chkstat_table) / sizeof(skd_chkstat_table[0]);
2275         for (i = 0; i < n; i++) {
2276                 struct sns_info *sns = &skd_chkstat_table[i];
2277
2278                 if (sns->mask & 0x10)
2279                         if (skerr->type != sns->type)
2280                                 continue;
2281
2282                 if (sns->mask & 0x08)
2283                         if (cmp_status != sns->stat)
2284                                 continue;
2285
2286                 if (sns->mask & 0x04)
2287                         if (skerr->key != sns->key)
2288                                 continue;
2289
2290                 if (sns->mask & 0x02)
2291                         if (skerr->code != sns->asc)
2292                                 continue;
2293
2294                 if (sns->mask & 0x01)
2295                         if (skerr->qual != sns->ascq)
2296                                 continue;
2297
2298                 if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
2299                         pr_err("(%s): SMART Alert: sense key/asc/ascq "
2300                                "%02x/%02x/%02x\n",
2301                                skd_name(skdev), skerr->key,
2302                                skerr->code, skerr->qual);
2303                 }
2304                 return sns->action;
2305         }
2306
2307         /* No other match, so nonzero status means error,
2308          * zero status means good
2309          */
2310         if (cmp_status) {
2311                 pr_debug("%s:%s:%d status check: error\n",
2312                          skdev->name, __func__, __LINE__);
2313                 return SKD_CHECK_STATUS_REPORT_ERROR;
2314         }
2315
2316         pr_debug("%s:%s:%d status check good default\n",
2317                  skdev->name, __func__, __LINE__);
2318         return SKD_CHECK_STATUS_REPORT_GOOD;
2319 }
2320
2321 static void skd_resolve_req_exception(struct skd_device *skdev,
2322                                       struct skd_request_context *skreq)
2323 {
2324         u8 cmp_status = skreq->completion.status;
2325
2326         switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
2327         case SKD_CHECK_STATUS_REPORT_GOOD:
2328         case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
2329                 skd_end_request(skdev, skreq, 0);
2330                 break;
2331
2332         case SKD_CHECK_STATUS_BUSY_IMMINENT:
2333                 skd_log_skreq(skdev, skreq, "retry(busy)");
2334                 blk_requeue_request(skdev->queue, skreq->req);
2335                 pr_info("(%s) drive BUSY imminent\n", skd_name(skdev));
2336                 skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
2337                 skdev->timer_countdown = SKD_TIMER_MINUTES(20);
2338                 skd_quiesce_dev(skdev);
2339                 break;
2340
2341         case SKD_CHECK_STATUS_REQUEUE_REQUEST:
2342                 if ((unsigned long) ++skreq->req->special < SKD_MAX_RETRIES) {
2343                         skd_log_skreq(skdev, skreq, "retry");
2344                         blk_requeue_request(skdev->queue, skreq->req);
2345                         break;
2346                 }
2347         /* fall through to report error */
2348
2349         case SKD_CHECK_STATUS_REPORT_ERROR:
2350         default:
2351                 skd_end_request(skdev, skreq, -EIO);
2352                 break;
2353         }
2354 }
2355
2356 /* assume spinlock is already held */
2357 static void skd_release_skreq(struct skd_device *skdev,
2358                               struct skd_request_context *skreq)
2359 {
2360         u32 msg_slot;
2361         struct skd_fitmsg_context *skmsg;
2362
2363         u32 timo_slot;
2364
2365         /*
2366          * Reclaim the FIT msg buffer if this is
2367          * the first of the requests it carried to
2368          * be completed. The FIT msg buffer used to
2369          * send this request cannot be reused until
2370          * we are sure the s1120 card has copied
2371          * it to its memory. The FIT msg might have
2372          * contained several requests. As soon as
2373          * any of them are completed we know that
2374          * the entire FIT msg was transferred.
2375          * Only the first completed request will
2376          * match the FIT msg buffer id. The FIT
2377          * msg buffer id is immediately updated.
2378          * When subsequent requests complete the FIT
2379          * msg buffer id won't match, so we know
2380          * quite cheaply that it is already done.
2381          */
2382         msg_slot = skreq->fitmsg_id & SKD_ID_SLOT_MASK;
2383         SKD_ASSERT(msg_slot < skdev->num_fitmsg_context);
2384
2385         skmsg = &skdev->skmsg_table[msg_slot];
2386         if (skmsg->id == skreq->fitmsg_id) {
2387                 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_BUSY);
2388                 SKD_ASSERT(skmsg->outstanding > 0);
2389                 skmsg->outstanding--;
2390                 if (skmsg->outstanding == 0) {
2391                         skmsg->state = SKD_MSG_STATE_IDLE;
2392                         skmsg->id += SKD_ID_INCR;
2393                         skmsg->next = skdev->skmsg_free_list;
2394                         skdev->skmsg_free_list = skmsg;
2395                 }
2396         }
2397
2398         /*
2399          * Decrease the number of active requests.
2400          * Also decrements the count in the timeout slot.
2401          */
2402         SKD_ASSERT(skdev->in_flight > 0);
2403         skdev->in_flight -= 1;
2404
2405         timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
2406         SKD_ASSERT(skdev->timeout_slot[timo_slot] > 0);
2407         skdev->timeout_slot[timo_slot] -= 1;
2408
2409         /*
2410          * Reset backpointer
2411          */
2412         skreq->req = NULL;
2413
2414         /*
2415          * Reclaim the skd_request_context
2416          */
2417         skreq->state = SKD_REQ_STATE_IDLE;
2418         skreq->id += SKD_ID_INCR;
2419         skreq->next = skdev->skreq_free_list;
2420         skdev->skreq_free_list = skreq;
2421 }
2422
2423 #define DRIVER_INQ_EVPD_PAGE_CODE   0xDA
2424
2425 static void skd_do_inq_page_00(struct skd_device *skdev,
2426                                volatile struct fit_completion_entry_v1 *skcomp,
2427                                volatile struct fit_comp_error_info *skerr,
2428                                uint8_t *cdb, uint8_t *buf)
2429 {
2430         uint16_t insert_pt, max_bytes, drive_pages, drive_bytes, new_size;
2431
2432         /* Caller requested "supported pages".  The driver needs to insert
2433          * its page.
2434          */
2435         pr_debug("%s:%s:%d skd_do_driver_inquiry: modify supported pages.\n",
2436                  skdev->name, __func__, __LINE__);
2437
2438         /* If the device rejected the request because the CDB was
2439          * improperly formed, then just leave.
2440          */
2441         if (skcomp->status == SAM_STAT_CHECK_CONDITION &&
2442             skerr->key == ILLEGAL_REQUEST && skerr->code == 0x24)
2443                 return;
2444
2445         /* Get the amount of space the caller allocated */
2446         max_bytes = (cdb[3] << 8) | cdb[4];
2447
2448         /* Get the number of pages actually returned by the device */
2449         drive_pages = (buf[2] << 8) | buf[3];
2450         drive_bytes = drive_pages + 4;
2451         new_size = drive_pages + 1;
2452
2453         /* Supported pages must be in numerical order, so find where
2454          * the driver page needs to be inserted into the list of
2455          * pages returned by the device.
2456          */
2457         for (insert_pt = 4; insert_pt < drive_bytes; insert_pt++) {
2458                 if (buf[insert_pt] == DRIVER_INQ_EVPD_PAGE_CODE)
2459                         return; /* Device using this page code. abort */
2460                 else if (buf[insert_pt] > DRIVER_INQ_EVPD_PAGE_CODE)
2461                         break;
2462         }
2463
2464         if (insert_pt < max_bytes) {
2465                 uint16_t u;
2466
2467                 /* Shift everything up one byte to make room. */
2468                 for (u = new_size + 3; u > insert_pt; u--)
2469                         buf[u] = buf[u - 1];
2470                 buf[insert_pt] = DRIVER_INQ_EVPD_PAGE_CODE;
2471
2472                 /* SCSI byte order increment of num_returned_bytes by 1 */
2473                 skcomp->num_returned_bytes =
2474                         be32_to_cpu(skcomp->num_returned_bytes) + 1;
2475                 skcomp->num_returned_bytes =
2476                         be32_to_cpu(skcomp->num_returned_bytes);
2477         }
2478
2479         /* update page length field to reflect the driver's page too */
2480         buf[2] = (uint8_t)((new_size >> 8) & 0xFF);
2481         buf[3] = (uint8_t)((new_size >> 0) & 0xFF);
2482 }
2483
2484 static void skd_get_link_info(struct pci_dev *pdev, u8 *speed, u8 *width)
2485 {
2486         int pcie_reg;
2487         u16 pci_bus_speed;
2488         u8 pci_lanes;
2489
2490         pcie_reg = pci_find_capability(pdev, PCI_CAP_ID_EXP);
2491         if (pcie_reg) {
2492                 u16 linksta;
2493                 pci_read_config_word(pdev, pcie_reg + PCI_EXP_LNKSTA, &linksta);
2494
2495                 pci_bus_speed = linksta & 0xF;
2496                 pci_lanes = (linksta & 0x3F0) >> 4;
2497         } else {
2498                 *speed = STEC_LINK_UNKNOWN;
2499                 *width = 0xFF;
2500                 return;
2501         }
2502
2503         switch (pci_bus_speed) {
2504         case 1:
2505                 *speed = STEC_LINK_2_5GTS;
2506                 break;
2507         case 2:
2508                 *speed = STEC_LINK_5GTS;
2509                 break;
2510         case 3:
2511                 *speed = STEC_LINK_8GTS;
2512                 break;
2513         default:
2514                 *speed = STEC_LINK_UNKNOWN;
2515                 break;
2516         }
2517
2518         if (pci_lanes <= 0x20)
2519                 *width = pci_lanes;
2520         else
2521                 *width = 0xFF;
2522 }
2523
2524 static void skd_do_inq_page_da(struct skd_device *skdev,
2525                                volatile struct fit_completion_entry_v1 *skcomp,
2526                                volatile struct fit_comp_error_info *skerr,
2527                                uint8_t *cdb, uint8_t *buf)
2528 {
2529         struct pci_dev *pdev = skdev->pdev;
2530         unsigned max_bytes;
2531         struct driver_inquiry_data inq;
2532         u16 val;
2533
2534         pr_debug("%s:%s:%d skd_do_driver_inquiry: return driver page\n",
2535                  skdev->name, __func__, __LINE__);
2536
2537         memset(&inq, 0, sizeof(inq));
2538
2539         inq.page_code = DRIVER_INQ_EVPD_PAGE_CODE;
2540
2541         skd_get_link_info(pdev, &inq.pcie_link_speed, &inq.pcie_link_lanes);
2542         inq.pcie_bus_number = cpu_to_be16(pdev->bus->number);
2543         inq.pcie_device_number = PCI_SLOT(pdev->devfn);
2544         inq.pcie_function_number = PCI_FUNC(pdev->devfn);
2545
2546         pci_read_config_word(pdev, PCI_VENDOR_ID, &val);
2547         inq.pcie_vendor_id = cpu_to_be16(val);
2548
2549         pci_read_config_word(pdev, PCI_DEVICE_ID, &val);
2550         inq.pcie_device_id = cpu_to_be16(val);
2551
2552         pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &val);
2553         inq.pcie_subsystem_vendor_id = cpu_to_be16(val);
2554
2555         pci_read_config_word(pdev, PCI_SUBSYSTEM_ID, &val);
2556         inq.pcie_subsystem_device_id = cpu_to_be16(val);
2557
2558         /* Driver version, fixed lenth, padded with spaces on the right */
2559         inq.driver_version_length = sizeof(inq.driver_version);
2560         memset(&inq.driver_version, ' ', sizeof(inq.driver_version));
2561         memcpy(inq.driver_version, DRV_VER_COMPL,
2562                min(sizeof(inq.driver_version), strlen(DRV_VER_COMPL)));
2563
2564         inq.page_length = cpu_to_be16((sizeof(inq) - 4));
2565
2566         /* Clear the error set by the device */
2567         skcomp->status = SAM_STAT_GOOD;
2568         memset((void *)skerr, 0, sizeof(*skerr));
2569
2570         /* copy response into output buffer */
2571         max_bytes = (cdb[3] << 8) | cdb[4];
2572         memcpy(buf, &inq, min_t(unsigned, max_bytes, sizeof(inq)));
2573
2574         skcomp->num_returned_bytes =
2575                 be32_to_cpu(min_t(uint16_t, max_bytes, sizeof(inq)));
2576 }
2577
2578 static void skd_do_driver_inq(struct skd_device *skdev,
2579                               volatile struct fit_completion_entry_v1 *skcomp,
2580                               volatile struct fit_comp_error_info *skerr,
2581                               uint8_t *cdb, uint8_t *buf)
2582 {
2583         if (!buf)
2584                 return;
2585         else if (cdb[0] != INQUIRY)
2586                 return;         /* Not an INQUIRY */
2587         else if ((cdb[1] & 1) == 0)
2588                 return;         /* EVPD not set */
2589         else if (cdb[2] == 0)
2590                 /* Need to add driver's page to supported pages list */
2591                 skd_do_inq_page_00(skdev, skcomp, skerr, cdb, buf);
2592         else if (cdb[2] == DRIVER_INQ_EVPD_PAGE_CODE)
2593                 /* Caller requested driver's page */
2594                 skd_do_inq_page_da(skdev, skcomp, skerr, cdb, buf);
2595 }
2596
2597 static unsigned char *skd_sg_1st_page_ptr(struct scatterlist *sg)
2598 {
2599         if (!sg)
2600                 return NULL;
2601         if (!sg_page(sg))
2602                 return NULL;
2603         return sg_virt(sg);
2604 }
2605
2606 static void skd_process_scsi_inq(struct skd_device *skdev,
2607                                  volatile struct fit_completion_entry_v1
2608                                  *skcomp,
2609                                  volatile struct fit_comp_error_info *skerr,
2610                                  struct skd_special_context *skspcl)
2611 {
2612         uint8_t *buf;
2613         struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
2614         struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
2615
2616         dma_sync_sg_for_cpu(skdev->class_dev, skspcl->req.sg, skspcl->req.n_sg,
2617                             skspcl->req.sg_data_dir);
2618         buf = skd_sg_1st_page_ptr(skspcl->req.sg);
2619
2620         if (buf)
2621                 skd_do_driver_inq(skdev, skcomp, skerr, scsi_req->cdb, buf);
2622 }
2623
2624
2625 static int skd_isr_completion_posted(struct skd_device *skdev,
2626                                         int limit, int *enqueued)
2627 {
2628         volatile struct fit_completion_entry_v1 *skcmp = NULL;
2629         volatile struct fit_comp_error_info *skerr;
2630         u16 req_id;
2631         u32 req_slot;
2632         struct skd_request_context *skreq;
2633         u16 cmp_cntxt = 0;
2634         u8 cmp_status = 0;
2635         u8 cmp_cycle = 0;
2636         u32 cmp_bytes = 0;
2637         int rc = 0;
2638         int processed = 0;
2639
2640         for (;; ) {
2641                 SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
2642
2643                 skcmp = &skdev->skcomp_table[skdev->skcomp_ix];
2644                 cmp_cycle = skcmp->cycle;
2645                 cmp_cntxt = skcmp->tag;
2646                 cmp_status = skcmp->status;
2647                 cmp_bytes = be32_to_cpu(skcmp->num_returned_bytes);
2648
2649                 skerr = &skdev->skerr_table[skdev->skcomp_ix];
2650
2651                 pr_debug("%s:%s:%d "
2652                          "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d "
2653                          "busy=%d rbytes=0x%x proto=%d\n",
2654                          skdev->name, __func__, __LINE__, skdev->skcomp_cycle,
2655                          skdev->skcomp_ix, cmp_cycle, cmp_cntxt, cmp_status,
2656                          skdev->in_flight, cmp_bytes, skdev->proto_ver);
2657
2658                 if (cmp_cycle != skdev->skcomp_cycle) {
2659                         pr_debug("%s:%s:%d end of completions\n",
2660                                  skdev->name, __func__, __LINE__);
2661                         break;
2662                 }
2663                 /*
2664                  * Update the completion queue head index and possibly
2665                  * the completion cycle count. 8-bit wrap-around.
2666                  */
2667                 skdev->skcomp_ix++;
2668                 if (skdev->skcomp_ix >= SKD_N_COMPLETION_ENTRY) {
2669                         skdev->skcomp_ix = 0;
2670                         skdev->skcomp_cycle++;
2671                 }
2672
2673                 /*
2674                  * The command context is a unique 32-bit ID. The low order
2675                  * bits help locate the request. The request is usually a
2676                  * r/w request (see skd_start() above) or a special request.
2677                  */
2678                 req_id = cmp_cntxt;
2679                 req_slot = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
2680
2681                 /* Is this other than a r/w request? */
2682                 if (req_slot >= skdev->num_req_context) {
2683                         /*
2684                          * This is not a completion for a r/w request.
2685                          */
2686                         skd_complete_other(skdev, skcmp, skerr);
2687                         continue;
2688                 }
2689
2690                 skreq = &skdev->skreq_table[req_slot];
2691
2692                 /*
2693                  * Make sure the request ID for the slot matches.
2694                  */
2695                 if (skreq->id != req_id) {
2696                         pr_debug("%s:%s:%d mismatch comp_id=0x%x req_id=0x%x\n",
2697                                  skdev->name, __func__, __LINE__,
2698                                  req_id, skreq->id);
2699                         {
2700                                 u16 new_id = cmp_cntxt;
2701                                 pr_err("(%s): Completion mismatch "
2702                                        "comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
2703                                        skd_name(skdev), req_id,
2704                                        skreq->id, new_id);
2705
2706                                 continue;
2707                         }
2708                 }
2709
2710                 SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
2711
2712                 if (skreq->state == SKD_REQ_STATE_ABORTED) {
2713                         pr_debug("%s:%s:%d reclaim req %p id=%04x\n",
2714                                  skdev->name, __func__, __LINE__,
2715                                  skreq, skreq->id);
2716                         /* a previously timed out command can
2717                          * now be cleaned up */
2718                         skd_release_skreq(skdev, skreq);
2719                         continue;
2720                 }
2721
2722                 skreq->completion = *skcmp;
2723                 if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
2724                         skreq->err_info = *skerr;
2725                         skd_log_check_status(skdev, cmp_status, skerr->key,
2726                                              skerr->code, skerr->qual,
2727                                              skerr->fruc);
2728                 }
2729                 /* Release DMA resources for the request. */
2730                 if (skreq->n_sg > 0)
2731                         skd_postop_sg_list(skdev, skreq);
2732
2733                 if (!skreq->req) {
2734                         pr_debug("%s:%s:%d NULL backptr skdreq %p, "
2735                                  "req=0x%x req_id=0x%x\n",
2736                                  skdev->name, __func__, __LINE__,
2737                                  skreq, skreq->id, req_id);
2738                 } else {
2739                         /*
2740                          * Capture the outcome and post it back to the
2741                          * native request.
2742                          */
2743                         if (likely(cmp_status == SAM_STAT_GOOD))
2744                                 skd_end_request(skdev, skreq, 0);
2745                         else
2746                                 skd_resolve_req_exception(skdev, skreq);
2747                 }
2748
2749                 /*
2750                  * Release the skreq, its FIT msg (if one), timeout slot,
2751                  * and queue depth.
2752                  */
2753                 skd_release_skreq(skdev, skreq);
2754
2755                 /* skd_isr_comp_limit equal zero means no limit */
2756                 if (limit) {
2757                         if (++processed >= limit) {
2758                                 rc = 1;
2759                                 break;
2760                         }
2761                 }
2762         }
2763
2764         if ((skdev->state == SKD_DRVR_STATE_PAUSING)
2765                 && (skdev->in_flight) == 0) {
2766                 skdev->state = SKD_DRVR_STATE_PAUSED;
2767                 wake_up_interruptible(&skdev->waitq);
2768         }
2769
2770         return rc;
2771 }
2772
2773 static void skd_complete_other(struct skd_device *skdev,
2774                                volatile struct fit_completion_entry_v1 *skcomp,
2775                                volatile struct fit_comp_error_info *skerr)
2776 {
2777         u32 req_id = 0;
2778         u32 req_table;
2779         u32 req_slot;
2780         struct skd_special_context *skspcl;
2781
2782         req_id = skcomp->tag;
2783         req_table = req_id & SKD_ID_TABLE_MASK;
2784         req_slot = req_id & SKD_ID_SLOT_MASK;
2785
2786         pr_debug("%s:%s:%d table=0x%x id=0x%x slot=%d\n",
2787                  skdev->name, __func__, __LINE__,
2788                  req_table, req_id, req_slot);
2789
2790         /*
2791          * Based on the request id, determine how to dispatch this completion.
2792          * This swich/case is finding the good cases and forwarding the
2793          * completion entry. Errors are reported below the switch.
2794          */
2795         switch (req_table) {
2796         case SKD_ID_RW_REQUEST:
2797                 /*
2798                  * The caller, skd_completion_posted_isr() above,
2799                  * handles r/w requests. The only way we get here
2800                  * is if the req_slot is out of bounds.
2801                  */
2802                 break;
2803
2804         case SKD_ID_SPECIAL_REQUEST:
2805                 /*
2806                  * Make sure the req_slot is in bounds and that the id
2807                  * matches.
2808                  */
2809                 if (req_slot < skdev->n_special) {
2810                         skspcl = &skdev->skspcl_table[req_slot];
2811                         if (skspcl->req.id == req_id &&
2812                             skspcl->req.state == SKD_REQ_STATE_BUSY) {
2813                                 skd_complete_special(skdev,
2814                                                      skcomp, skerr, skspcl);
2815                                 return;
2816                         }
2817                 }
2818                 break;
2819
2820         case SKD_ID_INTERNAL:
2821                 if (req_slot == 0) {
2822                         skspcl = &skdev->internal_skspcl;
2823                         if (skspcl->req.id == req_id &&
2824                             skspcl->req.state == SKD_REQ_STATE_BUSY) {
2825                                 skd_complete_internal(skdev,
2826                                                       skcomp, skerr, skspcl);
2827                                 return;
2828                         }
2829                 }
2830                 break;
2831
2832         case SKD_ID_FIT_MSG:
2833                 /*
2834                  * These id's should never appear in a completion record.
2835                  */
2836                 break;
2837
2838         default:
2839                 /*
2840                  * These id's should never appear anywhere;
2841                  */
2842                 break;
2843         }
2844
2845         /*
2846          * If we get here it is a bad or stale id.
2847          */
2848 }
2849
2850 static void skd_complete_special(struct skd_device *skdev,
2851                                  volatile struct fit_completion_entry_v1
2852                                  *skcomp,
2853                                  volatile struct fit_comp_error_info *skerr,
2854                                  struct skd_special_context *skspcl)
2855 {
2856         pr_debug("%s:%s:%d  completing special request %p\n",
2857                  skdev->name, __func__, __LINE__, skspcl);
2858         if (skspcl->orphaned) {
2859                 /* Discard orphaned request */
2860                 /* ?: Can this release directly or does it need
2861                  * to use a worker? */
2862                 pr_debug("%s:%s:%d release orphaned %p\n",
2863                          skdev->name, __func__, __LINE__, skspcl);
2864                 skd_release_special(skdev, skspcl);
2865                 return;
2866         }
2867
2868         skd_process_scsi_inq(skdev, skcomp, skerr, skspcl);
2869
2870         skspcl->req.state = SKD_REQ_STATE_COMPLETED;
2871         skspcl->req.completion = *skcomp;
2872         skspcl->req.err_info = *skerr;
2873
2874         skd_log_check_status(skdev, skspcl->req.completion.status, skerr->key,
2875                              skerr->code, skerr->qual, skerr->fruc);
2876
2877         wake_up_interruptible(&skdev->waitq);
2878 }
2879
2880 /* assume spinlock is already held */
2881 static void skd_release_special(struct skd_device *skdev,
2882                                 struct skd_special_context *skspcl)
2883 {
2884         int i, was_depleted;
2885
2886         for (i = 0; i < skspcl->req.n_sg; i++) {
2887                 struct page *page = sg_page(&skspcl->req.sg[i]);
2888                 __free_page(page);
2889         }
2890
2891         was_depleted = (skdev->skspcl_free_list == NULL);
2892
2893         skspcl->req.state = SKD_REQ_STATE_IDLE;
2894         skspcl->req.id += SKD_ID_INCR;
2895         skspcl->req.next =
2896                 (struct skd_request_context *)skdev->skspcl_free_list;
2897         skdev->skspcl_free_list = (struct skd_special_context *)skspcl;
2898
2899         if (was_depleted) {
2900                 pr_debug("%s:%s:%d skspcl was depleted\n",
2901                          skdev->name, __func__, __LINE__);
2902                 /* Free list was depleted. Their might be waiters. */
2903                 wake_up_interruptible(&skdev->waitq);
2904         }
2905 }
2906
2907 static void skd_reset_skcomp(struct skd_device *skdev)
2908 {
2909         u32 nbytes;
2910         struct fit_completion_entry_v1 *skcomp;
2911
2912         nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
2913         nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
2914
2915         memset(skdev->skcomp_table, 0, nbytes);
2916
2917         skdev->skcomp_ix = 0;
2918         skdev->skcomp_cycle = 1;
2919 }
2920
2921 /*
2922  *****************************************************************************
2923  * INTERRUPTS
2924  *****************************************************************************
2925  */
2926 static void skd_completion_worker(struct work_struct *work)
2927 {
2928         struct skd_device *skdev =
2929                 container_of(work, struct skd_device, completion_worker);
2930         unsigned long flags;
2931         int flush_enqueued = 0;
2932
2933         spin_lock_irqsave(&skdev->lock, flags);
2934
2935         /*
2936          * pass in limit=0, which means no limit..
2937          * process everything in compq
2938          */
2939         skd_isr_completion_posted(skdev, 0, &flush_enqueued);
2940         skd_request_fn(skdev->queue);
2941
2942         spin_unlock_irqrestore(&skdev->lock, flags);
2943 }
2944
2945 static void skd_isr_msg_from_dev(struct skd_device *skdev);
2946
2947 static irqreturn_t
2948 skd_isr(int irq, void *ptr)
2949 {
2950         struct skd_device *skdev;
2951         u32 intstat;
2952         u32 ack;
2953         int rc = 0;
2954         int deferred = 0;
2955         int flush_enqueued = 0;
2956
2957         skdev = (struct skd_device *)ptr;
2958         spin_lock(&skdev->lock);
2959
2960         for (;; ) {
2961                 intstat = SKD_READL(skdev, FIT_INT_STATUS_HOST);
2962
2963                 ack = FIT_INT_DEF_MASK;
2964                 ack &= intstat;
2965
2966                 pr_debug("%s:%s:%d intstat=0x%x ack=0x%x\n",
2967                          skdev->name, __func__, __LINE__, intstat, ack);
2968
2969                 /* As long as there is an int pending on device, keep
2970                  * running loop.  When none, get out, but if we've never
2971                  * done any processing, call completion handler?
2972                  */
2973                 if (ack == 0) {
2974                         /* No interrupts on device, but run the completion
2975                          * processor anyway?
2976                          */
2977                         if (rc == 0)
2978                                 if (likely (skdev->state
2979                                         == SKD_DRVR_STATE_ONLINE))
2980                                         deferred = 1;
2981                         break;
2982                 }
2983
2984                 rc = IRQ_HANDLED;
2985
2986                 SKD_WRITEL(skdev, ack, FIT_INT_STATUS_HOST);
2987
2988                 if (likely((skdev->state != SKD_DRVR_STATE_LOAD) &&
2989                            (skdev->state != SKD_DRVR_STATE_STOPPING))) {
2990                         if (intstat & FIT_ISH_COMPLETION_POSTED) {
2991                                 /*
2992                                  * If we have already deferred completion
2993                                  * processing, don't bother running it again
2994                                  */
2995                                 if (deferred == 0)
2996                                         deferred =
2997                                                 skd_isr_completion_posted(skdev,
2998                                                 skd_isr_comp_limit, &flush_enqueued);
2999                         }
3000
3001                         if (intstat & FIT_ISH_FW_STATE_CHANGE) {
3002                                 skd_isr_fwstate(skdev);
3003                                 if (skdev->state == SKD_DRVR_STATE_FAULT ||
3004                                     skdev->state ==
3005                                     SKD_DRVR_STATE_DISAPPEARED) {
3006                                         spin_unlock(&skdev->lock);
3007                                         return rc;
3008                                 }
3009                         }
3010
3011                         if (intstat & FIT_ISH_MSG_FROM_DEV)
3012                                 skd_isr_msg_from_dev(skdev);
3013                 }
3014         }
3015
3016         if (unlikely(flush_enqueued))
3017                 skd_request_fn(skdev->queue);
3018
3019         if (deferred)
3020                 schedule_work(&skdev->completion_worker);
3021         else if (!flush_enqueued)
3022                 skd_request_fn(skdev->queue);
3023
3024         spin_unlock(&skdev->lock);
3025
3026         return rc;
3027 }
3028
3029 static void skd_drive_fault(struct skd_device *skdev)
3030 {
3031         skdev->state = SKD_DRVR_STATE_FAULT;
3032         pr_err("(%s): Drive FAULT\n", skd_name(skdev));
3033 }
3034
3035 static void skd_drive_disappeared(struct skd_device *skdev)
3036 {
3037         skdev->state = SKD_DRVR_STATE_DISAPPEARED;
3038         pr_err("(%s): Drive DISAPPEARED\n", skd_name(skdev));
3039 }
3040
3041 static void skd_isr_fwstate(struct skd_device *skdev)
3042 {
3043         u32 sense;
3044         u32 state;
3045         u32 mtd;
3046         int prev_driver_state = skdev->state;
3047
3048         sense = SKD_READL(skdev, FIT_STATUS);
3049         state = sense & FIT_SR_DRIVE_STATE_MASK;
3050
3051         pr_err("(%s): s1120 state %s(%d)=>%s(%d)\n",
3052                skd_name(skdev),
3053                skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
3054                skd_drive_state_to_str(state), state);
3055
3056         skdev->drive_state = state;
3057
3058         switch (skdev->drive_state) {
3059         case FIT_SR_DRIVE_INIT:
3060                 if (skdev->state == SKD_DRVR_STATE_PROTOCOL_MISMATCH) {
3061                         skd_disable_interrupts(skdev);
3062                         break;
3063                 }
3064                 if (skdev->state == SKD_DRVR_STATE_RESTARTING)
3065                         skd_recover_requests(skdev, 0);
3066                 if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
3067                         skdev->timer_countdown = SKD_STARTING_TIMO;
3068                         skdev->state = SKD_DRVR_STATE_STARTING;
3069                         skd_soft_reset(skdev);
3070                         break;
3071                 }
3072                 mtd = FIT_MXD_CONS(FIT_MTD_FITFW_INIT, 0, 0);
3073                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3074                 skdev->last_mtd = mtd;
3075                 break;
3076
3077         case FIT_SR_DRIVE_ONLINE:
3078                 skdev->cur_max_queue_depth = skd_max_queue_depth;
3079                 if (skdev->cur_max_queue_depth > skdev->dev_max_queue_depth)
3080                         skdev->cur_max_queue_depth = skdev->dev_max_queue_depth;
3081
3082                 skdev->queue_low_water_mark =
3083                         skdev->cur_max_queue_depth * 2 / 3 + 1;
3084                 if (skdev->queue_low_water_mark < 1)
3085                         skdev->queue_low_water_mark = 1;
3086                 pr_info(
3087                        "(%s): Queue depth limit=%d dev=%d lowat=%d\n",
3088                        skd_name(skdev),
3089                        skdev->cur_max_queue_depth,
3090                        skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
3091
3092                 skd_refresh_device_data(skdev);
3093                 break;
3094
3095         case FIT_SR_DRIVE_BUSY:
3096                 skdev->state = SKD_DRVR_STATE_BUSY;
3097                 skdev->timer_countdown = SKD_BUSY_TIMO;
3098                 skd_quiesce_dev(skdev);
3099                 break;
3100         case FIT_SR_DRIVE_BUSY_SANITIZE:
3101                 /* set timer for 3 seconds, we'll abort any unfinished
3102                  * commands after that expires
3103                  */
3104                 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3105                 skdev->timer_countdown = SKD_TIMER_SECONDS(3);
3106                 blk_start_queue(skdev->queue);
3107                 break;
3108         case FIT_SR_DRIVE_BUSY_ERASE:
3109                 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3110                 skdev->timer_countdown = SKD_BUSY_TIMO;
3111                 break;
3112         case FIT_SR_DRIVE_OFFLINE:
3113                 skdev->state = SKD_DRVR_STATE_IDLE;
3114                 break;
3115         case FIT_SR_DRIVE_SOFT_RESET:
3116                 switch (skdev->state) {
3117                 case SKD_DRVR_STATE_STARTING:
3118                 case SKD_DRVR_STATE_RESTARTING:
3119                         /* Expected by a caller of skd_soft_reset() */
3120                         break;
3121                 default:
3122                         skdev->state = SKD_DRVR_STATE_RESTARTING;
3123                         break;
3124                 }
3125                 break;
3126         case FIT_SR_DRIVE_FW_BOOTING:
3127                 pr_debug("%s:%s:%d ISR FIT_SR_DRIVE_FW_BOOTING %s\n",
3128                          skdev->name, __func__, __LINE__, skdev->name);
3129                 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3130                 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3131                 break;
3132
3133         case FIT_SR_DRIVE_DEGRADED:
3134         case FIT_SR_PCIE_LINK_DOWN:
3135         case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
3136                 break;
3137
3138         case FIT_SR_DRIVE_FAULT:
3139                 skd_drive_fault(skdev);
3140                 skd_recover_requests(skdev, 0);
3141                 blk_start_queue(skdev->queue);
3142                 break;
3143
3144         /* PCIe bus returned all Fs? */
3145         case 0xFF:
3146                 pr_info("(%s): state=0x%x sense=0x%x\n",
3147                        skd_name(skdev), state, sense);
3148                 skd_drive_disappeared(skdev);
3149                 skd_recover_requests(skdev, 0);
3150                 blk_start_queue(skdev->queue);
3151                 break;
3152         default:
3153                 /*
3154                  * Uknown FW State. Wait for a state we recognize.
3155                  */
3156                 break;
3157         }
3158         pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3159                skd_name(skdev),
3160                skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
3161                skd_skdev_state_to_str(skdev->state), skdev->state);
3162 }
3163
3164 static void skd_recover_requests(struct skd_device *skdev, int requeue)
3165 {
3166         int i;
3167
3168         for (i = 0; i < skdev->num_req_context; i++) {
3169                 struct skd_request_context *skreq = &skdev->skreq_table[i];
3170
3171                 if (skreq->state == SKD_REQ_STATE_BUSY) {
3172                         skd_log_skreq(skdev, skreq, "recover");
3173
3174                         SKD_ASSERT((skreq->id & SKD_ID_INCR) != 0);
3175                         SKD_ASSERT(skreq->req != NULL);
3176
3177                         /* Release DMA resources for the request. */
3178                         if (skreq->n_sg > 0)
3179                                 skd_postop_sg_list(skdev, skreq);
3180
3181                         if (requeue &&
3182                             (unsigned long) ++skreq->req->special <
3183                             SKD_MAX_RETRIES)
3184                                 blk_requeue_request(skdev->queue, skreq->req);
3185                         else
3186                                 skd_end_request(skdev, skreq, -EIO);
3187
3188                         skreq->req = NULL;
3189
3190                         skreq->state = SKD_REQ_STATE_IDLE;
3191                         skreq->id += SKD_ID_INCR;
3192                 }
3193                 if (i > 0)
3194                         skreq[-1].next = skreq;
3195                 skreq->next = NULL;
3196         }
3197         skdev->skreq_free_list = skdev->skreq_table;
3198
3199         for (i = 0; i < skdev->num_fitmsg_context; i++) {
3200                 struct skd_fitmsg_context *skmsg = &skdev->skmsg_table[i];
3201
3202                 if (skmsg->state == SKD_MSG_STATE_BUSY) {
3203                         skd_log_skmsg(skdev, skmsg, "salvaged");
3204                         SKD_ASSERT((skmsg->id & SKD_ID_INCR) != 0);
3205                         skmsg->state = SKD_MSG_STATE_IDLE;
3206                         skmsg->id += SKD_ID_INCR;
3207                 }
3208                 if (i > 0)
3209                         skmsg[-1].next = skmsg;
3210                 skmsg->next = NULL;
3211         }
3212         skdev->skmsg_free_list = skdev->skmsg_table;
3213
3214         for (i = 0; i < skdev->n_special; i++) {
3215                 struct skd_special_context *skspcl = &skdev->skspcl_table[i];
3216
3217                 /* If orphaned, reclaim it because it has already been reported
3218                  * to the process as an error (it was just waiting for
3219                  * a completion that didn't come, and now it will never come)
3220                  * If busy, change to a state that will cause it to error
3221                  * out in the wait routine and let it do the normal
3222                  * reporting and reclaiming
3223                  */
3224                 if (skspcl->req.state == SKD_REQ_STATE_BUSY) {
3225                         if (skspcl->orphaned) {
3226                                 pr_debug("%s:%s:%d orphaned %p\n",
3227                                          skdev->name, __func__, __LINE__,
3228                                          skspcl);
3229                                 skd_release_special(skdev, skspcl);
3230                         } else {
3231                                 pr_debug("%s:%s:%d not orphaned %p\n",
3232                                          skdev->name, __func__, __LINE__,
3233                                          skspcl);
3234                                 skspcl->req.state = SKD_REQ_STATE_ABORTED;
3235                         }
3236                 }
3237         }
3238         skdev->skspcl_free_list = skdev->skspcl_table;
3239
3240         for (i = 0; i < SKD_N_TIMEOUT_SLOT; i++)
3241                 skdev->timeout_slot[i] = 0;
3242
3243         skdev->in_flight = 0;
3244 }
3245
3246 static void skd_isr_msg_from_dev(struct skd_device *skdev)
3247 {
3248         u32 mfd;
3249         u32 mtd;
3250         u32 data;
3251
3252         mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3253
3254         pr_debug("%s:%s:%d mfd=0x%x last_mtd=0x%x\n",
3255                  skdev->name, __func__, __LINE__, mfd, skdev->last_mtd);
3256
3257         /* ignore any mtd that is an ack for something we didn't send */
3258         if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
3259                 return;
3260
3261         switch (FIT_MXD_TYPE(mfd)) {
3262         case FIT_MTD_FITFW_INIT:
3263                 skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
3264
3265                 if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
3266                         pr_err("(%s): protocol mismatch\n",
3267                                skdev->name);
3268                         pr_err("(%s):   got=%d support=%d\n",
3269                                skdev->name, skdev->proto_ver,
3270                                FIT_PROTOCOL_VERSION_1);
3271                         pr_err("(%s):   please upgrade driver\n",
3272                                skdev->name);
3273                         skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
3274                         skd_soft_reset(skdev);
3275                         break;
3276                 }
3277                 mtd = FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH, 0, 0);
3278                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3279                 skdev->last_mtd = mtd;
3280                 break;
3281
3282         case FIT_MTD_GET_CMDQ_DEPTH:
3283                 skdev->dev_max_queue_depth = FIT_MXD_DATA(mfd);
3284                 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH, 0,
3285                                    SKD_N_COMPLETION_ENTRY);
3286                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3287                 skdev->last_mtd = mtd;
3288                 break;
3289
3290         case FIT_MTD_SET_COMPQ_DEPTH:
3291                 SKD_WRITEQ(skdev, skdev->cq_dma_address, FIT_MSG_TO_DEVICE_ARG);
3292                 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR, 0, 0);
3293                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3294                 skdev->last_mtd = mtd;
3295                 break;
3296
3297         case FIT_MTD_SET_COMPQ_ADDR:
3298                 skd_reset_skcomp(skdev);
3299                 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID, 0, skdev->devno);
3300                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3301                 skdev->last_mtd = mtd;
3302                 break;
3303
3304         case FIT_MTD_CMD_LOG_HOST_ID:
3305                 skdev->connect_time_stamp = get_seconds();
3306                 data = skdev->connect_time_stamp & 0xFFFF;
3307                 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO, 0, data);
3308                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3309                 skdev->last_mtd = mtd;
3310                 break;
3311
3312         case FIT_MTD_CMD_LOG_TIME_STAMP_LO:
3313                 skdev->drive_jiffies = FIT_MXD_DATA(mfd);
3314                 data = (skdev->connect_time_stamp >> 16) & 0xFFFF;
3315                 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI, 0, data);
3316                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3317                 skdev->last_mtd = mtd;
3318                 break;
3319
3320         case FIT_MTD_CMD_LOG_TIME_STAMP_HI:
3321                 skdev->drive_jiffies |= (FIT_MXD_DATA(mfd) << 16);
3322                 mtd = FIT_MXD_CONS(FIT_MTD_ARM_QUEUE, 0, 0);
3323                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3324                 skdev->last_mtd = mtd;
3325
3326                 pr_err("(%s): Time sync driver=0x%x device=0x%x\n",
3327                        skd_name(skdev),
3328                        skdev->connect_time_stamp, skdev->drive_jiffies);
3329                 break;
3330
3331         case FIT_MTD_ARM_QUEUE:
3332                 skdev->last_mtd = 0;
3333                 /*
3334                  * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
3335                  */
3336                 break;
3337
3338         default:
3339                 break;
3340         }
3341 }
3342
3343 static void skd_disable_interrupts(struct skd_device *skdev)
3344 {
3345         u32 sense;
3346
3347         sense = SKD_READL(skdev, FIT_CONTROL);
3348         sense &= ~FIT_CR_ENABLE_INTERRUPTS;
3349         SKD_WRITEL(skdev, sense, FIT_CONTROL);
3350         pr_debug("%s:%s:%d sense 0x%x\n",
3351                  skdev->name, __func__, __LINE__, sense);
3352
3353         /* Note that the 1s is written. A 1-bit means
3354          * disable, a 0 means enable.
3355          */
3356         SKD_WRITEL(skdev, ~0, FIT_INT_MASK_HOST);
3357 }
3358
3359 static void skd_enable_interrupts(struct skd_device *skdev)
3360 {
3361         u32 val;
3362
3363         /* unmask interrupts first */
3364         val = FIT_ISH_FW_STATE_CHANGE +
3365               FIT_ISH_COMPLETION_POSTED + FIT_ISH_MSG_FROM_DEV;
3366
3367         /* Note that the compliment of mask is written. A 1-bit means
3368          * disable, a 0 means enable. */
3369         SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
3370         pr_debug("%s:%s:%d interrupt mask=0x%x\n",
3371                  skdev->name, __func__, __LINE__, ~val);
3372
3373         val = SKD_READL(skdev, FIT_CONTROL);
3374         val |= FIT_CR_ENABLE_INTERRUPTS;
3375         pr_debug("%s:%s:%d control=0x%x\n",
3376                  skdev->name, __func__, __LINE__, val);
3377         SKD_WRITEL(skdev, val, FIT_CONTROL);
3378 }
3379
3380 /*
3381  *****************************************************************************
3382  * START, STOP, RESTART, QUIESCE, UNQUIESCE
3383  *****************************************************************************
3384  */
3385
3386 static void skd_soft_reset(struct skd_device *skdev)
3387 {
3388         u32 val;
3389
3390         val = SKD_READL(skdev, FIT_CONTROL);
3391         val |= (FIT_CR_SOFT_RESET);
3392         pr_debug("%s:%s:%d control=0x%x\n",
3393                  skdev->name, __func__, __LINE__, val);
3394         SKD_WRITEL(skdev, val, FIT_CONTROL);
3395 }
3396
3397 static void skd_start_device(struct skd_device *skdev)
3398 {
3399         unsigned long flags;
3400         u32 sense;
3401         u32 state;
3402
3403         spin_lock_irqsave(&skdev->lock, flags);
3404
3405         /* ack all ghost interrupts */
3406         SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3407
3408         sense = SKD_READL(skdev, FIT_STATUS);
3409
3410         pr_debug("%s:%s:%d initial status=0x%x\n",
3411                  skdev->name, __func__, __LINE__, sense);
3412
3413         state = sense & FIT_SR_DRIVE_STATE_MASK;
3414         skdev->drive_state = state;
3415         skdev->last_mtd = 0;
3416
3417         skdev->state = SKD_DRVR_STATE_STARTING;
3418         skdev->timer_countdown = SKD_STARTING_TIMO;
3419
3420         skd_enable_interrupts(skdev);
3421
3422         switch (skdev->drive_state) {
3423         case FIT_SR_DRIVE_OFFLINE:
3424                 pr_err("(%s): Drive offline...\n", skd_name(skdev));
3425                 break;
3426
3427         case FIT_SR_DRIVE_FW_BOOTING:
3428                 pr_debug("%s:%s:%d FIT_SR_DRIVE_FW_BOOTING %s\n",
3429                          skdev->name, __func__, __LINE__, skdev->name);
3430                 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3431                 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3432                 break;
3433
3434         case FIT_SR_DRIVE_BUSY_SANITIZE:
3435                 pr_info("(%s): Start: BUSY_SANITIZE\n",
3436                        skd_name(skdev));
3437                 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3438                 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3439                 break;
3440
3441         case FIT_SR_DRIVE_BUSY_ERASE:
3442                 pr_info("(%s): Start: BUSY_ERASE\n", skd_name(skdev));
3443                 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3444                 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3445                 break;
3446
3447         case FIT_SR_DRIVE_INIT:
3448         case FIT_SR_DRIVE_ONLINE:
3449                 skd_soft_reset(skdev);
3450                 break;
3451
3452         case FIT_SR_DRIVE_BUSY:
3453                 pr_err("(%s): Drive Busy...\n", skd_name(skdev));
3454                 skdev->state = SKD_DRVR_STATE_BUSY;
3455                 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3456                 break;
3457
3458         case FIT_SR_DRIVE_SOFT_RESET:
3459                 pr_err("(%s) drive soft reset in prog\n",
3460                        skd_name(skdev));
3461                 break;
3462
3463         case FIT_SR_DRIVE_FAULT:
3464                 /* Fault state is bad...soft reset won't do it...
3465                  * Hard reset, maybe, but does it work on device?
3466                  * For now, just fault so the system doesn't hang.
3467                  */
3468                 skd_drive_fault(skdev);
3469                 /*start the queue so we can respond with error to requests */
3470                 pr_debug("%s:%s:%d starting %s queue\n",
3471                          skdev->name, __func__, __LINE__, skdev->name);
3472                 blk_start_queue(skdev->queue);
3473                 skdev->gendisk_on = -1;
3474                 wake_up_interruptible(&skdev->waitq);
3475                 break;
3476
3477         case 0xFF:
3478                 /* Most likely the device isn't there or isn't responding
3479                  * to the BAR1 addresses. */
3480                 skd_drive_disappeared(skdev);
3481                 /*start the queue so we can respond with error to requests */
3482                 pr_debug("%s:%s:%d starting %s queue to error-out reqs\n",
3483                          skdev->name, __func__, __LINE__, skdev->name);
3484                 blk_start_queue(skdev->queue);
3485                 skdev->gendisk_on = -1;
3486                 wake_up_interruptible(&skdev->waitq);
3487                 break;
3488
3489         default:
3490                 pr_err("(%s) Start: unknown state %x\n",
3491                        skd_name(skdev), skdev->drive_state);
3492                 break;
3493         }
3494
3495         state = SKD_READL(skdev, FIT_CONTROL);
3496         pr_debug("%s:%s:%d FIT Control Status=0x%x\n",
3497                  skdev->name, __func__, __LINE__, state);
3498
3499         state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
3500         pr_debug("%s:%s:%d Intr Status=0x%x\n",
3501                  skdev->name, __func__, __LINE__, state);
3502
3503         state = SKD_READL(skdev, FIT_INT_MASK_HOST);
3504         pr_debug("%s:%s:%d Intr Mask=0x%x\n",
3505                  skdev->name, __func__, __LINE__, state);
3506
3507         state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3508         pr_debug("%s:%s:%d Msg from Dev=0x%x\n",
3509                  skdev->name, __func__, __LINE__, state);
3510
3511         state = SKD_READL(skdev, FIT_HW_VERSION);
3512         pr_debug("%s:%s:%d HW version=0x%x\n",
3513                  skdev->name, __func__, __LINE__, state);
3514
3515         spin_unlock_irqrestore(&skdev->lock, flags);
3516 }
3517
3518 static void skd_stop_device(struct skd_device *skdev)
3519 {
3520         unsigned long flags;
3521         struct skd_special_context *skspcl = &skdev->internal_skspcl;
3522         u32 dev_state;
3523         int i;
3524
3525         spin_lock_irqsave(&skdev->lock, flags);
3526
3527         if (skdev->state != SKD_DRVR_STATE_ONLINE) {
3528                 pr_err("(%s): skd_stop_device not online no sync\n",
3529                        skd_name(skdev));
3530                 goto stop_out;
3531         }
3532
3533         if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
3534                 pr_err("(%s): skd_stop_device no special\n",
3535                        skd_name(skdev));
3536                 goto stop_out;
3537         }
3538
3539         skdev->state = SKD_DRVR_STATE_SYNCING;
3540         skdev->sync_done = 0;
3541
3542         skd_send_internal_skspcl(skdev, skspcl, SYNCHRONIZE_CACHE);
3543
3544         spin_unlock_irqrestore(&skdev->lock, flags);
3545
3546         wait_event_interruptible_timeout(skdev->waitq,
3547                                          (skdev->sync_done), (10 * HZ));
3548
3549         spin_lock_irqsave(&skdev->lock, flags);
3550
3551         switch (skdev->sync_done) {
3552         case 0:
3553                 pr_err("(%s): skd_stop_device no sync\n",
3554                        skd_name(skdev));
3555                 break;
3556         case 1:
3557                 pr_err("(%s): skd_stop_device sync done\n",
3558                        skd_name(skdev));
3559                 break;
3560         default:
3561                 pr_err("(%s): skd_stop_device sync error\n",
3562                        skd_name(skdev));
3563         }
3564
3565 stop_out:
3566         skdev->state = SKD_DRVR_STATE_STOPPING;
3567         spin_unlock_irqrestore(&skdev->lock, flags);
3568
3569         skd_kill_timer(skdev);
3570
3571         spin_lock_irqsave(&skdev->lock, flags);
3572         skd_disable_interrupts(skdev);
3573
3574         /* ensure all ints on device are cleared */
3575         /* soft reset the device to unload with a clean slate */
3576         SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3577         SKD_WRITEL(skdev, FIT_CR_SOFT_RESET, FIT_CONTROL);
3578
3579         spin_unlock_irqrestore(&skdev->lock, flags);
3580
3581         /* poll every 100ms, 1 second timeout */
3582         for (i = 0; i < 10; i++) {
3583                 dev_state =
3584                         SKD_READL(skdev, FIT_STATUS) & FIT_SR_DRIVE_STATE_MASK;
3585                 if (dev_state == FIT_SR_DRIVE_INIT)
3586                         break;
3587                 set_current_state(TASK_INTERRUPTIBLE);
3588                 schedule_timeout(msecs_to_jiffies(100));
3589         }
3590
3591         if (dev_state != FIT_SR_DRIVE_INIT)
3592                 pr_err("(%s): skd_stop_device state error 0x%02x\n",
3593                        skd_name(skdev), dev_state);
3594 }
3595
3596 /* assume spinlock is held */
3597 static void skd_restart_device(struct skd_device *skdev)
3598 {
3599         u32 state;
3600
3601         /* ack all ghost interrupts */
3602         SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3603
3604         state = SKD_READL(skdev, FIT_STATUS);
3605
3606         pr_debug("%s:%s:%d drive status=0x%x\n",
3607                  skdev->name, __func__, __LINE__, state);
3608
3609         state &= FIT_SR_DRIVE_STATE_MASK;
3610         skdev->drive_state = state;
3611         skdev->last_mtd = 0;
3612
3613         skdev->state = SKD_DRVR_STATE_RESTARTING;
3614         skdev->timer_countdown = SKD_RESTARTING_TIMO;
3615
3616         skd_soft_reset(skdev);
3617 }
3618
3619 /* assume spinlock is held */
3620 static int skd_quiesce_dev(struct skd_device *skdev)
3621 {
3622         int rc = 0;
3623
3624         switch (skdev->state) {
3625         case SKD_DRVR_STATE_BUSY:
3626         case SKD_DRVR_STATE_BUSY_IMMINENT:
3627                 pr_debug("%s:%s:%d stopping %s queue\n",
3628                          skdev->name, __func__, __LINE__, skdev->name);
3629                 blk_stop_queue(skdev->queue);
3630                 break;
3631         case SKD_DRVR_STATE_ONLINE:
3632         case SKD_DRVR_STATE_STOPPING:
3633         case SKD_DRVR_STATE_SYNCING:
3634         case SKD_DRVR_STATE_PAUSING:
3635         case SKD_DRVR_STATE_PAUSED:
3636         case SKD_DRVR_STATE_STARTING:
3637         case SKD_DRVR_STATE_RESTARTING:
3638         case SKD_DRVR_STATE_RESUMING:
3639         default:
3640                 rc = -EINVAL;
3641                 pr_debug("%s:%s:%d state [%d] not implemented\n",
3642                          skdev->name, __func__, __LINE__, skdev->state);
3643         }
3644         return rc;
3645 }
3646
3647 /* assume spinlock is held */
3648 static int skd_unquiesce_dev(struct skd_device *skdev)
3649 {
3650         int prev_driver_state = skdev->state;
3651
3652         skd_log_skdev(skdev, "unquiesce");
3653         if (skdev->state == SKD_DRVR_STATE_ONLINE) {
3654                 pr_debug("%s:%s:%d **** device already ONLINE\n",
3655                          skdev->name, __func__, __LINE__);
3656                 return 0;
3657         }
3658         if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
3659                 /*
3660                  * If there has been an state change to other than
3661                  * ONLINE, we will rely on controller state change
3662                  * to come back online and restart the queue.
3663                  * The BUSY state means that driver is ready to
3664                  * continue normal processing but waiting for controller
3665                  * to become available.
3666                  */
3667                 skdev->state = SKD_DRVR_STATE_BUSY;
3668                 pr_debug("%s:%s:%d drive BUSY state\n",
3669                          skdev->name, __func__, __LINE__);
3670                 return 0;
3671         }
3672
3673         /*
3674          * Drive has just come online, driver is either in startup,
3675          * paused performing a task, or bust waiting for hardware.
3676          */
3677         switch (skdev->state) {
3678         case SKD_DRVR_STATE_PAUSED:
3679         case SKD_DRVR_STATE_BUSY:
3680         case SKD_DRVR_STATE_BUSY_IMMINENT:
3681         case SKD_DRVR_STATE_BUSY_ERASE:
3682         case SKD_DRVR_STATE_STARTING:
3683         case SKD_DRVR_STATE_RESTARTING:
3684         case SKD_DRVR_STATE_FAULT:
3685         case SKD_DRVR_STATE_IDLE:
3686         case SKD_DRVR_STATE_LOAD:
3687                 skdev->state = SKD_DRVR_STATE_ONLINE;
3688                 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3689                        skd_name(skdev),
3690                        skd_skdev_state_to_str(prev_driver_state),
3691                        prev_driver_state, skd_skdev_state_to_str(skdev->state),
3692                        skdev->state);
3693                 pr_debug("%s:%s:%d **** device ONLINE...starting block queue\n",
3694                          skdev->name, __func__, __LINE__);
3695                 pr_debug("%s:%s:%d starting %s queue\n",
3696                          skdev->name, __func__, __LINE__, skdev->name);
3697                 pr_info("(%s): STEC s1120 ONLINE\n", skd_name(skdev));
3698                 blk_start_queue(skdev->queue);
3699                 skdev->gendisk_on = 1;
3700                 wake_up_interruptible(&skdev->waitq);
3701                 break;
3702
3703         case SKD_DRVR_STATE_DISAPPEARED:
3704         default:
3705                 pr_debug("%s:%s:%d **** driver state %d, not implemented \n",
3706                          skdev->name, __func__, __LINE__,
3707                          skdev->state);
3708                 return -EBUSY;
3709         }
3710         return 0;
3711 }
3712
3713 /*
3714  *****************************************************************************
3715  * PCIe MSI/MSI-X INTERRUPT HANDLERS
3716  *****************************************************************************
3717  */
3718
3719 static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
3720 {
3721         struct skd_device *skdev = skd_host_data;
3722         unsigned long flags;
3723
3724         spin_lock_irqsave(&skdev->lock, flags);
3725         pr_debug("%s:%s:%d MSIX = 0x%x\n",
3726                  skdev->name, __func__, __LINE__,
3727                  SKD_READL(skdev, FIT_INT_STATUS_HOST));
3728         pr_err("(%s): MSIX reserved irq %d = 0x%x\n", skd_name(skdev),
3729                irq, SKD_READL(skdev, FIT_INT_STATUS_HOST));
3730         SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
3731         spin_unlock_irqrestore(&skdev->lock, flags);
3732         return IRQ_HANDLED;
3733 }
3734
3735 static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
3736 {
3737         struct skd_device *skdev = skd_host_data;
3738         unsigned long flags;
3739
3740         spin_lock_irqsave(&skdev->lock, flags);
3741         pr_debug("%s:%s:%d MSIX = 0x%x\n",
3742                  skdev->name, __func__, __LINE__,
3743                  SKD_READL(skdev, FIT_INT_STATUS_HOST));
3744         SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
3745         skd_isr_fwstate(skdev);
3746         spin_unlock_irqrestore(&skdev->lock, flags);
3747         return IRQ_HANDLED;
3748 }
3749
3750 static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
3751 {
3752         struct skd_device *skdev = skd_host_data;
3753         unsigned long flags;
3754         int flush_enqueued = 0;
3755         int deferred;
3756
3757         spin_lock_irqsave(&skdev->lock, flags);
3758         pr_debug("%s:%s:%d MSIX = 0x%x\n",
3759                  skdev->name, __func__, __LINE__,
3760                  SKD_READL(skdev, FIT_INT_STATUS_HOST));
3761         SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
3762         deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
3763                                                 &flush_enqueued);
3764         if (flush_enqueued)
3765                 skd_request_fn(skdev->queue);
3766
3767         if (deferred)
3768                 schedule_work(&skdev->completion_worker);
3769         else if (!flush_enqueued)
3770                 skd_request_fn(skdev->queue);
3771
3772         spin_unlock_irqrestore(&skdev->lock, flags);
3773
3774         return IRQ_HANDLED;
3775 }
3776
3777 static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
3778 {
3779         struct skd_device *skdev = skd_host_data;
3780         unsigned long flags;
3781
3782         spin_lock_irqsave(&skdev->lock, flags);
3783         pr_debug("%s:%s:%d MSIX = 0x%x\n",
3784                  skdev->name, __func__, __LINE__,
3785                  SKD_READL(skdev, FIT_INT_STATUS_HOST));
3786         SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
3787         skd_isr_msg_from_dev(skdev);
3788         spin_unlock_irqrestore(&skdev->lock, flags);
3789         return IRQ_HANDLED;
3790 }
3791
3792 static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
3793 {
3794         struct skd_device *skdev = skd_host_data;
3795         unsigned long flags;
3796
3797         spin_lock_irqsave(&skdev->lock, flags);
3798         pr_debug("%s:%s:%d MSIX = 0x%x\n",
3799                  skdev->name, __func__, __LINE__,
3800                  SKD_READL(skdev, FIT_INT_STATUS_HOST));
3801         SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
3802         spin_unlock_irqrestore(&skdev->lock, flags);
3803         return IRQ_HANDLED;
3804 }
3805
3806 /*
3807  *****************************************************************************
3808  * PCIe MSI/MSI-X SETUP
3809  *****************************************************************************
3810  */
3811
3812 struct skd_msix_entry {
3813         char isr_name[30];
3814 };
3815
3816 struct skd_init_msix_entry {
3817         const char *name;
3818         irq_handler_t handler;
3819 };
3820
3821 #define SKD_MAX_MSIX_COUNT              13
3822 #define SKD_MIN_MSIX_COUNT              7
3823 #define SKD_BASE_MSIX_IRQ               4
3824
3825 static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
3826         { "(DMA 0)",        skd_reserved_isr },
3827         { "(DMA 1)",        skd_reserved_isr },
3828         { "(DMA 2)",        skd_reserved_isr },
3829         { "(DMA 3)",        skd_reserved_isr },
3830         { "(State Change)", skd_statec_isr   },
3831         { "(COMPL_Q)",      skd_comp_q       },
3832         { "(MSG)",          skd_msg_isr      },
3833         { "(Reserved)",     skd_reserved_isr },
3834         { "(Reserved)",     skd_reserved_isr },
3835         { "(Queue Full 0)", skd_qfull_isr    },
3836         { "(Queue Full 1)", skd_qfull_isr    },
3837         { "(Queue Full 2)", skd_qfull_isr    },
3838         { "(Queue Full 3)", skd_qfull_isr    },
3839 };
3840
3841 static int skd_acquire_msix(struct skd_device *skdev)
3842 {
3843         int i, rc;
3844         struct pci_dev *pdev = skdev->pdev;
3845
3846         rc = pci_alloc_irq_vectors(pdev, SKD_MAX_MSIX_COUNT, SKD_MAX_MSIX_COUNT,
3847                         PCI_IRQ_MSIX);
3848         if (rc < 0) {
3849                 pr_err("(%s): failed to enable MSI-X %d\n",
3850                        skd_name(skdev), rc);
3851                 goto out;
3852         }
3853
3854         skdev->msix_entries = kcalloc(SKD_MAX_MSIX_COUNT,
3855                         sizeof(struct skd_msix_entry), GFP_KERNEL);
3856         if (!skdev->msix_entries) {
3857                 rc = -ENOMEM;
3858                 pr_err("(%s): msix table allocation error\n",
3859                        skd_name(skdev));
3860                 goto out;
3861         }
3862
3863         /* Enable MSI-X vectors for the base queue */
3864         for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
3865                 struct skd_msix_entry *qentry = &skdev->msix_entries[i];
3866
3867                 snprintf(qentry->isr_name, sizeof(qentry->isr_name),
3868                          "%s%d-msix %s", DRV_NAME, skdev->devno,
3869                          msix_entries[i].name);
3870
3871                 rc = devm_request_irq(&skdev->pdev->dev,
3872                                 pci_irq_vector(skdev->pdev, i),
3873                                 msix_entries[i].handler, 0,
3874                                 qentry->isr_name, skdev);
3875                 if (rc) {
3876                         pr_err("(%s): Unable to register(%d) MSI-X "
3877                                "handler %d: %s\n",
3878                                skd_name(skdev), rc, i, qentry->isr_name);
3879                         goto msix_out;
3880                 }
3881         }
3882
3883         pr_debug("%s:%s:%d %s: <%s> msix %d irq(s) enabled\n",
3884                  skdev->name, __func__, __LINE__,
3885                  pci_name(pdev), skdev->name, SKD_MAX_MSIX_COUNT);
3886         return 0;
3887
3888 msix_out:
3889         while (--i >= 0)
3890                 devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), skdev);
3891 out:
3892         kfree(skdev->msix_entries);
3893         skdev->msix_entries = NULL;
3894         return rc;
3895 }
3896
3897 static int skd_acquire_irq(struct skd_device *skdev)
3898 {
3899         struct pci_dev *pdev = skdev->pdev;
3900         unsigned int irq_flag = PCI_IRQ_LEGACY;
3901         int rc;
3902
3903         if (skd_isr_type == SKD_IRQ_MSIX) {
3904                 rc = skd_acquire_msix(skdev);
3905                 if (!rc)
3906                         return 0;
3907
3908                 pr_err("(%s): failed to enable MSI-X, re-trying with MSI %d\n",
3909                        skd_name(skdev), rc);
3910         }
3911
3912         snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d", DRV_NAME,
3913                         skdev->devno);
3914
3915         if (skd_isr_type != SKD_IRQ_LEGACY)
3916                 irq_flag |= PCI_IRQ_MSI;
3917         rc = pci_alloc_irq_vectors(pdev, 1, 1, irq_flag);
3918         if (rc < 0) {
3919                 pr_err("(%s): failed to allocate the MSI interrupt %d\n",
3920                         skd_name(skdev), rc);
3921                 return rc;
3922         }
3923
3924         rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
3925                         pdev->msi_enabled ? 0 : IRQF_SHARED,
3926                         skdev->isr_name, skdev);
3927         if (rc) {
3928                 pci_free_irq_vectors(pdev);
3929                 pr_err("(%s): failed to allocate interrupt %d\n",
3930                         skd_name(skdev), rc);
3931                 return rc;
3932         }
3933
3934         return 0;
3935 }
3936
3937 static void skd_release_irq(struct skd_device *skdev)
3938 {
3939         struct pci_dev *pdev = skdev->pdev;
3940
3941         if (skdev->msix_entries) {
3942                 int i;
3943
3944                 for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
3945                         devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i),
3946                                         skdev);
3947                 }
3948
3949                 kfree(skdev->msix_entries);
3950                 skdev->msix_entries = NULL;
3951         } else {
3952                 devm_free_irq(&pdev->dev, pdev->irq, skdev);
3953         }
3954
3955         pci_free_irq_vectors(pdev);
3956 }
3957
3958 /*
3959  *****************************************************************************
3960  * CONSTRUCT
3961  *****************************************************************************
3962  */
3963
3964 static int skd_cons_skcomp(struct skd_device *skdev)
3965 {
3966         int rc = 0;
3967         struct fit_completion_entry_v1 *skcomp;
3968         u32 nbytes;
3969
3970         nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
3971         nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
3972
3973         pr_debug("%s:%s:%d comp pci_alloc, total bytes %d entries %d\n",
3974                  skdev->name, __func__, __LINE__,
3975                  nbytes, SKD_N_COMPLETION_ENTRY);
3976
3977         skcomp = pci_zalloc_consistent(skdev->pdev, nbytes,
3978                                        &skdev->cq_dma_address);
3979
3980         if (skcomp == NULL) {
3981                 rc = -ENOMEM;
3982                 goto err_out;
3983         }
3984
3985         skdev->skcomp_table = skcomp;
3986         skdev->skerr_table = (struct fit_comp_error_info *)((char *)skcomp +
3987                                                            sizeof(*skcomp) *
3988                                                            SKD_N_COMPLETION_ENTRY);
3989
3990 err_out:
3991         return rc;
3992 }
3993
3994 static int skd_cons_skmsg(struct skd_device *skdev)
3995 {
3996         int rc = 0;
3997         u32 i;
3998
3999         pr_debug("%s:%s:%d skmsg_table kzalloc, struct %lu, count %u total %lu\n",
4000                  skdev->name, __func__, __LINE__,
4001                  sizeof(struct skd_fitmsg_context),
4002                  skdev->num_fitmsg_context,
4003                  sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
4004
4005         skdev->skmsg_table = kzalloc(sizeof(struct skd_fitmsg_context)
4006                                      *skdev->num_fitmsg_context, GFP_KERNEL);
4007         if (skdev->skmsg_table == NULL) {
4008                 rc = -ENOMEM;
4009                 goto err_out;
4010         }
4011
4012         for (i = 0; i < skdev->num_fitmsg_context; i++) {
4013                 struct skd_fitmsg_context *skmsg;
4014
4015                 skmsg = &skdev->skmsg_table[i];
4016
4017                 skmsg->id = i + SKD_ID_FIT_MSG;
4018
4019                 skmsg->state = SKD_MSG_STATE_IDLE;
4020                 skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
4021                                                       SKD_N_FITMSG_BYTES + 64,
4022                                                       &skmsg->mb_dma_address);
4023
4024                 if (skmsg->msg_buf == NULL) {
4025                         rc = -ENOMEM;
4026                         goto err_out;
4027                 }
4028
4029                 skmsg->offset = (u32)((u64)skmsg->msg_buf &
4030                                       (~FIT_QCMD_BASE_ADDRESS_MASK));
4031                 skmsg->msg_buf += ~FIT_QCMD_BASE_ADDRESS_MASK;
4032                 skmsg->msg_buf = (u8 *)((u64)skmsg->msg_buf &
4033                                        FIT_QCMD_BASE_ADDRESS_MASK);
4034                 skmsg->mb_dma_address += ~FIT_QCMD_BASE_ADDRESS_MASK;
4035                 skmsg->mb_dma_address &= FIT_QCMD_BASE_ADDRESS_MASK;
4036                 memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
4037
4038                 skmsg->next = &skmsg[1];
4039         }
4040
4041         /* Free list is in order starting with the 0th entry. */
4042         skdev->skmsg_table[i - 1].next = NULL;
4043         skdev->skmsg_free_list = skdev->skmsg_table;
4044
4045 err_out:
4046         return rc;
4047 }
4048
4049 static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4050                                                   u32 n_sg,
4051                                                   dma_addr_t *ret_dma_addr)
4052 {
4053         struct fit_sg_descriptor *sg_list;
4054         u32 nbytes;
4055
4056         nbytes = sizeof(*sg_list) * n_sg;
4057
4058         sg_list = pci_alloc_consistent(skdev->pdev, nbytes, ret_dma_addr);
4059
4060         if (sg_list != NULL) {
4061                 uint64_t dma_address = *ret_dma_addr;
4062                 u32 i;
4063
4064                 memset(sg_list, 0, nbytes);
4065
4066                 for (i = 0; i < n_sg - 1; i++) {
4067                         uint64_t ndp_off;
4068                         ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
4069
4070                         sg_list[i].next_desc_ptr = dma_address + ndp_off;
4071                 }
4072                 sg_list[i].next_desc_ptr = 0LL;
4073         }
4074
4075         return sg_list;
4076 }
4077
4078 static int skd_cons_skreq(struct skd_device *skdev)
4079 {
4080         int rc = 0;
4081         u32 i;
4082
4083         pr_debug("%s:%s:%d skreq_table kzalloc, struct %lu, count %u total %lu\n",
4084                  skdev->name, __func__, __LINE__,
4085                  sizeof(struct skd_request_context),
4086                  skdev->num_req_context,
4087                  sizeof(struct skd_request_context) * skdev->num_req_context);
4088
4089         skdev->skreq_table = kzalloc(sizeof(struct skd_request_context)
4090                                      * skdev->num_req_context, GFP_KERNEL);
4091         if (skdev->skreq_table == NULL) {
4092                 rc = -ENOMEM;
4093                 goto err_out;
4094         }
4095
4096         pr_debug("%s:%s:%d alloc sg_table sg_per_req %u scatlist %lu total %lu\n",
4097                  skdev->name, __func__, __LINE__,
4098                  skdev->sgs_per_request, sizeof(struct scatterlist),
4099                  skdev->sgs_per_request * sizeof(struct scatterlist));
4100
4101         for (i = 0; i < skdev->num_req_context; i++) {
4102                 struct skd_request_context *skreq;
4103
4104                 skreq = &skdev->skreq_table[i];
4105
4106                 skreq->id = i + SKD_ID_RW_REQUEST;
4107                 skreq->state = SKD_REQ_STATE_IDLE;
4108
4109                 skreq->sg = kzalloc(sizeof(struct scatterlist) *
4110                                     skdev->sgs_per_request, GFP_KERNEL);
4111                 if (skreq->sg == NULL) {
4112                         rc = -ENOMEM;
4113                         goto err_out;
4114                 }
4115                 sg_init_table(skreq->sg, skdev->sgs_per_request);
4116
4117                 skreq->sksg_list = skd_cons_sg_list(skdev,
4118                                                     skdev->sgs_per_request,
4119                                                     &skreq->sksg_dma_address);
4120
4121                 if (skreq->sksg_list == NULL) {
4122                         rc = -ENOMEM;
4123                         goto err_out;
4124                 }
4125
4126                 skreq->next = &skreq[1];
4127         }
4128
4129         /* Free list is in order starting with the 0th entry. */
4130         skdev->skreq_table[i - 1].next = NULL;
4131         skdev->skreq_free_list = skdev->skreq_table;
4132
4133 err_out:
4134         return rc;
4135 }
4136
4137 static int skd_cons_skspcl(struct skd_device *skdev)
4138 {
4139         int rc = 0;
4140         u32 i, nbytes;
4141
4142         pr_debug("%s:%s:%d skspcl_table kzalloc, struct %lu, count %u total %lu\n",
4143                  skdev->name, __func__, __LINE__,
4144                  sizeof(struct skd_special_context),
4145                  skdev->n_special,
4146                  sizeof(struct skd_special_context) * skdev->n_special);
4147
4148         skdev->skspcl_table = kzalloc(sizeof(struct skd_special_context)
4149                                       * skdev->n_special, GFP_KERNEL);
4150         if (skdev->skspcl_table == NULL) {
4151                 rc = -ENOMEM;
4152                 goto err_out;
4153         }
4154
4155         for (i = 0; i < skdev->n_special; i++) {
4156                 struct skd_special_context *skspcl;
4157
4158                 skspcl = &skdev->skspcl_table[i];
4159
4160                 skspcl->req.id = i + SKD_ID_SPECIAL_REQUEST;
4161                 skspcl->req.state = SKD_REQ_STATE_IDLE;
4162
4163                 skspcl->req.next = &skspcl[1].req;
4164
4165                 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4166
4167                 skspcl->msg_buf =
4168                         pci_zalloc_consistent(skdev->pdev, nbytes,
4169                                               &skspcl->mb_dma_address);
4170                 if (skspcl->msg_buf == NULL) {
4171                         rc = -ENOMEM;
4172                         goto err_out;
4173                 }
4174
4175                 skspcl->req.sg = kzalloc(sizeof(struct scatterlist) *
4176                                          SKD_N_SG_PER_SPECIAL, GFP_KERNEL);
4177                 if (skspcl->req.sg == NULL) {
4178                         rc = -ENOMEM;
4179                         goto err_out;
4180                 }
4181
4182                 skspcl->req.sksg_list = skd_cons_sg_list(skdev,
4183                                                          SKD_N_SG_PER_SPECIAL,
4184                                                          &skspcl->req.
4185                                                          sksg_dma_address);
4186                 if (skspcl->req.sksg_list == NULL) {
4187                         rc = -ENOMEM;
4188                         goto err_out;
4189                 }
4190         }
4191
4192         /* Free list is in order starting with the 0th entry. */
4193         skdev->skspcl_table[i - 1].req.next = NULL;
4194         skdev->skspcl_free_list = skdev->skspcl_table;
4195
4196         return rc;
4197
4198 err_out:
4199         return rc;
4200 }
4201
4202 static int skd_cons_sksb(struct skd_device *skdev)
4203 {
4204         int rc = 0;
4205         struct skd_special_context *skspcl;
4206         u32 nbytes;
4207
4208         skspcl = &skdev->internal_skspcl;
4209
4210         skspcl->req.id = 0 + SKD_ID_INTERNAL;
4211         skspcl->req.state = SKD_REQ_STATE_IDLE;
4212
4213         nbytes = SKD_N_INTERNAL_BYTES;
4214
4215         skspcl->data_buf = pci_zalloc_consistent(skdev->pdev, nbytes,
4216                                                  &skspcl->db_dma_address);
4217         if (skspcl->data_buf == NULL) {
4218                 rc = -ENOMEM;
4219                 goto err_out;
4220         }
4221
4222         nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4223         skspcl->msg_buf = pci_zalloc_consistent(skdev->pdev, nbytes,
4224                                                 &skspcl->mb_dma_address);
4225         if (skspcl->msg_buf == NULL) {
4226                 rc = -ENOMEM;
4227                 goto err_out;
4228         }
4229
4230         skspcl->req.sksg_list = skd_cons_sg_list(skdev, 1,
4231                                                  &skspcl->req.sksg_dma_address);
4232         if (skspcl->req.sksg_list == NULL) {
4233                 rc = -ENOMEM;
4234                 goto err_out;
4235         }
4236
4237         if (!skd_format_internal_skspcl(skdev)) {
4238                 rc = -EINVAL;
4239                 goto err_out;
4240         }
4241
4242 err_out:
4243         return rc;
4244 }
4245
4246 static int skd_cons_disk(struct skd_device *skdev)
4247 {
4248         int rc = 0;
4249         struct gendisk *disk;
4250         struct request_queue *q;
4251         unsigned long flags;
4252
4253         disk = alloc_disk(SKD_MINORS_PER_DEVICE);
4254         if (!disk) {
4255                 rc = -ENOMEM;
4256                 goto err_out;
4257         }
4258
4259         skdev->disk = disk;
4260         sprintf(disk->disk_name, DRV_NAME "%u", skdev->devno);
4261
4262         disk->major = skdev->major;
4263         disk->first_minor = skdev->devno * SKD_MINORS_PER_DEVICE;
4264         disk->fops = &skd_blockdev_ops;
4265         disk->private_data = skdev;
4266
4267         q = blk_init_queue(skd_request_fn, &skdev->lock);
4268         if (!q) {
4269                 rc = -ENOMEM;
4270                 goto err_out;
4271         }
4272
4273         skdev->queue = q;
4274         disk->queue = q;
4275         q->queuedata = skdev;
4276
4277         blk_queue_write_cache(q, true, true);
4278         blk_queue_max_segments(q, skdev->sgs_per_request);
4279         blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
4280
4281         /* set sysfs ptimal_io_size to 8K */
4282         blk_queue_io_opt(q, 8192);
4283
4284         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
4285         queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q);
4286
4287         spin_lock_irqsave(&skdev->lock, flags);
4288         pr_debug("%s:%s:%d stopping %s queue\n",
4289                  skdev->name, __func__, __LINE__, skdev->name);
4290         blk_stop_queue(skdev->queue);
4291         spin_unlock_irqrestore(&skdev->lock, flags);
4292
4293 err_out:
4294         return rc;
4295 }
4296
4297 #define SKD_N_DEV_TABLE         16u
4298 static u32 skd_next_devno;
4299
4300 static struct skd_device *skd_construct(struct pci_dev *pdev)
4301 {
4302         struct skd_device *skdev;
4303         int blk_major = skd_major;
4304         int rc;
4305
4306         skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
4307
4308         if (!skdev) {
4309                 pr_err(PFX "(%s): memory alloc failure\n",
4310                        pci_name(pdev));
4311                 return NULL;
4312         }
4313
4314         skdev->state = SKD_DRVR_STATE_LOAD;
4315         skdev->pdev = pdev;
4316         skdev->devno = skd_next_devno++;
4317         skdev->major = blk_major;
4318         sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
4319         skdev->dev_max_queue_depth = 0;
4320
4321         skdev->num_req_context = skd_max_queue_depth;
4322         skdev->num_fitmsg_context = skd_max_queue_depth;
4323         skdev->n_special = skd_max_pass_thru;
4324         skdev->cur_max_queue_depth = 1;
4325         skdev->queue_low_water_mark = 1;
4326         skdev->proto_ver = 99;
4327         skdev->sgs_per_request = skd_sgs_per_request;
4328         skdev->dbg_level = skd_dbg_level;
4329
4330         atomic_set(&skdev->device_count, 0);
4331
4332         spin_lock_init(&skdev->lock);
4333
4334         INIT_WORK(&skdev->completion_worker, skd_completion_worker);
4335
4336         pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
4337         rc = skd_cons_skcomp(skdev);
4338         if (rc < 0)
4339                 goto err_out;
4340
4341         pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
4342         rc = skd_cons_skmsg(skdev);
4343         if (rc < 0)
4344                 goto err_out;
4345
4346         pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
4347         rc = skd_cons_skreq(skdev);
4348         if (rc < 0)
4349                 goto err_out;
4350
4351         pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
4352         rc = skd_cons_skspcl(skdev);
4353         if (rc < 0)
4354                 goto err_out;
4355
4356         pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
4357         rc = skd_cons_sksb(skdev);
4358         if (rc < 0)
4359                 goto err_out;
4360
4361         pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
4362         rc = skd_cons_disk(skdev);
4363         if (rc < 0)
4364                 goto err_out;
4365
4366         pr_debug("%s:%s:%d VICTORY\n", skdev->name, __func__, __LINE__);
4367         return skdev;
4368
4369 err_out:
4370         pr_debug("%s:%s:%d construct failed\n",
4371                  skdev->name, __func__, __LINE__);
4372         skd_destruct(skdev);
4373         return NULL;
4374 }
4375
4376 /*
4377  *****************************************************************************
4378  * DESTRUCT (FREE)
4379  *****************************************************************************
4380  */
4381
4382 static void skd_free_skcomp(struct skd_device *skdev)
4383 {
4384         if (skdev->skcomp_table != NULL) {
4385                 u32 nbytes;
4386
4387                 nbytes = sizeof(skdev->skcomp_table[0]) *
4388                          SKD_N_COMPLETION_ENTRY;
4389                 pci_free_consistent(skdev->pdev, nbytes,
4390                                     skdev->skcomp_table, skdev->cq_dma_address);
4391         }
4392
4393         skdev->skcomp_table = NULL;
4394         skdev->cq_dma_address = 0;
4395 }
4396
4397 static void skd_free_skmsg(struct skd_device *skdev)
4398 {
4399         u32 i;
4400
4401         if (skdev->skmsg_table == NULL)
4402                 return;
4403
4404         for (i = 0; i < skdev->num_fitmsg_context; i++) {
4405                 struct skd_fitmsg_context *skmsg;
4406
4407                 skmsg = &skdev->skmsg_table[i];
4408
4409                 if (skmsg->msg_buf != NULL) {
4410                         skmsg->msg_buf += skmsg->offset;
4411                         skmsg->mb_dma_address += skmsg->offset;
4412                         pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
4413                                             skmsg->msg_buf,
4414                                             skmsg->mb_dma_address);
4415                 }
4416                 skmsg->msg_buf = NULL;
4417                 skmsg->mb_dma_address = 0;
4418         }
4419
4420         kfree(skdev->skmsg_table);
4421         skdev->skmsg_table = NULL;
4422 }
4423
4424 static void skd_free_sg_list(struct skd_device *skdev,
4425                              struct fit_sg_descriptor *sg_list,
4426                              u32 n_sg, dma_addr_t dma_addr)
4427 {
4428         if (sg_list != NULL) {
4429                 u32 nbytes;
4430
4431                 nbytes = sizeof(*sg_list) * n_sg;
4432
4433                 pci_free_consistent(skdev->pdev, nbytes, sg_list, dma_addr);
4434         }
4435 }
4436
4437 static void skd_free_skreq(struct skd_device *skdev)
4438 {
4439         u32 i;
4440
4441         if (skdev->skreq_table == NULL)
4442                 return;
4443
4444         for (i = 0; i < skdev->num_req_context; i++) {
4445                 struct skd_request_context *skreq;
4446
4447                 skreq = &skdev->skreq_table[i];
4448
4449                 skd_free_sg_list(skdev, skreq->sksg_list,
4450                                  skdev->sgs_per_request,
4451                                  skreq->sksg_dma_address);
4452
4453                 skreq->sksg_list = NULL;
4454                 skreq->sksg_dma_address = 0;
4455
4456                 kfree(skreq->sg);
4457         }
4458
4459         kfree(skdev->skreq_table);
4460         skdev->skreq_table = NULL;
4461 }
4462
4463 static void skd_free_skspcl(struct skd_device *skdev)
4464 {
4465         u32 i;
4466         u32 nbytes;
4467
4468         if (skdev->skspcl_table == NULL)
4469                 return;
4470
4471         for (i = 0; i < skdev->n_special; i++) {
4472                 struct skd_special_context *skspcl;
4473
4474                 skspcl = &skdev->skspcl_table[i];
4475
4476                 if (skspcl->msg_buf != NULL) {
4477                         nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4478                         pci_free_consistent(skdev->pdev, nbytes,
4479                                             skspcl->msg_buf,
4480                                             skspcl->mb_dma_address);
4481                 }
4482
4483                 skspcl->msg_buf = NULL;
4484                 skspcl->mb_dma_address = 0;
4485
4486                 skd_free_sg_list(skdev, skspcl->req.sksg_list,
4487                                  SKD_N_SG_PER_SPECIAL,
4488                                  skspcl->req.sksg_dma_address);
4489
4490                 skspcl->req.sksg_list = NULL;
4491                 skspcl->req.sksg_dma_address = 0;
4492
4493                 kfree(skspcl->req.sg);
4494         }
4495
4496         kfree(skdev->skspcl_table);
4497         skdev->skspcl_table = NULL;
4498 }
4499
4500 static void skd_free_sksb(struct skd_device *skdev)
4501 {
4502         struct skd_special_context *skspcl;
4503         u32 nbytes;
4504
4505         skspcl = &skdev->internal_skspcl;
4506
4507         if (skspcl->data_buf != NULL) {
4508                 nbytes = SKD_N_INTERNAL_BYTES;
4509
4510                 pci_free_consistent(skdev->pdev, nbytes,
4511                                     skspcl->data_buf, skspcl->db_dma_address);
4512         }
4513
4514         skspcl->data_buf = NULL;
4515         skspcl->db_dma_address = 0;
4516
4517         if (skspcl->msg_buf != NULL) {
4518                 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4519                 pci_free_consistent(skdev->pdev, nbytes,
4520                                     skspcl->msg_buf, skspcl->mb_dma_address);
4521         }
4522
4523         skspcl->msg_buf = NULL;
4524         skspcl->mb_dma_address = 0;
4525
4526         skd_free_sg_list(skdev, skspcl->req.sksg_list, 1,
4527                          skspcl->req.sksg_dma_address);
4528
4529         skspcl->req.sksg_list = NULL;
4530         skspcl->req.sksg_dma_address = 0;
4531 }
4532
4533 static void skd_free_disk(struct skd_device *skdev)
4534 {
4535         struct gendisk *disk = skdev->disk;
4536
4537         if (disk != NULL) {
4538                 struct request_queue *q = disk->queue;
4539
4540                 if (disk->flags & GENHD_FL_UP)
4541                         del_gendisk(disk);
4542                 if (q)
4543                         blk_cleanup_queue(q);
4544                 put_disk(disk);
4545         }
4546         skdev->disk = NULL;
4547 }
4548
4549 static void skd_destruct(struct skd_device *skdev)
4550 {
4551         if (skdev == NULL)
4552                 return;
4553
4554
4555         pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
4556         skd_free_disk(skdev);
4557
4558         pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
4559         skd_free_sksb(skdev);
4560
4561         pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
4562         skd_free_skspcl(skdev);
4563
4564         pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
4565         skd_free_skreq(skdev);
4566
4567         pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
4568         skd_free_skmsg(skdev);
4569
4570         pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
4571         skd_free_skcomp(skdev);
4572
4573         pr_debug("%s:%s:%d skdev\n", skdev->name, __func__, __LINE__);
4574         kfree(skdev);
4575 }
4576
4577 /*
4578  *****************************************************************************
4579  * BLOCK DEVICE (BDEV) GLUE
4580  *****************************************************************************
4581  */
4582
4583 static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
4584 {
4585         struct skd_device *skdev;
4586         u64 capacity;
4587
4588         skdev = bdev->bd_disk->private_data;
4589
4590         pr_debug("%s:%s:%d %s: CMD[%s] getgeo device\n",
4591                  skdev->name, __func__, __LINE__,
4592                  bdev->bd_disk->disk_name, current->comm);
4593
4594         if (skdev->read_cap_is_valid) {
4595                 capacity = get_capacity(skdev->disk);
4596                 geo->heads = 64;
4597                 geo->sectors = 255;
4598                 geo->cylinders = (capacity) / (255 * 64);
4599
4600                 return 0;
4601         }
4602         return -EIO;
4603 }
4604
4605 static int skd_bdev_attach(struct device *parent, struct skd_device *skdev)
4606 {
4607         pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__);
4608         device_add_disk(parent, skdev->disk);
4609         return 0;
4610 }
4611
4612 static const struct block_device_operations skd_blockdev_ops = {
4613         .owner          = THIS_MODULE,
4614         .ioctl          = skd_bdev_ioctl,
4615         .getgeo         = skd_bdev_getgeo,
4616 };
4617
4618
4619 /*
4620  *****************************************************************************
4621  * PCIe DRIVER GLUE
4622  *****************************************************************************
4623  */
4624
4625 static const struct pci_device_id skd_pci_tbl[] = {
4626         { PCI_VENDOR_ID_STEC, PCI_DEVICE_ID_S1120,
4627           PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
4628         { 0 }                     /* terminate list */
4629 };
4630
4631 MODULE_DEVICE_TABLE(pci, skd_pci_tbl);
4632
4633 static char *skd_pci_info(struct skd_device *skdev, char *str)
4634 {
4635         int pcie_reg;
4636
4637         strcpy(str, "PCIe (");
4638         pcie_reg = pci_find_capability(skdev->pdev, PCI_CAP_ID_EXP);
4639
4640         if (pcie_reg) {
4641
4642                 char lwstr[6];
4643                 uint16_t pcie_lstat, lspeed, lwidth;
4644
4645                 pcie_reg += 0x12;
4646                 pci_read_config_word(skdev->pdev, pcie_reg, &pcie_lstat);
4647                 lspeed = pcie_lstat & (0xF);
4648                 lwidth = (pcie_lstat & 0x3F0) >> 4;
4649
4650                 if (lspeed == 1)
4651                         strcat(str, "2.5GT/s ");
4652                 else if (lspeed == 2)
4653                         strcat(str, "5.0GT/s ");
4654                 else
4655                         strcat(str, "<unknown> ");
4656                 snprintf(lwstr, sizeof(lwstr), "%dX)", lwidth);
4657                 strcat(str, lwstr);
4658         }
4659         return str;
4660 }
4661
4662 static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4663 {
4664         int i;
4665         int rc = 0;
4666         char pci_str[32];
4667         struct skd_device *skdev;
4668
4669         pr_info("STEC s1120 Driver(%s) version %s-b%s\n",
4670                DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
4671         pr_info("(skd?:??:[%s]): vendor=%04X device=%04x\n",
4672                pci_name(pdev), pdev->vendor, pdev->device);
4673
4674         rc = pci_enable_device(pdev);
4675         if (rc)
4676                 return rc;
4677         rc = pci_request_regions(pdev, DRV_NAME);
4678         if (rc)
4679                 goto err_out;
4680         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
4681         if (!rc) {
4682                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
4683
4684                         pr_err("(%s): consistent DMA mask error %d\n",
4685                                pci_name(pdev), rc);
4686                 }
4687         } else {
4688                 (rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)));
4689                 if (rc) {
4690
4691                         pr_err("(%s): DMA mask error %d\n",
4692                                pci_name(pdev), rc);
4693                         goto err_out_regions;
4694                 }
4695         }
4696
4697         if (!skd_major) {
4698                 rc = register_blkdev(0, DRV_NAME);
4699                 if (rc < 0)
4700                         goto err_out_regions;
4701                 BUG_ON(!rc);
4702                 skd_major = rc;
4703         }
4704
4705         skdev = skd_construct(pdev);
4706         if (skdev == NULL) {
4707                 rc = -ENOMEM;
4708                 goto err_out_regions;
4709         }
4710
4711         skd_pci_info(skdev, pci_str);
4712         pr_info("(%s): %s 64bit\n", skd_name(skdev), pci_str);
4713
4714         pci_set_master(pdev);
4715         rc = pci_enable_pcie_error_reporting(pdev);
4716         if (rc) {
4717                 pr_err(
4718                        "(%s): bad enable of PCIe error reporting rc=%d\n",
4719                        skd_name(skdev), rc);
4720                 skdev->pcie_error_reporting_is_enabled = 0;
4721         } else
4722                 skdev->pcie_error_reporting_is_enabled = 1;
4723
4724
4725         pci_set_drvdata(pdev, skdev);
4726
4727         for (i = 0; i < SKD_MAX_BARS; i++) {
4728                 skdev->mem_phys[i] = pci_resource_start(pdev, i);
4729                 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
4730                 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
4731                                             skdev->mem_size[i]);
4732                 if (!skdev->mem_map[i]) {
4733                         pr_err("(%s): Unable to map adapter memory!\n",
4734                                skd_name(skdev));
4735                         rc = -ENODEV;
4736                         goto err_out_iounmap;
4737                 }
4738                 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
4739                          skdev->name, __func__, __LINE__,
4740                          skdev->mem_map[i],
4741                          (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
4742         }
4743
4744         rc = skd_acquire_irq(skdev);
4745         if (rc) {
4746                 pr_err("(%s): interrupt resource error %d\n",
4747                        skd_name(skdev), rc);
4748                 goto err_out_iounmap;
4749         }
4750
4751         rc = skd_start_timer(skdev);
4752         if (rc)
4753                 goto err_out_timer;
4754
4755         init_waitqueue_head(&skdev->waitq);
4756
4757         skd_start_device(skdev);
4758
4759         rc = wait_event_interruptible_timeout(skdev->waitq,
4760                                               (skdev->gendisk_on),
4761                                               (SKD_START_WAIT_SECONDS * HZ));
4762         if (skdev->gendisk_on > 0) {
4763                 /* device came on-line after reset */
4764                 skd_bdev_attach(&pdev->dev, skdev);
4765                 rc = 0;
4766         } else {
4767                 /* we timed out, something is wrong with the device,
4768                    don't add the disk structure */
4769                 pr_err(
4770                        "(%s): error: waiting for s1120 timed out %d!\n",
4771                        skd_name(skdev), rc);
4772                 /* in case of no error; we timeout with ENXIO */
4773                 if (!rc)
4774                         rc = -ENXIO;
4775                 goto err_out_timer;
4776         }
4777
4778
4779 #ifdef SKD_VMK_POLL_HANDLER
4780         if (skdev->irq_type == SKD_IRQ_MSIX) {
4781                 /* MSIX completion handler is being used for coredump */
4782                 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4783                                                   skdev->msix_entries[5].vector,
4784                                                   skd_comp_q, skdev);
4785         } else {
4786                 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
4787                                                   skdev->pdev->irq, skd_isr,
4788                                                   skdev);
4789         }
4790 #endif  /* SKD_VMK_POLL_HANDLER */
4791
4792         return rc;
4793
4794 err_out_timer:
4795         skd_stop_device(skdev);
4796         skd_release_irq(skdev);
4797
4798 err_out_iounmap:
4799         for (i = 0; i < SKD_MAX_BARS; i++)
4800                 if (skdev->mem_map[i])
4801                         iounmap(skdev->mem_map[i]);
4802
4803         if (skdev->pcie_error_reporting_is_enabled)
4804                 pci_disable_pcie_error_reporting(pdev);
4805
4806         skd_destruct(skdev);
4807
4808 err_out_regions:
4809         pci_release_regions(pdev);
4810
4811 err_out:
4812         pci_disable_device(pdev);
4813         pci_set_drvdata(pdev, NULL);
4814         return rc;
4815 }
4816
4817 static void skd_pci_remove(struct pci_dev *pdev)
4818 {
4819         int i;
4820         struct skd_device *skdev;
4821
4822         skdev = pci_get_drvdata(pdev);
4823         if (!skdev) {
4824                 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4825                 return;
4826         }
4827         skd_stop_device(skdev);
4828         skd_release_irq(skdev);
4829
4830         for (i = 0; i < SKD_MAX_BARS; i++)
4831                 if (skdev->mem_map[i])
4832                         iounmap((u32 *)skdev->mem_map[i]);
4833
4834         if (skdev->pcie_error_reporting_is_enabled)
4835                 pci_disable_pcie_error_reporting(pdev);
4836
4837         skd_destruct(skdev);
4838
4839         pci_release_regions(pdev);
4840         pci_disable_device(pdev);
4841         pci_set_drvdata(pdev, NULL);
4842
4843         return;
4844 }
4845
4846 static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
4847 {
4848         int i;
4849         struct skd_device *skdev;
4850
4851         skdev = pci_get_drvdata(pdev);
4852         if (!skdev) {
4853                 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4854                 return -EIO;
4855         }
4856
4857         skd_stop_device(skdev);
4858
4859         skd_release_irq(skdev);
4860
4861         for (i = 0; i < SKD_MAX_BARS; i++)
4862                 if (skdev->mem_map[i])
4863                         iounmap((u32 *)skdev->mem_map[i]);
4864
4865         if (skdev->pcie_error_reporting_is_enabled)
4866                 pci_disable_pcie_error_reporting(pdev);
4867
4868         pci_release_regions(pdev);
4869         pci_save_state(pdev);
4870         pci_disable_device(pdev);
4871         pci_set_power_state(pdev, pci_choose_state(pdev, state));
4872         return 0;
4873 }
4874
4875 static int skd_pci_resume(struct pci_dev *pdev)
4876 {
4877         int i;
4878         int rc = 0;
4879         struct skd_device *skdev;
4880
4881         skdev = pci_get_drvdata(pdev);
4882         if (!skdev) {
4883                 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4884                 return -1;
4885         }
4886
4887         pci_set_power_state(pdev, PCI_D0);
4888         pci_enable_wake(pdev, PCI_D0, 0);
4889         pci_restore_state(pdev);
4890
4891         rc = pci_enable_device(pdev);
4892         if (rc)
4893                 return rc;
4894         rc = pci_request_regions(pdev, DRV_NAME);
4895         if (rc)
4896                 goto err_out;
4897         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
4898         if (!rc) {
4899                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
4900
4901                         pr_err("(%s): consistent DMA mask error %d\n",
4902                                pci_name(pdev), rc);
4903                 }
4904         } else {
4905                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
4906                 if (rc) {
4907
4908                         pr_err("(%s): DMA mask error %d\n",
4909                                pci_name(pdev), rc);
4910                         goto err_out_regions;
4911                 }
4912         }
4913
4914         pci_set_master(pdev);
4915         rc = pci_enable_pcie_error_reporting(pdev);
4916         if (rc) {
4917                 pr_err("(%s): bad enable of PCIe error reporting rc=%d\n",
4918                        skdev->name, rc);
4919                 skdev->pcie_error_reporting_is_enabled = 0;
4920         } else
4921                 skdev->pcie_error_reporting_is_enabled = 1;
4922
4923         for (i = 0; i < SKD_MAX_BARS; i++) {
4924
4925                 skdev->mem_phys[i] = pci_resource_start(pdev, i);
4926                 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
4927                 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
4928                                             skdev->mem_size[i]);
4929                 if (!skdev->mem_map[i]) {
4930                         pr_err("(%s): Unable to map adapter memory!\n",
4931                                skd_name(skdev));
4932                         rc = -ENODEV;
4933                         goto err_out_iounmap;
4934                 }
4935                 pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
4936                          skdev->name, __func__, __LINE__,
4937                          skdev->mem_map[i],
4938                          (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
4939         }
4940         rc = skd_acquire_irq(skdev);
4941         if (rc) {
4942
4943                 pr_err("(%s): interrupt resource error %d\n",
4944                        pci_name(pdev), rc);
4945                 goto err_out_iounmap;
4946         }
4947
4948         rc = skd_start_timer(skdev);
4949         if (rc)
4950                 goto err_out_timer;
4951
4952         init_waitqueue_head(&skdev->waitq);
4953
4954         skd_start_device(skdev);
4955
4956         return rc;
4957
4958 err_out_timer:
4959         skd_stop_device(skdev);
4960         skd_release_irq(skdev);
4961
4962 err_out_iounmap:
4963         for (i = 0; i < SKD_MAX_BARS; i++)
4964                 if (skdev->mem_map[i])
4965                         iounmap(skdev->mem_map[i]);
4966
4967         if (skdev->pcie_error_reporting_is_enabled)
4968                 pci_disable_pcie_error_reporting(pdev);
4969
4970 err_out_regions:
4971         pci_release_regions(pdev);
4972
4973 err_out:
4974         pci_disable_device(pdev);
4975         return rc;
4976 }
4977
4978 static void skd_pci_shutdown(struct pci_dev *pdev)
4979 {
4980         struct skd_device *skdev;
4981
4982         pr_err("skd_pci_shutdown called\n");
4983
4984         skdev = pci_get_drvdata(pdev);
4985         if (!skdev) {
4986                 pr_err("%s: no device data for PCI\n", pci_name(pdev));
4987                 return;
4988         }
4989
4990         pr_err("%s: calling stop\n", skd_name(skdev));
4991         skd_stop_device(skdev);
4992 }
4993
4994 static struct pci_driver skd_driver = {
4995         .name           = DRV_NAME,
4996         .id_table       = skd_pci_tbl,
4997         .probe          = skd_pci_probe,
4998         .remove         = skd_pci_remove,
4999         .suspend        = skd_pci_suspend,
5000         .resume         = skd_pci_resume,
5001         .shutdown       = skd_pci_shutdown,
5002 };
5003
5004 /*
5005  *****************************************************************************
5006  * LOGGING SUPPORT
5007  *****************************************************************************
5008  */
5009
5010 static const char *skd_name(struct skd_device *skdev)
5011 {
5012         memset(skdev->id_str, 0, sizeof(skdev->id_str));
5013
5014         if (skdev->inquiry_is_valid)
5015                 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:%s:[%s]",
5016                          skdev->name, skdev->inq_serial_num,
5017                          pci_name(skdev->pdev));
5018         else
5019                 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:??:[%s]",
5020                          skdev->name, pci_name(skdev->pdev));
5021
5022         return skdev->id_str;
5023 }
5024
5025 const char *skd_drive_state_to_str(int state)
5026 {
5027         switch (state) {
5028         case FIT_SR_DRIVE_OFFLINE:
5029                 return "OFFLINE";
5030         case FIT_SR_DRIVE_INIT:
5031                 return "INIT";
5032         case FIT_SR_DRIVE_ONLINE:
5033                 return "ONLINE";
5034         case FIT_SR_DRIVE_BUSY:
5035                 return "BUSY";
5036         case FIT_SR_DRIVE_FAULT:
5037                 return "FAULT";
5038         case FIT_SR_DRIVE_DEGRADED:
5039                 return "DEGRADED";
5040         case FIT_SR_PCIE_LINK_DOWN:
5041                 return "INK_DOWN";
5042         case FIT_SR_DRIVE_SOFT_RESET:
5043                 return "SOFT_RESET";
5044         case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
5045                 return "NEED_FW";
5046         case FIT_SR_DRIVE_INIT_FAULT:
5047                 return "INIT_FAULT";
5048         case FIT_SR_DRIVE_BUSY_SANITIZE:
5049                 return "BUSY_SANITIZE";
5050         case FIT_SR_DRIVE_BUSY_ERASE:
5051                 return "BUSY_ERASE";
5052         case FIT_SR_DRIVE_FW_BOOTING:
5053                 return "FW_BOOTING";
5054         default:
5055                 return "???";
5056         }
5057 }
5058
5059 const char *skd_skdev_state_to_str(enum skd_drvr_state state)
5060 {
5061         switch (state) {
5062         case SKD_DRVR_STATE_LOAD:
5063                 return "LOAD";
5064         case SKD_DRVR_STATE_IDLE:
5065                 return "IDLE";
5066         case SKD_DRVR_STATE_BUSY:
5067                 return "BUSY";
5068         case SKD_DRVR_STATE_STARTING:
5069                 return "STARTING";
5070         case SKD_DRVR_STATE_ONLINE:
5071                 return "ONLINE";
5072         case SKD_DRVR_STATE_PAUSING:
5073                 return "PAUSING";
5074         case SKD_DRVR_STATE_PAUSED:
5075                 return "PAUSED";
5076         case SKD_DRVR_STATE_DRAINING_TIMEOUT:
5077                 return "DRAINING_TIMEOUT";
5078         case SKD_DRVR_STATE_RESTARTING:
5079                 return "RESTARTING";
5080         case SKD_DRVR_STATE_RESUMING:
5081                 return "RESUMING";
5082         case SKD_DRVR_STATE_STOPPING:
5083                 return "STOPPING";
5084         case SKD_DRVR_STATE_SYNCING:
5085                 return "SYNCING";
5086         case SKD_DRVR_STATE_FAULT:
5087                 return "FAULT";
5088         case SKD_DRVR_STATE_DISAPPEARED:
5089                 return "DISAPPEARED";
5090         case SKD_DRVR_STATE_BUSY_ERASE:
5091                 return "BUSY_ERASE";
5092         case SKD_DRVR_STATE_BUSY_SANITIZE:
5093                 return "BUSY_SANITIZE";
5094         case SKD_DRVR_STATE_BUSY_IMMINENT:
5095                 return "BUSY_IMMINENT";
5096         case SKD_DRVR_STATE_WAIT_BOOT:
5097                 return "WAIT_BOOT";
5098
5099         default:
5100                 return "???";
5101         }
5102 }
5103
5104 static const char *skd_skmsg_state_to_str(enum skd_fit_msg_state state)
5105 {
5106         switch (state) {
5107         case SKD_MSG_STATE_IDLE:
5108                 return "IDLE";
5109         case SKD_MSG_STATE_BUSY:
5110                 return "BUSY";
5111         default:
5112                 return "???";
5113         }
5114 }
5115
5116 static const char *skd_skreq_state_to_str(enum skd_req_state state)
5117 {
5118         switch (state) {
5119         case SKD_REQ_STATE_IDLE:
5120                 return "IDLE";
5121         case SKD_REQ_STATE_SETUP:
5122                 return "SETUP";
5123         case SKD_REQ_STATE_BUSY:
5124                 return "BUSY";
5125         case SKD_REQ_STATE_COMPLETED:
5126                 return "COMPLETED";
5127         case SKD_REQ_STATE_TIMEOUT:
5128                 return "TIMEOUT";
5129         case SKD_REQ_STATE_ABORTED:
5130                 return "ABORTED";
5131         default:
5132                 return "???";
5133         }
5134 }
5135
5136 static void skd_log_skdev(struct skd_device *skdev, const char *event)
5137 {
5138         pr_debug("%s:%s:%d (%s) skdev=%p event='%s'\n",
5139                  skdev->name, __func__, __LINE__, skdev->name, skdev, event);
5140         pr_debug("%s:%s:%d   drive_state=%s(%d) driver_state=%s(%d)\n",
5141                  skdev->name, __func__, __LINE__,
5142                  skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
5143                  skd_skdev_state_to_str(skdev->state), skdev->state);
5144         pr_debug("%s:%s:%d   busy=%d limit=%d dev=%d lowat=%d\n",
5145                  skdev->name, __func__, __LINE__,
5146                  skdev->in_flight, skdev->cur_max_queue_depth,
5147                  skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
5148         pr_debug("%s:%s:%d   timestamp=0x%x cycle=%d cycle_ix=%d\n",
5149                  skdev->name, __func__, __LINE__,
5150                  skdev->timeout_stamp, skdev->skcomp_cycle, skdev->skcomp_ix);
5151 }
5152
5153 static void skd_log_skmsg(struct skd_device *skdev,
5154                           struct skd_fitmsg_context *skmsg, const char *event)
5155 {
5156         pr_debug("%s:%s:%d (%s) skmsg=%p event='%s'\n",
5157                  skdev->name, __func__, __LINE__, skdev->name, skmsg, event);
5158         pr_debug("%s:%s:%d   state=%s(%d) id=0x%04x length=%d\n",
5159                  skdev->name, __func__, __LINE__,
5160                  skd_skmsg_state_to_str(skmsg->state), skmsg->state,
5161                  skmsg->id, skmsg->length);
5162 }
5163
5164 static void skd_log_skreq(struct skd_device *skdev,
5165                           struct skd_request_context *skreq, const char *event)
5166 {
5167         pr_debug("%s:%s:%d (%s) skreq=%p event='%s'\n",
5168                  skdev->name, __func__, __LINE__, skdev->name, skreq, event);
5169         pr_debug("%s:%s:%d   state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
5170                  skdev->name, __func__, __LINE__,
5171                  skd_skreq_state_to_str(skreq->state), skreq->state,
5172                  skreq->id, skreq->fitmsg_id);
5173         pr_debug("%s:%s:%d   timo=0x%x sg_dir=%d n_sg=%d\n",
5174                  skdev->name, __func__, __LINE__,
5175                  skreq->timeout_stamp, skreq->sg_data_dir, skreq->n_sg);
5176
5177         if (skreq->req != NULL) {
5178                 struct request *req = skreq->req;
5179                 u32 lba = (u32)blk_rq_pos(req);
5180                 u32 count = blk_rq_sectors(req);
5181
5182                 pr_debug("%s:%s:%d "
5183                          "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
5184                          skdev->name, __func__, __LINE__,
5185                          req, lba, lba, count, count,
5186                          (int)rq_data_dir(req));
5187         } else
5188                 pr_debug("%s:%s:%d req=NULL\n",
5189                          skdev->name, __func__, __LINE__);
5190 }
5191
5192 /*
5193  *****************************************************************************
5194  * MODULE GLUE
5195  *****************************************************************************
5196  */
5197
5198 static int __init skd_init(void)
5199 {
5200         pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
5201
5202         switch (skd_isr_type) {
5203         case SKD_IRQ_LEGACY:
5204         case SKD_IRQ_MSI:
5205         case SKD_IRQ_MSIX:
5206                 break;
5207         default:
5208                 pr_err(PFX "skd_isr_type %d invalid, re-set to %d\n",
5209                        skd_isr_type, SKD_IRQ_DEFAULT);
5210                 skd_isr_type = SKD_IRQ_DEFAULT;
5211         }
5212
5213         if (skd_max_queue_depth < 1 ||
5214             skd_max_queue_depth > SKD_MAX_QUEUE_DEPTH) {
5215                 pr_err(PFX "skd_max_queue_depth %d invalid, re-set to %d\n",
5216                        skd_max_queue_depth, SKD_MAX_QUEUE_DEPTH_DEFAULT);
5217                 skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
5218         }
5219
5220         if (skd_max_req_per_msg < 1 || skd_max_req_per_msg > 14) {
5221                 pr_err(PFX "skd_max_req_per_msg %d invalid, re-set to %d\n",
5222                        skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
5223                 skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
5224         }
5225
5226         if (skd_sgs_per_request < 1 || skd_sgs_per_request > 4096) {
5227                 pr_err(PFX "skd_sg_per_request %d invalid, re-set to %d\n",
5228                        skd_sgs_per_request, SKD_N_SG_PER_REQ_DEFAULT);
5229                 skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
5230         }
5231
5232         if (skd_dbg_level < 0 || skd_dbg_level > 2) {
5233                 pr_err(PFX "skd_dbg_level %d invalid, re-set to %d\n",
5234                        skd_dbg_level, 0);
5235                 skd_dbg_level = 0;
5236         }
5237
5238         if (skd_isr_comp_limit < 0) {
5239                 pr_err(PFX "skd_isr_comp_limit %d invalid, set to %d\n",
5240                        skd_isr_comp_limit, 0);
5241                 skd_isr_comp_limit = 0;
5242         }
5243
5244         if (skd_max_pass_thru < 1 || skd_max_pass_thru > 50) {
5245                 pr_err(PFX "skd_max_pass_thru %d invalid, re-set to %d\n",
5246                        skd_max_pass_thru, SKD_N_SPECIAL_CONTEXT);
5247                 skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
5248         }
5249
5250         return pci_register_driver(&skd_driver);
5251 }
5252
5253 static void __exit skd_exit(void)
5254 {
5255         pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
5256
5257         pci_unregister_driver(&skd_driver);
5258
5259         if (skd_major)
5260                 unregister_blkdev(skd_major, DRV_NAME);
5261 }
5262
5263 module_init(skd_init);
5264 module_exit(skd_exit);