drivers/block/rsxx/core.c

   1 /*
   2 * Filename: core.c
   3 *
   4 *
   5 * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
   6 *       Philip Kelleher <pjk1939@linux.vnet.ibm.com>
   7 *
   8 * (C) Copyright 2013 IBM Corporation
   9 *
  10 * This program is free software; you can redistribute it and/or
  11 * modify it under the terms of the GNU General Public License as
  12 * published by the Free Software Foundation; either version 2 of the
  13 * License, or (at your option) any later version.
  14 *
  15 * This program is distributed in the hope that it will be useful, but
  16 * WITHOUT ANY WARRANTY; without even the implied warranty of
  17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18 * General Public License for more details.
  19 *
  20 * You should have received a copy of the GNU General Public License
  21 * along with this program; if not, write to the Free Software Foundation,
  22 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  23 */
  24
  25 #include <linux/kernel.h>
  26 #include <linux/init.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/module.h>
  29 #include <linux/pci.h>
  30 #include <linux/reboot.h>
  31 #include <linux/slab.h>
  32 #include <linux/bitops.h>
  33 #include <linux/delay.h>
  34 #include <linux/debugfs.h>
  35 #include <linux/seq_file.h>
  36
  37 #include <linux/genhd.h>
  38 #include <linux/idr.h>
  39
  40 #include "rsxx_priv.h"
  41 #include "rsxx_cfg.h"
  42
  43 #define NO_LEGACY 0
  44 #define SYNC_START_TIMEOUT (10 * 60) /* 10 minutes */
  45
  46 MODULE_DESCRIPTION("IBM Flash Adapter 900GB Full Height Device Driver");
  47 MODULE_AUTHOR("Joshua Morris/Philip Kelleher, IBM");
  48 MODULE_LICENSE("GPL");
  49 MODULE_VERSION(DRIVER_VERSION);
  50
  51 static unsigned int force_legacy = NO_LEGACY;
  52 module_param(force_legacy, uint, 0444);
  53 MODULE_PARM_DESC(force_legacy, "Force the use of legacy type PCI interrupts");
  54
  55 static unsigned int sync_start = 1;
  56 module_param(sync_start, uint, 0444);
  57 MODULE_PARM_DESC(sync_start, "On by Default: Driver load will not complete "
  58                              "until the card startup has completed.");
  59
  60 static DEFINE_IDA(rsxx_disk_ida);
  61 static DEFINE_SPINLOCK(rsxx_ida_lock);
  62
  63 /* --------------------Debugfs Setup ------------------- */
  64
  65 struct rsxx_cram {
  66         u32 f_pos;
  67         u32 offset;
  68         void *i_private;
  69 };
  70
  71 static int rsxx_attr_pci_regs_show(struct seq_file *m, void *p)
  72 {
  73         struct rsxx_cardinfo *card = m->private;
  74
  75         seq_printf(m, "HWID             0x%08x\n",
  76                                         ioread32(card->regmap + HWID));
  77         seq_printf(m, "SCRATCH          0x%08x\n",
  78                                         ioread32(card->regmap + SCRATCH));
  79         seq_printf(m, "IER              0x%08x\n",
  80                                         ioread32(card->regmap + IER));
  81         seq_printf(m, "IPR              0x%08x\n",
  82                                         ioread32(card->regmap + IPR));
  83         seq_printf(m, "CREG_CMD         0x%08x\n",
  84                                         ioread32(card->regmap + CREG_CMD));
  85         seq_printf(m, "CREG_ADD         0x%08x\n",
  86                                         ioread32(card->regmap + CREG_ADD));
  87         seq_printf(m, "CREG_CNT         0x%08x\n",
  88                                         ioread32(card->regmap + CREG_CNT));
  89         seq_printf(m, "CREG_STAT        0x%08x\n",
  90                                         ioread32(card->regmap + CREG_STAT));
  91         seq_printf(m, "CREG_DATA0       0x%08x\n",
  92                                         ioread32(card->regmap + CREG_DATA0));
  93         seq_printf(m, "CREG_DATA1       0x%08x\n",
  94                                         ioread32(card->regmap + CREG_DATA1));
  95         seq_printf(m, "CREG_DATA2       0x%08x\n",
  96                                         ioread32(card->regmap + CREG_DATA2));
  97         seq_printf(m, "CREG_DATA3       0x%08x\n",
  98                                         ioread32(card->regmap + CREG_DATA3));
  99         seq_printf(m, "CREG_DATA4       0x%08x\n",
 100                                         ioread32(card->regmap + CREG_DATA4));
 101         seq_printf(m, "CREG_DATA5       0x%08x\n",
 102                                         ioread32(card->regmap + CREG_DATA5));
 103         seq_printf(m, "CREG_DATA6       0x%08x\n",
 104                                         ioread32(card->regmap + CREG_DATA6));
 105         seq_printf(m, "CREG_DATA7       0x%08x\n",
 106                                         ioread32(card->regmap + CREG_DATA7));
 107         seq_printf(m, "INTR_COAL        0x%08x\n",
 108                                         ioread32(card->regmap + INTR_COAL));
 109         seq_printf(m, "HW_ERROR         0x%08x\n",
 110                                         ioread32(card->regmap + HW_ERROR));
 111         seq_printf(m, "DEBUG0           0x%08x\n",
 112                                         ioread32(card->regmap + PCI_DEBUG0));
 113         seq_printf(m, "DEBUG1           0x%08x\n",
 114                                         ioread32(card->regmap + PCI_DEBUG1));
 115         seq_printf(m, "DEBUG2           0x%08x\n",
 116                                         ioread32(card->regmap + PCI_DEBUG2));
 117         seq_printf(m, "DEBUG3           0x%08x\n",
 118                                         ioread32(card->regmap + PCI_DEBUG3));
 119         seq_printf(m, "DEBUG4           0x%08x\n",
 120                                         ioread32(card->regmap + PCI_DEBUG4));
 121         seq_printf(m, "DEBUG5           0x%08x\n",
 122                                         ioread32(card->regmap + PCI_DEBUG5));
 123         seq_printf(m, "DEBUG6           0x%08x\n",
 124                                         ioread32(card->regmap + PCI_DEBUG6));
 125         seq_printf(m, "DEBUG7           0x%08x\n",
 126                                         ioread32(card->regmap + PCI_DEBUG7));
 127         seq_printf(m, "RECONFIG         0x%08x\n",
 128                                         ioread32(card->regmap + PCI_RECONFIG));
 129
 130         return 0;
 131 }
 132
 133 static int rsxx_attr_stats_show(struct seq_file *m, void *p)
 134 {
 135         struct rsxx_cardinfo *card = m->private;
 136         int i;
 137
 138         for (i = 0; i < card->n_targets; i++) {
 139                 seq_printf(m, "Ctrl %d CRC Errors       = %d\n",
 140                                 i, card->ctrl[i].stats.crc_errors);
 141                 seq_printf(m, "Ctrl %d Hard Errors      = %d\n",
 142                                 i, card->ctrl[i].stats.hard_errors);
 143                 seq_printf(m, "Ctrl %d Soft Errors      = %d\n",
 144                                 i, card->ctrl[i].stats.soft_errors);
 145                 seq_printf(m, "Ctrl %d Writes Issued    = %d\n",
 146                                 i, card->ctrl[i].stats.writes_issued);
 147                 seq_printf(m, "Ctrl %d Writes Failed    = %d\n",
 148                                 i, card->ctrl[i].stats.writes_failed);
 149                 seq_printf(m, "Ctrl %d Reads Issued     = %d\n",
 150                                 i, card->ctrl[i].stats.reads_issued);
 151                 seq_printf(m, "Ctrl %d Reads Failed     = %d\n",
 152                                 i, card->ctrl[i].stats.reads_failed);
 153                 seq_printf(m, "Ctrl %d Reads Retried    = %d\n",
 154                                 i, card->ctrl[i].stats.reads_retried);
 155                 seq_printf(m, "Ctrl %d Discards Issued  = %d\n",
 156                                 i, card->ctrl[i].stats.discards_issued);
 157                 seq_printf(m, "Ctrl %d Discards Failed  = %d\n",
 158                                 i, card->ctrl[i].stats.discards_failed);
 159                 seq_printf(m, "Ctrl %d DMA SW Errors    = %d\n",
 160                                 i, card->ctrl[i].stats.dma_sw_err);
 161                 seq_printf(m, "Ctrl %d DMA HW Faults    = %d\n",
 162                                 i, card->ctrl[i].stats.dma_hw_fault);
 163                 seq_printf(m, "Ctrl %d DMAs Cancelled   = %d\n",
 164                                 i, card->ctrl[i].stats.dma_cancelled);
 165                 seq_printf(m, "Ctrl %d SW Queue Depth   = %d\n",
 166                                 i, card->ctrl[i].stats.sw_q_depth);
 167                 seq_printf(m, "Ctrl %d HW Queue Depth   = %d\n",
 168                         i, atomic_read(&card->ctrl[i].stats.hw_q_depth));
 169         }
 170
 171         return 0;
 172 }
 173
 174 static int rsxx_attr_stats_open(struct inode *inode, struct file *file)
 175 {
 176         return single_open(file, rsxx_attr_stats_show, inode->i_private);
 177 }
 178
 179 static int rsxx_attr_pci_regs_open(struct inode *inode, struct file *file)
 180 {
 181         return single_open(file, rsxx_attr_pci_regs_show, inode->i_private);
 182 }
 183
 184 static ssize_t rsxx_cram_read(struct file *fp, char __user *ubuf,
 185                               size_t cnt, loff_t *ppos)
 186 {
 187         struct rsxx_cram *info = fp->private_data;
 188         struct rsxx_cardinfo *card = info->i_private;
 189         char *buf;
 190         int st;
 191
 192         buf = kzalloc(sizeof(*buf) * cnt, GFP_KERNEL);
 193         if (!buf)
 194                 return -ENOMEM;
 195
 196         info->f_pos = (u32)*ppos + info->offset;
 197
 198         st = rsxx_creg_read(card, CREG_ADD_CRAM + info->f_pos, cnt, buf, 1);
 199         if (st)
 200                 return st;
 201
 202         st = copy_to_user(ubuf, buf, cnt);
 203         if (st)
 204                 return st;
 205
 206         info->offset += cnt;
 207
 208         kfree(buf);
 209
 210         return cnt;
 211 }
 212
 213 static ssize_t rsxx_cram_write(struct file *fp, const char __user *ubuf,
 214                                size_t cnt, loff_t *ppos)
 215 {
 216         struct rsxx_cram *info = fp->private_data;
 217         struct rsxx_cardinfo *card = info->i_private;
 218         char *buf;
 219         int st;
 220
 221         buf = kzalloc(sizeof(*buf) * cnt, GFP_KERNEL);
 222         if (!buf)
 223                 return -ENOMEM;
 224
 225         st = copy_from_user(buf, ubuf, cnt);
 226         if (st)
 227                 return st;
 228
 229         info->f_pos = (u32)*ppos + info->offset;
 230
 231         st = rsxx_creg_write(card, CREG_ADD_CRAM + info->f_pos, cnt, buf, 1);
 232         if (st)
 233                 return st;
 234
 235         info->offset += cnt;
 236
 237         kfree(buf);
 238
 239         return cnt;
 240 }
 241
 242 static int rsxx_cram_open(struct inode *inode, struct file *file)
 243 {
 244         struct rsxx_cram *info = kzalloc(sizeof(*info), GFP_KERNEL);
 245         if (!info)
 246                 return -ENOMEM;
 247
 248         info->i_private = inode->i_private;
 249         info->f_pos = file->f_pos;
 250         file->private_data = info;
 251
 252         return 0;
 253 }
 254
 255 static int rsxx_cram_release(struct inode *inode, struct file *file)
 256 {
 257         struct rsxx_cram *info = file->private_data;
 258
 259         if (!info)
 260                 return 0;
 261
 262         kfree(info);
 263         file->private_data = NULL;
 264
 265         return 0;
 266 }
 267
 268 static const struct file_operations debugfs_cram_fops = {
 269         .owner          = THIS_MODULE,
 270         .open           = rsxx_cram_open,
 271         .read           = rsxx_cram_read,
 272         .write          = rsxx_cram_write,
 273         .release        = rsxx_cram_release,
 274 };
 275
 276 static const struct file_operations debugfs_stats_fops = {
 277         .owner          = THIS_MODULE,
 278         .open           = rsxx_attr_stats_open,
 279         .read           = seq_read,
 280         .llseek         = seq_lseek,
 281         .release        = single_release,
 282 };
 283
 284 static const struct file_operations debugfs_pci_regs_fops = {
 285         .owner          = THIS_MODULE,
 286         .open           = rsxx_attr_pci_regs_open,
 287         .read           = seq_read,
 288         .llseek         = seq_lseek,
 289         .release        = single_release,
 290 };
 291
 292 static void rsxx_debugfs_dev_new(struct rsxx_cardinfo *card)
 293 {
 294         struct dentry *debugfs_stats;
 295         struct dentry *debugfs_pci_regs;
 296         struct dentry *debugfs_cram;
 297
 298         card->debugfs_dir = debugfs_create_dir(card->gendisk->disk_name, NULL);
 299         if (IS_ERR_OR_NULL(card->debugfs_dir))
 300                 goto failed_debugfs_dir;
 301
 302         debugfs_stats = debugfs_create_file("stats", S_IRUGO,
 303                                             card->debugfs_dir, card,
 304                                             &debugfs_stats_fops);
 305         if (IS_ERR_OR_NULL(debugfs_stats))
 306                 goto failed_debugfs_stats;
 307
 308         debugfs_pci_regs = debugfs_create_file("pci_regs", S_IRUGO,
 309                                                card->debugfs_dir, card,
 310                                                &debugfs_pci_regs_fops);
 311         if (IS_ERR_OR_NULL(debugfs_pci_regs))
 312                 goto failed_debugfs_pci_regs;
 313
 314         debugfs_cram = debugfs_create_file("cram", S_IRUGO | S_IWUSR,
 315                                            card->debugfs_dir, card,
 316                                            &debugfs_cram_fops);
 317         if (IS_ERR_OR_NULL(debugfs_cram))
 318                 goto failed_debugfs_cram;
 319
 320         return;
 321 failed_debugfs_cram:
 322         debugfs_remove(debugfs_pci_regs);
 323 failed_debugfs_pci_regs:
 324         debugfs_remove(debugfs_stats);
 325 failed_debugfs_stats:
 326         debugfs_remove(card->debugfs_dir);
 327 failed_debugfs_dir:
 328         card->debugfs_dir = NULL;
 329 }
 330
 331 /*----------------- Interrupt Control & Handling -------------------*/
 332
 333 static void rsxx_mask_interrupts(struct rsxx_cardinfo *card)
 334 {
 335         card->isr_mask = 0;
 336         card->ier_mask = 0;
 337 }
 338
 339 static void __enable_intr(unsigned int *mask, unsigned int intr)
 340 {
 341         *mask |= intr;
 342 }
 343
 344 static void __disable_intr(unsigned int *mask, unsigned int intr)
 345 {
 346         *mask &= ~intr;
 347 }
 348
 349 /*
 350  * NOTE: Disabling the IER will disable the hardware interrupt.
 351  * Disabling the ISR will disable the software handling of the ISR bit.
 352  *
 353  * Enable/Disable interrupt functions assume the card->irq_lock
 354  * is held by the caller.
 355  */
 356 void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr)
 357 {
 358         if (unlikely(card->halt) ||
 359             unlikely(card->eeh_state))
 360                 return;
 361
 362         __enable_intr(&card->ier_mask, intr);
 363         iowrite32(card->ier_mask, card->regmap + IER);
 364 }
 365
 366 void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr)
 367 {
 368         if (unlikely(card->eeh_state))
 369                 return;
 370
 371         __disable_intr(&card->ier_mask, intr);
 372         iowrite32(card->ier_mask, card->regmap + IER);
 373 }
 374
 375 void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
 376                                  unsigned int intr)
 377 {
 378         if (unlikely(card->halt) ||
 379             unlikely(card->eeh_state))
 380                 return;
 381
 382         __enable_intr(&card->isr_mask, intr);
 383         __enable_intr(&card->ier_mask, intr);
 384         iowrite32(card->ier_mask, card->regmap + IER);
 385 }
 386 void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
 387                                   unsigned int intr)
 388 {
 389         if (unlikely(card->eeh_state))
 390                 return;
 391
 392         __disable_intr(&card->isr_mask, intr);
 393         __disable_intr(&card->ier_mask, intr);
 394         iowrite32(card->ier_mask, card->regmap + IER);
 395 }
 396
 397 static irqreturn_t rsxx_isr(int irq, void *pdata)
 398 {
 399         struct rsxx_cardinfo *card = pdata;
 400         unsigned int isr;
 401         int handled = 0;
 402         int reread_isr;
 403         int i;
 404
 405         spin_lock(&card->irq_lock);
 406
 407         do {
 408                 reread_isr = 0;
 409
 410                 if (unlikely(card->eeh_state))
 411                         break;
 412
 413                 isr = ioread32(card->regmap + ISR);
 414                 if (isr == 0xffffffff) {
 415                         /*
 416                          * A few systems seem to have an intermittent issue
 417                          * where PCI reads return all Fs, but retrying the read
 418                          * a little later will return as expected.
 419                          */
 420                         dev_info(CARD_TO_DEV(card),
 421                                 "ISR = 0xFFFFFFFF, retrying later\n");
 422                         break;
 423                 }
 424
 425                 isr &= card->isr_mask;
 426                 if (!isr)
 427                         break;
 428
 429                 for (i = 0; i < card->n_targets; i++) {
 430                         if (isr & CR_INTR_DMA(i)) {
 431                                 if (card->ier_mask & CR_INTR_DMA(i)) {
 432                                         rsxx_disable_ier(card, CR_INTR_DMA(i));
 433                                         reread_isr = 1;
 434                                 }
 435                                 queue_work(card->ctrl[i].done_wq,
 436                                            &card->ctrl[i].dma_done_work);
 437                                 handled++;
 438                         }
 439                 }
 440
 441                 if (isr & CR_INTR_CREG) {
 442                         queue_work(card->creg_ctrl.creg_wq,
 443                                    &card->creg_ctrl.done_work);
 444                         handled++;
 445                 }
 446
 447                 if (isr & CR_INTR_EVENT) {
 448                         queue_work(card->event_wq, &card->event_work);
 449                         rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
 450                         handled++;
 451                 }
 452         } while (reread_isr);
 453
 454         spin_unlock(&card->irq_lock);
 455
 456         return handled ? IRQ_HANDLED : IRQ_NONE;
 457 }
 458
 459 /*----------------- Card Event Handler -------------------*/
 460 static const char * const rsxx_card_state_to_str(unsigned int state)
 461 {
 462         static const char * const state_strings[] = {
 463                 "Unknown", "Shutdown", "Starting", "Formatting",
 464                 "Uninitialized", "Good", "Shutting Down",
 465                 "Fault", "Read Only Fault", "dStroying"
 466         };
 467
 468         return state_strings[ffs(state)];
 469 }
 470
 471 static void card_state_change(struct rsxx_cardinfo *card,
 472                               unsigned int new_state)
 473 {
 474         int st;
 475
 476         dev_info(CARD_TO_DEV(card),
 477                 "card state change detected.(%s -> %s)\n",
 478                 rsxx_card_state_to_str(card->state),
 479                 rsxx_card_state_to_str(new_state));
 480
 481         card->state = new_state;
 482
 483         /* Don't attach DMA interfaces if the card has an invalid config */
 484         if (!card->config_valid)
 485                 return;
 486
 487         switch (new_state) {
 488         case CARD_STATE_RD_ONLY_FAULT:
 489                 dev_crit(CARD_TO_DEV(card),
 490                         "Hardware has entered read-only mode!\n");
 491                 /*
 492                  * Fall through so the DMA devices can be attached and
 493                  * the user can attempt to pull off their data.
 494                  */
 495         case CARD_STATE_GOOD:
 496                 st = rsxx_get_card_size8(card, &card->size8);
 497                 if (st)
 498                         dev_err(CARD_TO_DEV(card),
 499                                 "Failed attaching DMA devices\n");
 500
 501                 if (card->config_valid)
 502                         set_capacity(card->gendisk, card->size8 >> 9);
 503                 break;
 504
 505         case CARD_STATE_FAULT:
 506                 dev_crit(CARD_TO_DEV(card),
 507                         "Hardware Fault reported!\n");
 508                 /* Fall through. */
 509
 510         /* Everything else, detach DMA interface if it's attached. */
 511         case CARD_STATE_SHUTDOWN:
 512         case CARD_STATE_STARTING:
 513         case CARD_STATE_FORMATTING:
 514         case CARD_STATE_UNINITIALIZED:
 515         case CARD_STATE_SHUTTING_DOWN:
 516         /*
 517          * dStroy is a term coined by marketing to represent the low level
 518          * secure erase.
 519          */
 520         case CARD_STATE_DSTROYING:
 521                 set_capacity(card->gendisk, 0);
 522                 break;
 523         }
 524 }
 525
 526 static void card_event_handler(struct work_struct *work)
 527 {
 528         struct rsxx_cardinfo *card;
 529         unsigned int state;
 530         unsigned long flags;
 531         int st;
 532
 533         card = container_of(work, struct rsxx_cardinfo, event_work);
 534
 535         if (unlikely(card->halt))
 536                 return;
 537
 538         /*
 539          * Enable the interrupt now to avoid any weird race conditions where a
 540          * state change might occur while rsxx_get_card_state() is
 541          * processing a returned creg cmd.
 542          */
 543         spin_lock_irqsave(&card->irq_lock, flags);
 544         rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
 545         spin_unlock_irqrestore(&card->irq_lock, flags);
 546
 547         st = rsxx_get_card_state(card, &state);
 548         if (st) {
 549                 dev_info(CARD_TO_DEV(card),
 550                         "Failed reading state after event.\n");
 551                 return;
 552         }
 553
 554         if (card->state != state)
 555                 card_state_change(card, state);
 556
 557         if (card->creg_ctrl.creg_stats.stat & CREG_STAT_LOG_PENDING)
 558                 rsxx_read_hw_log(card);
 559 }
 560
 561 /*----------------- Card Operations -------------------*/
 562 static int card_shutdown(struct rsxx_cardinfo *card)
 563 {
 564         unsigned int state;
 565         signed long start;
 566         const int timeout = msecs_to_jiffies(120000);
 567         int st;
 568
 569         /* We can't issue a shutdown if the card is in a transition state */
 570         start = jiffies;
 571         do {
 572                 st = rsxx_get_card_state(card, &state);
 573                 if (st)
 574                         return st;
 575         } while (state == CARD_STATE_STARTING &&
 576                  (jiffies - start < timeout));
 577
 578         if (state == CARD_STATE_STARTING)
 579                 return -ETIMEDOUT;
 580
 581         /* Only issue a shutdown if we need to */
 582         if ((state != CARD_STATE_SHUTTING_DOWN) &&
 583             (state != CARD_STATE_SHUTDOWN)) {
 584                 st = rsxx_issue_card_cmd(card, CARD_CMD_SHUTDOWN);
 585                 if (st)
 586                         return st;
 587         }
 588
 589         start = jiffies;
 590         do {
 591                 st = rsxx_get_card_state(card, &state);
 592                 if (st)
 593                         return st;
 594         } while (state != CARD_STATE_SHUTDOWN &&
 595                  (jiffies - start < timeout));
 596
 597         if (state != CARD_STATE_SHUTDOWN)
 598                 return -ETIMEDOUT;
 599
 600         return 0;
 601 }
 602
 603 static int rsxx_eeh_frozen(struct pci_dev *dev)
 604 {
 605         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
 606         int i;
 607         int st;
 608
 609         dev_warn(&dev->dev, "IBM Flash Adapter PCI: preparing for slot reset.\n");
 610
 611         card->eeh_state = 1;
 612         rsxx_mask_interrupts(card);
 613
 614         /*
 615          * We need to guarantee that the write for eeh_state and masking
 616          * interrupts does not become reordered. This will prevent a possible
 617          * race condition with the EEH code.
 618          */
 619         wmb();
 620
 621         pci_disable_device(dev);
 622
 623         st = rsxx_eeh_save_issued_dmas(card);
 624         if (st)
 625                 return st;
 626
 627         rsxx_eeh_save_issued_creg(card);
 628
 629         for (i = 0; i < card->n_targets; i++) {
 630                 if (card->ctrl[i].status.buf)
 631                         pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
 632                                             card->ctrl[i].status.buf,
 633                                             card->ctrl[i].status.dma_addr);
 634                 if (card->ctrl[i].cmd.buf)
 635                         pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
 636                                             card->ctrl[i].cmd.buf,
 637                                             card->ctrl[i].cmd.dma_addr);
 638         }
 639
 640         return 0;
 641 }
 642
 643 static void rsxx_eeh_failure(struct pci_dev *dev)
 644 {
 645         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
 646         int i;
 647         int cnt = 0;
 648
 649         dev_err(&dev->dev, "IBM Flash Adapter PCI: disabling failed card.\n");
 650
 651         card->eeh_state = 1;
 652         card->halt = 1;
 653
 654         for (i = 0; i < card->n_targets; i++) {
 655                 spin_lock_bh(&card->ctrl[i].queue_lock);
 656                 cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
 657                                              &card->ctrl[i].queue,
 658                                              COMPLETE_DMA);
 659                 spin_unlock_bh(&card->ctrl[i].queue_lock);
 660
 661                 cnt += rsxx_dma_cancel(&card->ctrl[i]);
 662
 663                 if (cnt)
 664                         dev_info(CARD_TO_DEV(card),
 665                                 "Freed %d queued DMAs on channel %d\n",
 666                                 cnt, card->ctrl[i].id);
 667         }
 668 }
 669
 670 static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
 671 {
 672         unsigned int status;
 673         int iter = 0;
 674
 675         /* We need to wait for the hardware to reset */
 676         while (iter++ < 10) {
 677                 status = ioread32(card->regmap + PCI_RECONFIG);
 678
 679                 if (status & RSXX_FLUSH_BUSY) {
 680                         ssleep(1);
 681                         continue;
 682                 }
 683
 684                 if (status & RSXX_FLUSH_TIMEOUT)
 685                         dev_warn(CARD_TO_DEV(card), "HW: flash controller timeout\n");
 686                 return 0;
 687         }
 688
 689         /* Hardware failed resetting itself. */
 690         return -1;
 691 }
 692
 693 static pci_ers_result_t rsxx_error_detected(struct pci_dev *dev,
 694                                             enum pci_channel_state error)
 695 {
 696         int st;
 697
 698         if (dev->revision < RSXX_EEH_SUPPORT)
 699                 return PCI_ERS_RESULT_NONE;
 700
 701         if (error == pci_channel_io_perm_failure) {
 702                 rsxx_eeh_failure(dev);
 703                 return PCI_ERS_RESULT_DISCONNECT;
 704         }
 705
 706         st = rsxx_eeh_frozen(dev);
 707         if (st) {
 708                 dev_err(&dev->dev, "Slot reset setup failed\n");
 709                 rsxx_eeh_failure(dev);
 710                 return PCI_ERS_RESULT_DISCONNECT;
 711         }
 712
 713         return PCI_ERS_RESULT_NEED_RESET;
 714 }
 715
 716 static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
 717 {
 718         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
 719         unsigned long flags;
 720         int i;
 721         int st;
 722
 723         dev_warn(&dev->dev,
 724                 "IBM Flash Adapter PCI: recovering from slot reset.\n");
 725
 726         st = pci_enable_device(dev);
 727         if (st)
 728                 goto failed_hw_setup;
 729
 730         pci_set_master(dev);
 731
 732         st = rsxx_eeh_fifo_flush_poll(card);
 733         if (st)
 734                 goto failed_hw_setup;
 735
 736         rsxx_dma_queue_reset(card);
 737
 738         for (i = 0; i < card->n_targets; i++) {
 739                 st = rsxx_hw_buffers_init(dev, &card->ctrl[i]);
 740                 if (st)
 741                         goto failed_hw_buffers_init;
 742         }
 743
 744         if (card->config_valid)
 745                 rsxx_dma_configure(card);
 746
 747         /* Clears the ISR register from spurious interrupts */
 748         st = ioread32(card->regmap + ISR);
 749
 750         card->eeh_state = 0;
 751
 752         spin_lock_irqsave(&card->irq_lock, flags);
 753         if (card->n_targets & RSXX_MAX_TARGETS)
 754                 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_G);
 755         else
 756                 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_C);
 757         spin_unlock_irqrestore(&card->irq_lock, flags);
 758
 759         rsxx_kick_creg_queue(card);
 760
 761         for (i = 0; i < card->n_targets; i++) {
 762                 spin_lock(&card->ctrl[i].queue_lock);
 763                 if (list_empty(&card->ctrl[i].queue)) {
 764                         spin_unlock(&card->ctrl[i].queue_lock);
 765                         continue;
 766                 }
 767                 spin_unlock(&card->ctrl[i].queue_lock);
 768
 769                 queue_work(card->ctrl[i].issue_wq,
 770                                 &card->ctrl[i].issue_dma_work);
 771         }
 772
 773         dev_info(&dev->dev, "IBM Flash Adapter PCI: recovery complete.\n");
 774
 775         return PCI_ERS_RESULT_RECOVERED;
 776
 777 failed_hw_buffers_init:
 778         for (i = 0; i < card->n_targets; i++) {
 779                 if (card->ctrl[i].status.buf)
 780                         pci_free_consistent(card->dev,
 781                                         STATUS_BUFFER_SIZE8,
 782                                         card->ctrl[i].status.buf,
 783                                         card->ctrl[i].status.dma_addr);
 784                 if (card->ctrl[i].cmd.buf)
 785                         pci_free_consistent(card->dev,
 786                                         COMMAND_BUFFER_SIZE8,
 787                                         card->ctrl[i].cmd.buf,
 788                                         card->ctrl[i].cmd.dma_addr);
 789         }
 790 failed_hw_setup:
 791         rsxx_eeh_failure(dev);
 792         return PCI_ERS_RESULT_DISCONNECT;
 793
 794 }
 795
 796 /*----------------- Driver Initialization & Setup -------------------*/
 797 /* Returns:   0 if the driver is compatible with the device
 798              -1 if the driver is NOT compatible with the device */
 799 static int rsxx_compatibility_check(struct rsxx_cardinfo *card)
 800 {
 801         unsigned char pci_rev;
 802
 803         pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
 804
 805         if (pci_rev > RS70_PCI_REV_SUPPORTED)
 806                 return -1;
 807         return 0;
 808 }
 809
 810 static int rsxx_pci_probe(struct pci_dev *dev,
 811                                         const struct pci_device_id *id)
 812 {
 813         struct rsxx_cardinfo *card;
 814         int st;
 815         unsigned int sync_timeout;
 816
 817         dev_info(&dev->dev, "PCI-Flash SSD discovered\n");
 818
 819         card = kzalloc(sizeof(*card), GFP_KERNEL);
 820         if (!card)
 821                 return -ENOMEM;
 822
 823         card->dev = dev;
 824         pci_set_drvdata(dev, card);
 825
 826         do {
 827                 if (!ida_pre_get(&rsxx_disk_ida, GFP_KERNEL)) {
 828                         st = -ENOMEM;
 829                         goto failed_ida_get;
 830                 }
 831
 832                 spin_lock(&rsxx_ida_lock);
 833                 st = ida_get_new(&rsxx_disk_ida, &card->disk_id);
 834                 spin_unlock(&rsxx_ida_lock);
 835         } while (st == -EAGAIN);
 836
 837         if (st)
 838                 goto failed_ida_get;
 839
 840         st = pci_enable_device(dev);
 841         if (st)
 842                 goto failed_enable;
 843
 844         pci_set_master(dev);
 845         pci_set_dma_max_seg_size(dev, RSXX_HW_BLK_SIZE);
 846
 847         st = pci_set_dma_mask(dev, DMA_BIT_MASK(64));
 848         if (st) {
 849                 dev_err(CARD_TO_DEV(card),
 850                         "No usable DMA configuration,aborting\n");
 851                 goto failed_dma_mask;
 852         }
 853
 854         st = pci_request_regions(dev, DRIVER_NAME);
 855         if (st) {
 856                 dev_err(CARD_TO_DEV(card),
 857                         "Failed to request memory region\n");
 858                 goto failed_request_regions;
 859         }
 860
 861         if (pci_resource_len(dev, 0) == 0) {
 862                 dev_err(CARD_TO_DEV(card), "BAR0 has length 0!\n");
 863                 st = -ENOMEM;
 864                 goto failed_iomap;
 865         }
 866
 867         card->regmap = pci_iomap(dev, 0, 0);
 868         if (!card->regmap) {
 869                 dev_err(CARD_TO_DEV(card), "Failed to map BAR0\n");
 870                 st = -ENOMEM;
 871                 goto failed_iomap;
 872         }
 873
 874         spin_lock_init(&card->irq_lock);
 875         card->halt = 0;
 876         card->eeh_state = 0;
 877
 878         spin_lock_irq(&card->irq_lock);
 879         rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
 880         spin_unlock_irq(&card->irq_lock);
 881
 882         if (!force_legacy) {
 883                 st = pci_enable_msi(dev);
 884                 if (st)
 885                         dev_warn(CARD_TO_DEV(card),
 886                                 "Failed to enable MSI\n");
 887         }
 888
 889         st = request_irq(dev->irq, rsxx_isr, IRQF_DISABLED | IRQF_SHARED,
 890                          DRIVER_NAME, card);
 891         if (st) {
 892                 dev_err(CARD_TO_DEV(card),
 893                         "Failed requesting IRQ%d\n", dev->irq);
 894                 goto failed_irq;
 895         }
 896
 897         /************* Setup Processor Command Interface *************/
 898         st = rsxx_creg_setup(card);
 899         if (st) {
 900                 dev_err(CARD_TO_DEV(card), "Failed to setup creg interface.\n");
 901                 goto failed_creg_setup;
 902         }
 903
 904         spin_lock_irq(&card->irq_lock);
 905         rsxx_enable_ier_and_isr(card, CR_INTR_CREG);
 906         spin_unlock_irq(&card->irq_lock);
 907
 908         st = rsxx_compatibility_check(card);
 909         if (st) {
 910                 dev_warn(CARD_TO_DEV(card),
 911                         "Incompatible driver detected. Please update the driver.\n");
 912                 st = -EINVAL;
 913                 goto failed_compatiblity_check;
 914         }
 915
 916         /************* Load Card Config *************/
 917         st = rsxx_load_config(card);
 918         if (st)
 919                 dev_err(CARD_TO_DEV(card),
 920                         "Failed loading card config\n");
 921
 922         /************* Setup DMA Engine *************/
 923         st = rsxx_get_num_targets(card, &card->n_targets);
 924         if (st)
 925                 dev_info(CARD_TO_DEV(card),
 926                         "Failed reading the number of DMA targets\n");
 927
 928         card->ctrl = kzalloc(card->n_targets * sizeof(*card->ctrl), GFP_KERNEL);
 929         if (!card->ctrl) {
 930                 st = -ENOMEM;
 931                 goto failed_dma_setup;
 932         }
 933
 934         st = rsxx_dma_setup(card);
 935         if (st) {
 936                 dev_info(CARD_TO_DEV(card),
 937                         "Failed to setup DMA engine\n");
 938                 goto failed_dma_setup;
 939         }
 940
 941         /************* Setup Card Event Handler *************/
 942         card->event_wq = create_singlethread_workqueue(DRIVER_NAME"_event");
 943         if (!card->event_wq) {
 944                 dev_err(CARD_TO_DEV(card), "Failed card event setup.\n");
 945                 goto failed_event_handler;
 946         }
 947
 948         INIT_WORK(&card->event_work, card_event_handler);
 949
 950         st = rsxx_setup_dev(card);
 951         if (st)
 952                 goto failed_create_dev;
 953
 954         rsxx_get_card_state(card, &card->state);
 955
 956         dev_info(CARD_TO_DEV(card),
 957                 "card state: %s\n",
 958                 rsxx_card_state_to_str(card->state));
 959
 960         /*
 961          * Now that the DMA Engine and devices have been setup,
 962          * we can enable the event interrupt(it kicks off actions in
 963          * those layers so we couldn't enable it right away.)
 964          */
 965         spin_lock_irq(&card->irq_lock);
 966         rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
 967         spin_unlock_irq(&card->irq_lock);
 968
 969         if (card->state == CARD_STATE_SHUTDOWN) {
 970                 st = rsxx_issue_card_cmd(card, CARD_CMD_STARTUP);
 971                 if (st)
 972                         dev_crit(CARD_TO_DEV(card),
 973                                 "Failed issuing card startup\n");
 974                 if (sync_start) {
 975                         sync_timeout = SYNC_START_TIMEOUT;
 976
 977                         dev_info(CARD_TO_DEV(card),
 978                                  "Waiting for card to startup\n");
 979
 980                         do {
 981                                 ssleep(1);
 982                                 sync_timeout--;
 983
 984                                 rsxx_get_card_state(card, &card->state);
 985                         } while (sync_timeout &&
 986                                 (card->state == CARD_STATE_STARTING));
 987
 988                         if (card->state == CARD_STATE_STARTING) {
 989                                 dev_warn(CARD_TO_DEV(card),
 990                                          "Card startup timed out\n");
 991                                 card->size8 = 0;
 992                         } else {
 993                                 dev_info(CARD_TO_DEV(card),
 994                                         "card state: %s\n",
 995                                         rsxx_card_state_to_str(card->state));
 996                                 st = rsxx_get_card_size8(card, &card->size8);
 997                                 if (st)
 998                                         card->size8 = 0;
 999                         }
1000                 }
1001         } else if (card->state == CARD_STATE_GOOD ||
1002                    card->state == CARD_STATE_RD_ONLY_FAULT) {
1003                 st = rsxx_get_card_size8(card, &card->size8);
1004                 if (st)
1005                         card->size8 = 0;
1006         }
1007
1008         rsxx_attach_dev(card);
1009
1010         /************* Setup Debugfs *************/
1011         rsxx_debugfs_dev_new(card);
1012
1013         return 0;
1014
1015 failed_create_dev:
1016         destroy_workqueue(card->event_wq);
1017         card->event_wq = NULL;
1018 failed_event_handler:
1019         rsxx_dma_destroy(card);
1020 failed_dma_setup:
1021 failed_compatiblity_check:
1022         destroy_workqueue(card->creg_ctrl.creg_wq);
1023         card->creg_ctrl.creg_wq = NULL;
1024 failed_creg_setup:
1025         spin_lock_irq(&card->irq_lock);
1026         rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
1027         spin_unlock_irq(&card->irq_lock);
1028         free_irq(dev->irq, card);
1029         if (!force_legacy)
1030                 pci_disable_msi(dev);
1031 failed_irq:
1032         pci_iounmap(dev, card->regmap);
1033 failed_iomap:
1034         pci_release_regions(dev);
1035 failed_request_regions:
1036 failed_dma_mask:
1037         pci_disable_device(dev);
1038 failed_enable:
1039         spin_lock(&rsxx_ida_lock);
1040         ida_remove(&rsxx_disk_ida, card->disk_id);
1041         spin_unlock(&rsxx_ida_lock);
1042 failed_ida_get:
1043         kfree(card);
1044
1045         return st;
1046 }
1047
1048 static void rsxx_pci_remove(struct pci_dev *dev)
1049 {
1050         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
1051         unsigned long flags;
1052         int st;
1053         int i;
1054
1055         if (!card)
1056                 return;
1057
1058         dev_info(CARD_TO_DEV(card),
1059                 "Removing PCI-Flash SSD.\n");
1060
1061         rsxx_detach_dev(card);
1062
1063         for (i = 0; i < card->n_targets; i++) {
1064                 spin_lock_irqsave(&card->irq_lock, flags);
1065                 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1066                 spin_unlock_irqrestore(&card->irq_lock, flags);
1067         }
1068
1069         st = card_shutdown(card);
1070         if (st)
1071                 dev_crit(CARD_TO_DEV(card), "Shutdown failed!\n");
1072
1073         /* Sync outstanding event handlers. */
1074         spin_lock_irqsave(&card->irq_lock, flags);
1075         rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
1076         spin_unlock_irqrestore(&card->irq_lock, flags);
1077
1078         cancel_work_sync(&card->event_work);
1079
1080         rsxx_destroy_dev(card);
1081         rsxx_dma_destroy(card);
1082
1083         spin_lock_irqsave(&card->irq_lock, flags);
1084         rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
1085         spin_unlock_irqrestore(&card->irq_lock, flags);
1086
1087         /* Prevent work_structs from re-queuing themselves. */
1088         card->halt = 1;
1089
1090         debugfs_remove_recursive(card->debugfs_dir);
1091
1092         free_irq(dev->irq, card);
1093
1094         if (!force_legacy)
1095                 pci_disable_msi(dev);
1096
1097         rsxx_creg_destroy(card);
1098
1099         pci_iounmap(dev, card->regmap);
1100
1101         pci_disable_device(dev);
1102         pci_release_regions(dev);
1103
1104         kfree(card);
1105 }
1106
1107 static int rsxx_pci_suspend(struct pci_dev *dev, pm_message_t state)
1108 {
1109         /* We don't support suspend at this time. */
1110         return -ENOSYS;
1111 }
1112
1113 static void rsxx_pci_shutdown(struct pci_dev *dev)
1114 {
1115         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
1116         unsigned long flags;
1117         int i;
1118
1119         if (!card)
1120                 return;
1121
1122         dev_info(CARD_TO_DEV(card), "Shutting down PCI-Flash SSD.\n");
1123
1124         rsxx_detach_dev(card);
1125
1126         for (i = 0; i < card->n_targets; i++) {
1127                 spin_lock_irqsave(&card->irq_lock, flags);
1128                 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1129                 spin_unlock_irqrestore(&card->irq_lock, flags);
1130         }
1131
1132         card_shutdown(card);
1133 }
1134
1135 static const struct pci_error_handlers rsxx_err_handler = {
1136         .error_detected = rsxx_error_detected,
1137         .slot_reset     = rsxx_slot_reset,
1138 };
1139
1140 static DEFINE_PCI_DEVICE_TABLE(rsxx_pci_ids) = {
1141         {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS70_FLASH)},
1142         {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS80_FLASH)},
1143         {0,},
1144 };
1145
1146 MODULE_DEVICE_TABLE(pci, rsxx_pci_ids);
1147
1148 static struct pci_driver rsxx_pci_driver = {
1149         .name           = DRIVER_NAME,
1150         .id_table       = rsxx_pci_ids,
1151         .probe          = rsxx_pci_probe,
1152         .remove         = rsxx_pci_remove,
1153         .suspend        = rsxx_pci_suspend,
1154         .shutdown       = rsxx_pci_shutdown,
1155         .err_handler    = &rsxx_err_handler,
1156 };
1157
1158 static int __init rsxx_core_init(void)
1159 {
1160         int st;
1161
1162         st = rsxx_dev_init();
1163         if (st)
1164                 return st;
1165
1166         st = rsxx_dma_init();
1167         if (st)
1168                 goto dma_init_failed;
1169
1170         st = rsxx_creg_init();
1171         if (st)
1172                 goto creg_init_failed;
1173
1174         return pci_register_driver(&rsxx_pci_driver);
1175
1176 creg_init_failed:
1177         rsxx_dma_cleanup();
1178 dma_init_failed:
1179         rsxx_dev_cleanup();
1180
1181         return st;
1182 }
1183
1184 static void __exit rsxx_core_cleanup(void)
1185 {
1186         pci_unregister_driver(&rsxx_pci_driver);
1187         rsxx_creg_cleanup();
1188         rsxx_dma_cleanup();
1189         rsxx_dev_cleanup();
1190 }
1191
1192 module_init(rsxx_core_init);
1193 module_exit(rsxx_core_cleanup);