drivers/crypto/cavium/nitrox/nitrox_main.c

   1 #include <linux/aer.h>
   2 #include <linux/delay.h>
   3 #include <linux/debugfs.h>
   4 #include <linux/firmware.h>
   5 #include <linux/list.h>
   6 #include <linux/module.h>
   7 #include <linux/mutex.h>
   8 #include <linux/pci.h>
   9 #include <linux/pci_ids.h>
  10
  11 #include "nitrox_dev.h"
  12 #include "nitrox_common.h"
  13 #include "nitrox_csr.h"
  14
  15 #define CNN55XX_DEV_ID  0x12
  16 #define MAX_PF_QUEUES   64
  17 #define UCODE_HLEN 48
  18 #define SE_GROUP 0
  19
  20 #define DRIVER_VERSION "1.0"
  21 #define FW_DIR "cavium/"
  22 /* SE microcode */
  23 #define SE_FW   FW_DIR "cnn55xx_se.fw"
  24
  25 static const char nitrox_driver_name[] = "CNN55XX";
  26
  27 static LIST_HEAD(ndevlist);
  28 static DEFINE_MUTEX(devlist_lock);
  29 static unsigned int num_devices;
  30
  31 /**
  32  * nitrox_pci_tbl - PCI Device ID Table
  33  */
  34 static const struct pci_device_id nitrox_pci_tbl[] = {
  35         {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
  36         /* required last entry */
  37         {0, }
  38 };
  39 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
  40
  41 static unsigned int qlen = DEFAULT_CMD_QLEN;
  42 module_param(qlen, uint, 0644);
  43 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
  44
  45 /**
  46  * struct ucode - Firmware Header
  47  * @id: microcode ID
  48  * @version: firmware version
  49  * @code_size: code section size
  50  * @raz: alignment
  51  * @code: code section
  52  */
  53 struct ucode {
  54         u8 id;
  55         char version[VERSION_LEN - 1];
  56         __be32 code_size;
  57         u8 raz[12];
  58         u64 code[0];
  59 };
  60
  61 /**
  62  * write_to_ucd_unit - Write Firmware to NITROX UCD unit
  63  */
  64 static void write_to_ucd_unit(struct nitrox_device *ndev,
  65                               struct ucode *ucode)
  66 {
  67         u32 code_size = be32_to_cpu(ucode->code_size) * 2;
  68         u64 offset, data;
  69         int i = 0;
  70
  71         /*
  72          * UCD structure
  73          *
  74          *  -------------
  75          *  |    BLK 7  |
  76          *  -------------
  77          *  |    BLK 6  |
  78          *  -------------
  79          *  |    ...    |
  80          *  -------------
  81          *  |    BLK 0  |
  82          *  -------------
  83          *  Total of 8 blocks, each size 32KB
  84          */
  85
  86         /* set the block number */
  87         offset = UCD_UCODE_LOAD_BLOCK_NUM;
  88         nitrox_write_csr(ndev, offset, 0);
  89
  90         code_size = roundup(code_size, 8);
  91         while (code_size) {
  92                 data = ucode->code[i];
  93                 /* write 8 bytes at a time */
  94                 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
  95                 nitrox_write_csr(ndev, offset, data);
  96                 code_size -= 8;
  97                 i++;
  98         }
  99
 100         /* put all SE cores in group 0 */
 101         offset = POM_GRP_EXECMASKX(SE_GROUP);
 102         nitrox_write_csr(ndev, offset, (~0ULL));
 103
 104         for (i = 0; i < ndev->hw.se_cores; i++) {
 105                 /*
 106                  * write block number and firware length
 107                  * bit:<2:0> block number
 108                  * bit:3 is set SE uses 32KB microcode
 109                  * bit:3 is clear SE uses 64KB microcode
 110                  */
 111                 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
 112                 nitrox_write_csr(ndev, offset, 0x8);
 113         }
 114         usleep_range(300, 400);
 115 }
 116
 117 static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
 118 {
 119         const struct firmware *fw;
 120         struct ucode *ucode;
 121         int ret;
 122
 123         dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
 124
 125         ret = request_firmware(&fw, fw_name, DEV(ndev));
 126         if (ret < 0) {
 127                 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
 128                 return ret;
 129         }
 130
 131         ucode = (struct ucode *)fw->data;
 132         /* copy the firmware version */
 133         memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
 134         ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
 135
 136         write_to_ucd_unit(ndev, ucode);
 137         release_firmware(fw);
 138
 139         set_bit(NITROX_UCODE_LOADED, &ndev->status);
 140         /* barrier to sync with other cpus */
 141         smp_mb__after_atomic();
 142         return 0;
 143 }
 144
 145 /**
 146  * nitrox_add_to_devlist - add NITROX device to global device list
 147  * @ndev: NITROX device
 148  */
 149 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
 150 {
 151         struct nitrox_device *dev;
 152         int ret = 0;
 153
 154         INIT_LIST_HEAD(&ndev->list);
 155         refcount_set(&ndev->refcnt, 1);
 156
 157         mutex_lock(&devlist_lock);
 158         list_for_each_entry(dev, &ndevlist, list) {
 159                 if (dev == ndev) {
 160                         ret = -EEXIST;
 161                         goto unlock;
 162                 }
 163         }
 164         ndev->idx = num_devices++;
 165         list_add_tail(&ndev->list, &ndevlist);
 166 unlock:
 167         mutex_unlock(&devlist_lock);
 168         return ret;
 169 }
 170
 171 /**
 172  * nitrox_remove_from_devlist - remove NITROX device from
 173  *   global device list
 174  * @ndev: NITROX device
 175  */
 176 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
 177 {
 178         mutex_lock(&devlist_lock);
 179         list_del(&ndev->list);
 180         num_devices--;
 181         mutex_unlock(&devlist_lock);
 182 }
 183
 184 struct nitrox_device *nitrox_get_first_device(void)
 185 {
 186         struct nitrox_device *ndev = NULL;
 187
 188         mutex_lock(&devlist_lock);
 189         list_for_each_entry(ndev, &ndevlist, list) {
 190                 if (nitrox_ready(ndev))
 191                         break;
 192         }
 193         mutex_unlock(&devlist_lock);
 194         if (!ndev)
 195                 return NULL;
 196
 197         refcount_inc(&ndev->refcnt);
 198         /* barrier to sync with other cpus */
 199         smp_mb__after_atomic();
 200         return ndev;
 201 }
 202
 203 void nitrox_put_device(struct nitrox_device *ndev)
 204 {
 205         if (!ndev)
 206                 return;
 207
 208         refcount_dec(&ndev->refcnt);
 209         /* barrier to sync with other cpus */
 210         smp_mb__after_atomic();
 211 }
 212
 213 static int nitrox_reset_device(struct pci_dev *pdev)
 214 {
 215         int pos = 0;
 216
 217         pos = pci_save_state(pdev);
 218         if (pos) {
 219                 dev_err(&pdev->dev, "Failed to save pci state\n");
 220                 return -ENOMEM;
 221         }
 222
 223         pos = pci_pcie_cap(pdev);
 224         if (!pos)
 225                 return -ENOTTY;
 226
 227         if (!pci_wait_for_pending_transaction(pdev))
 228                 dev_err(&pdev->dev, "waiting for pending transaction\n");
 229
 230         pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
 231         msleep(100);
 232         pci_restore_state(pdev);
 233
 234         return 0;
 235 }
 236
 237 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
 238 {
 239         int err;
 240
 241         err = nitrox_common_sw_init(ndev);
 242         if (err)
 243                 return err;
 244
 245         err = nitrox_pf_init_isr(ndev);
 246         if (err)
 247                 nitrox_common_sw_cleanup(ndev);
 248
 249         return err;
 250 }
 251
 252 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
 253 {
 254         nitrox_pf_cleanup_isr(ndev);
 255         nitrox_common_sw_cleanup(ndev);
 256 }
 257
 258 /**
 259  * nitrox_bist_check - Check NITORX BIST registers status
 260  * @ndev: NITROX device
 261  */
 262 static int nitrox_bist_check(struct nitrox_device *ndev)
 263 {
 264         u64 value = 0;
 265         int i;
 266
 267         for (i = 0; i < NR_CLUSTERS; i++) {
 268                 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
 269                 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
 270         }
 271         value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
 272         value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
 273         value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
 274         value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
 275         value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
 276         value += nitrox_read_csr(ndev, POM_BIST_REG);
 277         value += nitrox_read_csr(ndev, BMI_BIST_REG);
 278         value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
 279         value += nitrox_read_csr(ndev, BMO_BIST_REG);
 280         value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
 281         value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
 282         if (value)
 283                 return -EIO;
 284         return 0;
 285 }
 286
 287 static void nitrox_get_hwinfo(struct nitrox_device *ndev)
 288 {
 289         union emu_fuse_map emu_fuse;
 290         u64 offset;
 291         int i;
 292
 293         for (i = 0; i < NR_CLUSTERS; i++) {
 294                 u8 dead_cores;
 295
 296                 offset = EMU_FUSE_MAPX(i);
 297                 emu_fuse.value = nitrox_read_csr(ndev, offset);
 298                 if (emu_fuse.s.valid) {
 299                         dead_cores = hweight32(emu_fuse.s.ae_fuse);
 300                         ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
 301                         dead_cores = hweight16(emu_fuse.s.se_fuse);
 302                         ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
 303                 }
 304         }
 305 }
 306
 307 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
 308 {
 309         int err;
 310
 311         err = nitrox_bist_check(ndev);
 312         if (err) {
 313                 dev_err(&ndev->pdev->dev, "BIST check failed\n");
 314                 return err;
 315         }
 316         /* get cores information */
 317         nitrox_get_hwinfo(ndev);
 318
 319         nitrox_config_nps_unit(ndev);
 320         nitrox_config_pom_unit(ndev);
 321         nitrox_config_efl_unit(ndev);
 322         /* configure IO units */
 323         nitrox_config_bmi_unit(ndev);
 324         nitrox_config_bmo_unit(ndev);
 325         /* configure Local Buffer Cache */
 326         nitrox_config_lbc_unit(ndev);
 327         nitrox_config_rand_unit(ndev);
 328
 329         /* load firmware on SE cores */
 330         err = nitrox_load_fw(ndev, SE_FW);
 331         if (err)
 332                 return err;
 333
 334         nitrox_config_emu_unit(ndev);
 335
 336         return 0;
 337 }
 338
 339 #if IS_ENABLED(CONFIG_DEBUG_FS)
 340 static int registers_show(struct seq_file *s, void *v)
 341 {
 342         struct nitrox_device *ndev = s->private;
 343         u64 offset;
 344
 345         /* NPS DMA stats */
 346         offset = NPS_STATS_PKT_DMA_RD_CNT;
 347         seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT  0x%016llx\n",
 348                    nitrox_read_csr(ndev, offset));
 349         offset = NPS_STATS_PKT_DMA_WR_CNT;
 350         seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT  0x%016llx\n",
 351                    nitrox_read_csr(ndev, offset));
 352
 353         /* BMI/BMO stats */
 354         offset = BMI_NPS_PKT_CNT;
 355         seq_printf(s, "BMI_NPS_PKT_CNT  0x%016llx\n",
 356                    nitrox_read_csr(ndev, offset));
 357         offset = BMO_NPS_SLC_PKT_CNT;
 358         seq_printf(s, "BMO_NPS_PKT_CNT  0x%016llx\n",
 359                    nitrox_read_csr(ndev, offset));
 360
 361         return 0;
 362 }
 363
 364 static int registers_open(struct inode *inode, struct file *file)
 365 {
 366         return single_open(file, registers_show, inode->i_private);
 367 }
 368
 369 static const struct file_operations register_fops = {
 370         .owner = THIS_MODULE,
 371         .open = registers_open,
 372         .read = seq_read,
 373         .llseek = seq_lseek,
 374         .release = single_release,
 375 };
 376
 377 static int firmware_show(struct seq_file *s, void *v)
 378 {
 379         struct nitrox_device *ndev = s->private;
 380
 381         seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
 382         return 0;
 383 }
 384
 385 static int firmware_open(struct inode *inode, struct file *file)
 386 {
 387         return single_open(file, firmware_show, inode->i_private);
 388 }
 389
 390 static const struct file_operations firmware_fops = {
 391         .owner = THIS_MODULE,
 392         .open = firmware_open,
 393         .read = seq_read,
 394         .llseek = seq_lseek,
 395         .release = single_release,
 396 };
 397
 398 static int nitrox_show(struct seq_file *s, void *v)
 399 {
 400         struct nitrox_device *ndev = s->private;
 401
 402         seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
 403         seq_printf(s, "  Revision ID: 0x%0x\n", ndev->hw.revision_id);
 404         seq_printf(s, "  Cores [AE: %u  SE: %u]\n",
 405                    ndev->hw.ae_cores, ndev->hw.se_cores);
 406         seq_printf(s, "  Number of Queues: %u\n", ndev->nr_queues);
 407         seq_printf(s, "  Queue length: %u\n", ndev->qlen);
 408         seq_printf(s, "  Node: %u\n", ndev->node);
 409
 410         return 0;
 411 }
 412
 413 static int nitrox_open(struct inode *inode, struct file *file)
 414 {
 415         return single_open(file, nitrox_show, inode->i_private);
 416 }
 417
 418 static const struct file_operations nitrox_fops = {
 419         .owner = THIS_MODULE,
 420         .open = nitrox_open,
 421         .read = seq_read,
 422         .llseek = seq_lseek,
 423         .release = single_release,
 424 };
 425
 426 static void nitrox_debugfs_exit(struct nitrox_device *ndev)
 427 {
 428         debugfs_remove_recursive(ndev->debugfs_dir);
 429         ndev->debugfs_dir = NULL;
 430 }
 431
 432 static int nitrox_debugfs_init(struct nitrox_device *ndev)
 433 {
 434         struct dentry *dir, *f;
 435
 436         dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
 437         if (!dir)
 438                 return -ENOMEM;
 439
 440         ndev->debugfs_dir = dir;
 441         f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
 442         if (!f)
 443                 goto err;
 444         f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
 445         if (!f)
 446                 goto err;
 447         f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
 448         if (!f)
 449                 goto err;
 450
 451         return 0;
 452
 453 err:
 454         nitrox_debugfs_exit(ndev);
 455         return -ENODEV;
 456 }
 457 #else
 458 static int nitrox_debugfs_init(struct nitrox_device *ndev)
 459 {
 460         return 0;
 461 }
 462
 463 static void nitrox_debugfs_exit(struct nitrox_device *ndev)
 464 {
 465 }
 466 #endif
 467
 468 /**
 469  * nitrox_probe - NITROX Initialization function.
 470  * @pdev: PCI device information struct
 471  * @id: entry in nitrox_pci_tbl
 472  *
 473  * Return: 0, if the driver is bound to the device, or
 474  *         a negative error if there is failure.
 475  */
 476 static int nitrox_probe(struct pci_dev *pdev,
 477                         const struct pci_device_id *id)
 478 {
 479         struct nitrox_device *ndev;
 480         int err;
 481
 482         dev_info_once(&pdev->dev, "%s driver version %s\n",
 483                       nitrox_driver_name, DRIVER_VERSION);
 484
 485         err = pci_enable_device_mem(pdev);
 486         if (err)
 487                 return err;
 488
 489         /* do FLR */
 490         err = nitrox_reset_device(pdev);
 491         if (err) {
 492                 dev_err(&pdev->dev, "FLR failed\n");
 493                 pci_disable_device(pdev);
 494                 return err;
 495         }
 496
 497         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
 498                 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
 499         } else {
 500                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 501                 if (err) {
 502                         dev_err(&pdev->dev, "DMA configuration failed\n");
 503                         pci_disable_device(pdev);
 504                         return err;
 505                 }
 506         }
 507
 508         err = pci_request_mem_regions(pdev, nitrox_driver_name);
 509         if (err) {
 510                 pci_disable_device(pdev);
 511                 return err;
 512         }
 513         pci_set_master(pdev);
 514
 515         ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
 516         if (!ndev)
 517                 goto ndev_fail;
 518
 519         pci_set_drvdata(pdev, ndev);
 520         ndev->pdev = pdev;
 521
 522         /* add to device list */
 523         nitrox_add_to_devlist(ndev);
 524
 525         ndev->hw.vendor_id = pdev->vendor;
 526         ndev->hw.device_id = pdev->device;
 527         ndev->hw.revision_id = pdev->revision;
 528         /* command timeout in jiffies */
 529         ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
 530         ndev->node = dev_to_node(&pdev->dev);
 531         if (ndev->node == NUMA_NO_NODE)
 532                 ndev->node = 0;
 533
 534         ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
 535                                  pci_resource_len(pdev, 0));
 536         if (!ndev->bar_addr) {
 537                 err = -EIO;
 538                 goto ioremap_err;
 539         }
 540         /* allocate command queus based on cpus, max queues are 64 */
 541         ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
 542         ndev->qlen = qlen;
 543
 544         err = nitrox_pf_sw_init(ndev);
 545         if (err)
 546                 goto ioremap_err;
 547
 548         err = nitrox_pf_hw_init(ndev);
 549         if (err)
 550                 goto pf_hw_fail;
 551
 552         err = nitrox_debugfs_init(ndev);
 553         if (err)
 554                 goto pf_hw_fail;
 555
 556         set_bit(NITROX_READY, &ndev->status);
 557         /* barrier to sync with other cpus */
 558         smp_mb__after_atomic();
 559
 560         err = nitrox_crypto_register();
 561         if (err)
 562                 goto crypto_fail;
 563
 564         return 0;
 565
 566 crypto_fail:
 567         nitrox_debugfs_exit(ndev);
 568         clear_bit(NITROX_READY, &ndev->status);
 569         /* barrier to sync with other cpus */
 570         smp_mb__after_atomic();
 571 pf_hw_fail:
 572         nitrox_pf_sw_cleanup(ndev);
 573 ioremap_err:
 574         nitrox_remove_from_devlist(ndev);
 575         kfree(ndev);
 576         pci_set_drvdata(pdev, NULL);
 577 ndev_fail:
 578         pci_release_mem_regions(pdev);
 579         pci_disable_device(pdev);
 580         return err;
 581 }
 582
 583 /**
 584  * nitrox_remove - Unbind the driver from the device.
 585  * @pdev: PCI device information struct
 586  */
 587 static void nitrox_remove(struct pci_dev *pdev)
 588 {
 589         struct nitrox_device *ndev = pci_get_drvdata(pdev);
 590
 591         if (!ndev)
 592                 return;
 593
 594         if (!refcount_dec_and_test(&ndev->refcnt)) {
 595                 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
 596                         refcount_read(&ndev->refcnt));
 597                 return;
 598         }
 599
 600         dev_info(DEV(ndev), "Removing Device %x:%x\n",
 601                  ndev->hw.vendor_id, ndev->hw.device_id);
 602
 603         clear_bit(NITROX_READY, &ndev->status);
 604         /* barrier to sync with other cpus */
 605         smp_mb__after_atomic();
 606
 607         nitrox_remove_from_devlist(ndev);
 608         nitrox_crypto_unregister();
 609         nitrox_debugfs_exit(ndev);
 610         nitrox_pf_sw_cleanup(ndev);
 611
 612         iounmap(ndev->bar_addr);
 613         kfree(ndev);
 614
 615         pci_set_drvdata(pdev, NULL);
 616         pci_release_mem_regions(pdev);
 617         pci_disable_device(pdev);
 618 }
 619
 620 static void nitrox_shutdown(struct pci_dev *pdev)
 621 {
 622         pci_set_drvdata(pdev, NULL);
 623         pci_release_mem_regions(pdev);
 624         pci_disable_device(pdev);
 625 }
 626
 627 static struct pci_driver nitrox_driver = {
 628         .name = nitrox_driver_name,
 629         .id_table = nitrox_pci_tbl,
 630         .probe = nitrox_probe,
 631         .remove = nitrox_remove,
 632         .shutdown = nitrox_shutdown,
 633 };
 634
 635 module_pci_driver(nitrox_driver);
 636
 637 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
 638 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
 639 MODULE_LICENSE("GPL");
 640 MODULE_VERSION(DRIVER_VERSION);
 641 MODULE_FIRMWARE(SE_FW);