drivers/crypto/cavium/nitrox/nitrox_isr.c

   1 #include <linux/pci.h>
   2 #include <linux/printk.h>
   3 #include <linux/slab.h>
   4
   5 #include "nitrox_dev.h"
   6 #include "nitrox_csr.h"
   7 #include "nitrox_common.h"
   8
   9 #define NR_RING_VECTORS 3
  10 #define NPS_CORE_INT_ACTIVE_ENTRY 192
  11
  12 /**
  13  * nps_pkt_slc_isr - IRQ handler for NPS solicit port
  14  * @irq: irq number
  15  * @data: argument
  16  */
  17 static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
  18 {
  19         struct bh_data *slc = data;
  20         union nps_pkt_slc_cnts pkt_slc_cnts;
  21
  22         pkt_slc_cnts.value = readq(slc->completion_cnt_csr_addr);
  23         /* New packet on SLC output port */
  24         if (pkt_slc_cnts.s.slc_int)
  25                 tasklet_hi_schedule(&slc->resp_handler);
  26
  27         return IRQ_HANDLED;
  28 }
  29
  30 static void clear_nps_core_err_intr(struct nitrox_device *ndev)
  31 {
  32         u64 value;
  33
  34         /* Write 1 to clear */
  35         value = nitrox_read_csr(ndev, NPS_CORE_INT);
  36         nitrox_write_csr(ndev, NPS_CORE_INT, value);
  37
  38         dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT  0x%016llx\n", value);
  39 }
  40
  41 static void clear_nps_pkt_err_intr(struct nitrox_device *ndev)
  42 {
  43         union nps_pkt_int pkt_int;
  44         unsigned long value, offset;
  45         int i;
  46
  47         pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT);
  48         dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT  0x%016llx\n",
  49                             pkt_int.value);
  50
  51         if (pkt_int.s.slc_err) {
  52                 offset = NPS_PKT_SLC_ERR_TYPE;
  53                 value = nitrox_read_csr(ndev, offset);
  54                 nitrox_write_csr(ndev, offset, value);
  55                 dev_err_ratelimited(DEV(ndev),
  56                                     "NPS_PKT_SLC_ERR_TYPE  0x%016lx\n", value);
  57
  58                 offset = NPS_PKT_SLC_RERR_LO;
  59                 value = nitrox_read_csr(ndev, offset);
  60                 nitrox_write_csr(ndev, offset, value);
  61                 /* enable the solicit ports */
  62                 for_each_set_bit(i, &value, BITS_PER_LONG)
  63                         enable_pkt_solicit_port(ndev, i);
  64
  65                 dev_err_ratelimited(DEV(ndev),
  66                                     "NPS_PKT_SLC_RERR_LO  0x%016lx\n", value);
  67
  68                 offset = NPS_PKT_SLC_RERR_HI;
  69                 value = nitrox_read_csr(ndev, offset);
  70                 nitrox_write_csr(ndev, offset, value);
  71                 dev_err_ratelimited(DEV(ndev),
  72                                     "NPS_PKT_SLC_RERR_HI  0x%016lx\n", value);
  73         }
  74
  75         if (pkt_int.s.in_err) {
  76                 offset = NPS_PKT_IN_ERR_TYPE;
  77                 value = nitrox_read_csr(ndev, offset);
  78                 nitrox_write_csr(ndev, offset, value);
  79                 dev_err_ratelimited(DEV(ndev),
  80                                     "NPS_PKT_IN_ERR_TYPE  0x%016lx\n", value);
  81                 offset = NPS_PKT_IN_RERR_LO;
  82                 value = nitrox_read_csr(ndev, offset);
  83                 nitrox_write_csr(ndev, offset, value);
  84                 /* enable the input ring */
  85                 for_each_set_bit(i, &value, BITS_PER_LONG)
  86                         enable_pkt_input_ring(ndev, i);
  87
  88                 dev_err_ratelimited(DEV(ndev),
  89                                     "NPS_PKT_IN_RERR_LO  0x%016lx\n", value);
  90
  91                 offset = NPS_PKT_IN_RERR_HI;
  92                 value = nitrox_read_csr(ndev, offset);
  93                 nitrox_write_csr(ndev, offset, value);
  94                 dev_err_ratelimited(DEV(ndev),
  95                                     "NPS_PKT_IN_RERR_HI  0x%016lx\n", value);
  96         }
  97 }
  98
  99 static void clear_pom_err_intr(struct nitrox_device *ndev)
 100 {
 101         u64 value;
 102
 103         value = nitrox_read_csr(ndev, POM_INT);
 104         nitrox_write_csr(ndev, POM_INT, value);
 105         dev_err_ratelimited(DEV(ndev), "POM_INT  0x%016llx\n", value);
 106 }
 107
 108 static void clear_pem_err_intr(struct nitrox_device *ndev)
 109 {
 110         u64 value;
 111
 112         value = nitrox_read_csr(ndev, PEM0_INT);
 113         nitrox_write_csr(ndev, PEM0_INT, value);
 114         dev_err_ratelimited(DEV(ndev), "PEM(0)_INT  0x%016llx\n", value);
 115 }
 116
 117 static void clear_lbc_err_intr(struct nitrox_device *ndev)
 118 {
 119         union lbc_int lbc_int;
 120         u64 value, offset;
 121         int i;
 122
 123         lbc_int.value = nitrox_read_csr(ndev, LBC_INT);
 124         dev_err_ratelimited(DEV(ndev), "LBC_INT  0x%016llx\n", lbc_int.value);
 125
 126         if (lbc_int.s.dma_rd_err) {
 127                 for (i = 0; i < NR_CLUSTERS; i++) {
 128                         offset = EFL_CORE_VF_ERR_INT0X(i);
 129                         value = nitrox_read_csr(ndev, offset);
 130                         nitrox_write_csr(ndev, offset, value);
 131                         offset = EFL_CORE_VF_ERR_INT1X(i);
 132                         value = nitrox_read_csr(ndev, offset);
 133                         nitrox_write_csr(ndev, offset, value);
 134                 }
 135         }
 136
 137         if (lbc_int.s.cam_soft_err) {
 138                 dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n");
 139                 invalidate_lbc(ndev);
 140         }
 141
 142         if (lbc_int.s.pref_dat_len_mismatch_err) {
 143                 offset = LBC_PLM_VF1_64_INT;
 144                 value = nitrox_read_csr(ndev, offset);
 145                 nitrox_write_csr(ndev, offset, value);
 146                 offset = LBC_PLM_VF65_128_INT;
 147                 value = nitrox_read_csr(ndev, offset);
 148                 nitrox_write_csr(ndev, offset, value);
 149         }
 150
 151         if (lbc_int.s.rd_dat_len_mismatch_err) {
 152                 offset = LBC_ELM_VF1_64_INT;
 153                 value = nitrox_read_csr(ndev, offset);
 154                 nitrox_write_csr(ndev, offset, value);
 155                 offset = LBC_ELM_VF65_128_INT;
 156                 value = nitrox_read_csr(ndev, offset);
 157                 nitrox_write_csr(ndev, offset, value);
 158         }
 159         nitrox_write_csr(ndev, LBC_INT, lbc_int.value);
 160 }
 161
 162 static void clear_efl_err_intr(struct nitrox_device *ndev)
 163 {
 164         int i;
 165
 166         for (i = 0; i < NR_CLUSTERS; i++) {
 167                 union efl_core_int core_int;
 168                 u64 value, offset;
 169
 170                 offset = EFL_CORE_INTX(i);
 171                 core_int.value = nitrox_read_csr(ndev, offset);
 172                 nitrox_write_csr(ndev, offset, core_int.value);
 173                 dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT  0x%016llx\n",
 174                                     i, core_int.value);
 175                 if (core_int.s.se_err) {
 176                         offset = EFL_CORE_SE_ERR_INTX(i);
 177                         value = nitrox_read_csr(ndev, offset);
 178                         nitrox_write_csr(ndev, offset, value);
 179                 }
 180         }
 181 }
 182
 183 static void clear_bmi_err_intr(struct nitrox_device *ndev)
 184 {
 185         u64 value;
 186
 187         value = nitrox_read_csr(ndev, BMI_INT);
 188         nitrox_write_csr(ndev, BMI_INT, value);
 189         dev_err_ratelimited(DEV(ndev), "BMI_INT  0x%016llx\n", value);
 190 }
 191
 192 /**
 193  * clear_nps_core_int_active - clear NPS_CORE_INT_ACTIVE interrupts
 194  * @ndev: NITROX device
 195  */
 196 static void clear_nps_core_int_active(struct nitrox_device *ndev)
 197 {
 198         union nps_core_int_active core_int_active;
 199
 200         core_int_active.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
 201
 202         if (core_int_active.s.nps_core)
 203                 clear_nps_core_err_intr(ndev);
 204
 205         if (core_int_active.s.nps_pkt)
 206                 clear_nps_pkt_err_intr(ndev);
 207
 208         if (core_int_active.s.pom)
 209                 clear_pom_err_intr(ndev);
 210
 211         if (core_int_active.s.pem)
 212                 clear_pem_err_intr(ndev);
 213
 214         if (core_int_active.s.lbc)
 215                 clear_lbc_err_intr(ndev);
 216
 217         if (core_int_active.s.efl)
 218                 clear_efl_err_intr(ndev);
 219
 220         if (core_int_active.s.bmi)
 221                 clear_bmi_err_intr(ndev);
 222
 223         /* If more work callback the ISR, set resend */
 224         core_int_active.s.resend = 1;
 225         nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int_active.value);
 226 }
 227
 228 static irqreturn_t nps_core_int_isr(int irq, void *data)
 229 {
 230         struct nitrox_device *ndev = data;
 231
 232         clear_nps_core_int_active(ndev);
 233
 234         return IRQ_HANDLED;
 235 }
 236
 237 static int nitrox_enable_msix(struct nitrox_device *ndev)
 238 {
 239         struct msix_entry *entries;
 240         char **names;
 241         int i, nr_entries, ret;
 242
 243         /*
 244          * PF MSI-X vectors
 245          *
 246          * Entry 0: NPS PKT ring 0
 247          * Entry 1: AQMQ ring 0
 248          * Entry 2: ZQM ring 0
 249          * Entry 3: NPS PKT ring 1
 250          * Entry 4: AQMQ ring 1
 251          * Entry 5: ZQM ring 1
 252          * ....
 253          * Entry 192: NPS_CORE_INT_ACTIVE
 254          */
 255         nr_entries = (ndev->nr_queues * NR_RING_VECTORS) + 1;
 256         entries = kzalloc_node(nr_entries * sizeof(struct msix_entry),
 257                                GFP_KERNEL, ndev->node);
 258         if (!entries)
 259                 return -ENOMEM;
 260
 261         names = kcalloc(nr_entries, sizeof(char *), GFP_KERNEL);
 262         if (!names) {
 263                 kfree(entries);
 264                 return -ENOMEM;
 265         }
 266
 267         /* fill entires */
 268         for (i = 0; i < (nr_entries - 1); i++)
 269                 entries[i].entry = i;
 270
 271         entries[i].entry = NPS_CORE_INT_ACTIVE_ENTRY;
 272
 273         for (i = 0; i < nr_entries; i++) {
 274                 *(names + i) = kzalloc(MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
 275                 if (!(*(names + i))) {
 276                         ret = -ENOMEM;
 277                         goto msix_fail;
 278                 }
 279         }
 280         ndev->msix.entries = entries;
 281         ndev->msix.names = names;
 282         ndev->msix.nr_entries = nr_entries;
 283
 284         ret = pci_enable_msix_exact(ndev->pdev, ndev->msix.entries,
 285                                     ndev->msix.nr_entries);
 286         if (ret) {
 287                 dev_err(&ndev->pdev->dev, "Failed to enable MSI-X IRQ(s) %d\n",
 288                         ret);
 289                 goto msix_fail;
 290         }
 291         return 0;
 292
 293 msix_fail:
 294         for (i = 0; i < nr_entries; i++)
 295                 kfree(*(names + i));
 296
 297         kfree(entries);
 298         kfree(names);
 299         return ret;
 300 }
 301
 302 static void nitrox_cleanup_pkt_slc_bh(struct nitrox_device *ndev)
 303 {
 304         int i;
 305
 306         if (!ndev->bh.slc)
 307                 return;
 308
 309         for (i = 0; i < ndev->nr_queues; i++) {
 310                 struct bh_data *bh = &ndev->bh.slc[i];
 311
 312                 tasklet_disable(&bh->resp_handler);
 313                 tasklet_kill(&bh->resp_handler);
 314         }
 315         kfree(ndev->bh.slc);
 316         ndev->bh.slc = NULL;
 317 }
 318
 319 static int nitrox_setup_pkt_slc_bh(struct nitrox_device *ndev)
 320 {
 321         u32 size;
 322         int i;
 323
 324         size = ndev->nr_queues * sizeof(struct bh_data);
 325         ndev->bh.slc = kzalloc(size, GFP_KERNEL);
 326         if (!ndev->bh.slc)
 327                 return -ENOMEM;
 328
 329         for (i = 0; i < ndev->nr_queues; i++) {
 330                 struct bh_data *bh = &ndev->bh.slc[i];
 331                 u64 offset;
 332
 333                 offset = NPS_PKT_SLC_CNTSX(i);
 334                 /* pre calculate completion count address */
 335                 bh->completion_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
 336                 bh->cmdq = &ndev->pkt_cmdqs[i];
 337
 338                 tasklet_init(&bh->resp_handler, pkt_slc_resp_handler,
 339                              (unsigned long)bh);
 340         }
 341
 342         return 0;
 343 }
 344
 345 static int nitrox_request_irqs(struct nitrox_device *ndev)
 346 {
 347         struct pci_dev *pdev = ndev->pdev;
 348         struct msix_entry *msix_ent = ndev->msix.entries;
 349         int nr_ring_vectors, i = 0, ring, cpu, ret;
 350         char *name;
 351
 352         /*
 353          * PF MSI-X vectors
 354          *
 355          * Entry 0: NPS PKT ring 0
 356          * Entry 1: AQMQ ring 0
 357          * Entry 2: ZQM ring 0
 358          * Entry 3: NPS PKT ring 1
 359          * ....
 360          * Entry 192: NPS_CORE_INT_ACTIVE
 361          */
 362         nr_ring_vectors = ndev->nr_queues * NR_RING_VECTORS;
 363
 364         /* request irq for pkt ring/ports only */
 365         while (i < nr_ring_vectors) {
 366                 name = *(ndev->msix.names + i);
 367                 ring = (i / NR_RING_VECTORS);
 368                 snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-slc-ring%d",
 369                          ndev->idx, ring);
 370
 371                 ret = request_irq(msix_ent[i].vector, nps_pkt_slc_isr, 0,
 372                                   name, &ndev->bh.slc[ring]);
 373                 if (ret) {
 374                         dev_err(&pdev->dev, "failed to get irq %d for %s\n",
 375                                 msix_ent[i].vector, name);
 376                         return ret;
 377                 }
 378                 cpu = ring % num_online_cpus();
 379                 irq_set_affinity_hint(msix_ent[i].vector, get_cpu_mask(cpu));
 380
 381                 set_bit(i, ndev->msix.irqs);
 382                 i += NR_RING_VECTORS;
 383         }
 384
 385         /* Request IRQ for NPS_CORE_INT_ACTIVE */
 386         name = *(ndev->msix.names + i);
 387         snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-nps-core-int", ndev->idx);
 388         ret = request_irq(msix_ent[i].vector, nps_core_int_isr, 0, name, ndev);
 389         if (ret) {
 390                 dev_err(&pdev->dev, "failed to get irq %d for %s\n",
 391                         msix_ent[i].vector, name);
 392                 return ret;
 393         }
 394         set_bit(i, ndev->msix.irqs);
 395
 396         return 0;
 397 }
 398
 399 static void nitrox_disable_msix(struct nitrox_device *ndev)
 400 {
 401         struct msix_entry *msix_ent = ndev->msix.entries;
 402         char **names = ndev->msix.names;
 403         int i = 0, ring, nr_ring_vectors;
 404
 405         nr_ring_vectors = ndev->msix.nr_entries - 1;
 406
 407         /* clear pkt ring irqs */
 408         while (i < nr_ring_vectors) {
 409                 if (test_and_clear_bit(i, ndev->msix.irqs)) {
 410                         ring = (i / NR_RING_VECTORS);
 411                         irq_set_affinity_hint(msix_ent[i].vector, NULL);
 412                         free_irq(msix_ent[i].vector, &ndev->bh.slc[ring]);
 413                 }
 414                 i += NR_RING_VECTORS;
 415         }
 416         irq_set_affinity_hint(msix_ent[i].vector, NULL);
 417         free_irq(msix_ent[i].vector, ndev);
 418         clear_bit(i, ndev->msix.irqs);
 419
 420         kfree(ndev->msix.entries);
 421         for (i = 0; i < ndev->msix.nr_entries; i++)
 422                 kfree(*(names + i));
 423
 424         kfree(names);
 425         pci_disable_msix(ndev->pdev);
 426 }
 427
 428 /**
 429  * nitrox_pf_cleanup_isr: Cleanup PF MSI-X and IRQ
 430  * @ndev: NITROX device
 431  */
 432 void nitrox_pf_cleanup_isr(struct nitrox_device *ndev)
 433 {
 434         nitrox_disable_msix(ndev);
 435         nitrox_cleanup_pkt_slc_bh(ndev);
 436 }
 437
 438 /**
 439  * nitrox_init_isr - Initialize PF MSI-X vectors and IRQ
 440  * @ndev: NITROX device
 441  *
 442  * Return: 0 on success, a negative value on failure.
 443  */
 444 int nitrox_pf_init_isr(struct nitrox_device *ndev)
 445 {
 446         int err;
 447
 448         err = nitrox_setup_pkt_slc_bh(ndev);
 449         if (err)
 450                 return err;
 451
 452         err = nitrox_enable_msix(ndev);
 453         if (err)
 454                 goto msix_fail;
 455
 456         err = nitrox_request_irqs(ndev);
 457         if (err)
 458                 goto irq_fail;
 459
 460         return 0;
 461
 462 irq_fail:
 463         nitrox_disable_msix(ndev);
 464 msix_fail:
 465         nitrox_cleanup_pkt_slc_bh(ndev);
 466         return err;
 467 }