drivers/ntb/hw/intel/ntb_hw_gen1.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
   8  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
   9  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  10  *
  11  *   This program is free software; you can redistribute it and/or modify
  12  *   it under the terms of version 2 of the GNU General Public License as
  13  *   published by the Free Software Foundation.
  14  *
  15  *   BSD LICENSE
  16  *
  17  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  18  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  19  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  20  *
  21  *   Redistribution and use in source and binary forms, with or without
  22  *   modification, are permitted provided that the following conditions
  23  *   are met:
  24  *
  25  *     * Redistributions of source code must retain the above copyright
  26  *       notice, this list of conditions and the following disclaimer.
  27  *     * Redistributions in binary form must reproduce the above copy
  28  *       notice, this list of conditions and the following disclaimer in
  29  *       the documentation and/or other materials provided with the
  30  *       distribution.
  31  *     * Neither the name of Intel Corporation nor the names of its
  32  *       contributors may be used to endorse or promote products derived
  33  *       from this software without specific prior written permission.
  34  *
  35  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46  *
  47  * Intel PCIe NTB Linux driver
  48  */
  49
  50 #include <linux/debugfs.h>
  51 #include <linux/delay.h>
  52 #include <linux/init.h>
  53 #include <linux/interrupt.h>
  54 #include <linux/module.h>
  55 #include <linux/pci.h>
  56 #include <linux/random.h>
  57 #include <linux/slab.h>
  58 #include <linux/ntb.h>
  59
  60 #include "ntb_hw_intel.h"
  61 #include "ntb_hw_gen1.h"
  62 #include "ntb_hw_gen3.h"
  63
  64 #define NTB_NAME        "ntb_hw_intel"
  65 #define NTB_DESC        "Intel(R) PCI-E Non-Transparent Bridge Driver"
  66 #define NTB_VER         "2.0"
  67
  68 MODULE_DESCRIPTION(NTB_DESC);
  69 MODULE_VERSION(NTB_VER);
  70 MODULE_LICENSE("Dual BSD/GPL");
  71 MODULE_AUTHOR("Intel Corporation");
  72
  73 #define bar0_off(base, bar) ((base) + ((bar) << 2))
  74 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
  75
  76 static const struct intel_ntb_reg xeon_reg;
  77 static const struct intel_ntb_alt_reg xeon_pri_reg;
  78 static const struct intel_ntb_alt_reg xeon_sec_reg;
  79 static const struct intel_ntb_alt_reg xeon_b2b_reg;
  80 static const struct intel_ntb_xlat_reg xeon_pri_xlat;
  81 static const struct intel_ntb_xlat_reg xeon_sec_xlat;
  82 static const struct ntb_dev_ops intel_ntb_ops;
  83
  84 static const struct file_operations intel_ntb_debugfs_info;
  85 static struct dentry *debugfs_dir;
  86
  87 static int b2b_mw_idx = -1;
  88 module_param(b2b_mw_idx, int, 0644);
  89 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb.  A "
  90                  "value of zero or positive starts from first mw idx, and a "
  91                  "negative value starts from last mw idx.  Both sides MUST "
  92                  "set the same value here!");
  93
  94 static unsigned int b2b_mw_share;
  95 module_param(b2b_mw_share, uint, 0644);
  96 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
  97                  "ntb so that the peer ntb only occupies the first half of "
  98                  "the mw, so the second half can still be used as a mw.  Both "
  99                  "sides MUST set the same value here!");
 100
 101 module_param_named(xeon_b2b_usd_bar2_addr64,
 102                    xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
 103 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
 104                  "XEON B2B USD BAR 2 64-bit address");
 105
 106 module_param_named(xeon_b2b_usd_bar4_addr64,
 107                    xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
 108 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
 109                  "XEON B2B USD BAR 4 64-bit address");
 110
 111 module_param_named(xeon_b2b_usd_bar4_addr32,
 112                    xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
 113 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
 114                  "XEON B2B USD split-BAR 4 32-bit address");
 115
 116 module_param_named(xeon_b2b_usd_bar5_addr32,
 117                    xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
 118 MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
 119                  "XEON B2B USD split-BAR 5 32-bit address");
 120
 121 module_param_named(xeon_b2b_dsd_bar2_addr64,
 122                    xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
 123 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
 124                  "XEON B2B DSD BAR 2 64-bit address");
 125
 126 module_param_named(xeon_b2b_dsd_bar4_addr64,
 127                    xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
 128 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
 129                  "XEON B2B DSD BAR 4 64-bit address");
 130
 131 module_param_named(xeon_b2b_dsd_bar4_addr32,
 132                    xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
 133 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
 134                  "XEON B2B DSD split-BAR 4 32-bit address");
 135
 136 module_param_named(xeon_b2b_dsd_bar5_addr32,
 137                    xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
 138 MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
 139                  "XEON B2B DSD split-BAR 5 32-bit address");
 140
 141
 142 static int xeon_init_isr(struct intel_ntb_dev *ndev);
 143
 144 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
 145 {
 146         ndev->unsafe_flags = 0;
 147         ndev->unsafe_flags_ignore = 0;
 148
 149         /* Only B2B has a workaround to avoid SDOORBELL */
 150         if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
 151                 if (!ntb_topo_is_b2b(ndev->ntb.topo))
 152                         ndev->unsafe_flags |= NTB_UNSAFE_DB;
 153
 154         /* No low level workaround to avoid SB01BASE */
 155         if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
 156                 ndev->unsafe_flags |= NTB_UNSAFE_DB;
 157                 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
 158         }
 159 }
 160
 161 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
 162                                  unsigned long flag)
 163 {
 164         return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
 165 }
 166
 167 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
 168                                      unsigned long flag)
 169 {
 170         flag &= ndev->unsafe_flags;
 171         ndev->unsafe_flags_ignore |= flag;
 172
 173         return !!flag;
 174 }
 175
 176 int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
 177 {
 178         if (idx < 0 || idx >= ndev->mw_count)
 179                 return -EINVAL;
 180         return ndev->reg->mw_bar[idx];
 181 }
 182
 183 static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
 184                                phys_addr_t *db_addr, resource_size_t *db_size,
 185                                phys_addr_t reg_addr, unsigned long reg)
 186 {
 187         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 188                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 189
 190         if (db_addr) {
 191                 *db_addr = reg_addr + reg;
 192                 dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
 193         }
 194
 195         if (db_size) {
 196                 *db_size = ndev->reg->db_size;
 197                 dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
 198         }
 199
 200         return 0;
 201 }
 202
 203 u64 ndev_db_read(struct intel_ntb_dev *ndev,
 204                                void __iomem *mmio)
 205 {
 206         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 207                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 208
 209         return ndev->reg->db_ioread(mmio);
 210 }
 211
 212 int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
 213                                 void __iomem *mmio)
 214 {
 215         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 216                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 217
 218         if (db_bits & ~ndev->db_valid_mask)
 219                 return -EINVAL;
 220
 221         ndev->reg->db_iowrite(db_bits, mmio);
 222
 223         return 0;
 224 }
 225
 226 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 227                                    void __iomem *mmio)
 228 {
 229         unsigned long irqflags;
 230
 231         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 232                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 233
 234         if (db_bits & ~ndev->db_valid_mask)
 235                 return -EINVAL;
 236
 237         spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 238         {
 239                 ndev->db_mask |= db_bits;
 240                 ndev->reg->db_iowrite(ndev->db_mask, mmio);
 241         }
 242         spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 243
 244         return 0;
 245 }
 246
 247 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 248                                      void __iomem *mmio)
 249 {
 250         unsigned long irqflags;
 251
 252         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 253                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 254
 255         if (db_bits & ~ndev->db_valid_mask)
 256                 return -EINVAL;
 257
 258         spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 259         {
 260                 ndev->db_mask &= ~db_bits;
 261                 ndev->reg->db_iowrite(ndev->db_mask, mmio);
 262         }
 263         spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 264
 265         return 0;
 266 }
 267
 268 static inline u64 ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
 269 {
 270         u64 shift, mask;
 271
 272         shift = ndev->db_vec_shift;
 273         mask = BIT_ULL(shift) - 1;
 274
 275         return mask << (shift * db_vector);
 276 }
 277
 278 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
 279                                  phys_addr_t *spad_addr, phys_addr_t reg_addr,
 280                                  unsigned long reg)
 281 {
 282         if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 283                 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 284
 285         if (idx < 0 || idx >= ndev->spad_count)
 286                 return -EINVAL;
 287
 288         if (spad_addr) {
 289                 *spad_addr = reg_addr + reg + (idx << 2);
 290                 dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
 291                         *spad_addr);
 292         }
 293
 294         return 0;
 295 }
 296
 297 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
 298                                  void __iomem *mmio)
 299 {
 300         if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 301                 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 302
 303         if (idx < 0 || idx >= ndev->spad_count)
 304                 return 0;
 305
 306         return ioread32(mmio + (idx << 2));
 307 }
 308
 309 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
 310                                   void __iomem *mmio)
 311 {
 312         if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 313                 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 314
 315         if (idx < 0 || idx >= ndev->spad_count)
 316                 return -EINVAL;
 317
 318         iowrite32(val, mmio + (idx << 2));
 319
 320         return 0;
 321 }
 322
 323 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
 324 {
 325         u64 vec_mask;
 326
 327         vec_mask = ndev_vec_mask(ndev, vec);
 328
 329         if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
 330                 vec_mask |= ndev->db_link_mask;
 331
 332         dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
 333
 334         ndev->last_ts = jiffies;
 335
 336         if (vec_mask & ndev->db_link_mask) {
 337                 if (ndev->reg->poll_link(ndev))
 338                         ntb_link_event(&ndev->ntb);
 339         }
 340
 341         if (vec_mask & ndev->db_valid_mask)
 342                 ntb_db_event(&ndev->ntb, vec);
 343
 344         return IRQ_HANDLED;
 345 }
 346
 347 static irqreturn_t ndev_vec_isr(int irq, void *dev)
 348 {
 349         struct intel_ntb_vec *nvec = dev;
 350
 351         dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d  nvec->num: %d\n",
 352                 irq, nvec->num);
 353
 354         return ndev_interrupt(nvec->ndev, nvec->num);
 355 }
 356
 357 static irqreturn_t ndev_irq_isr(int irq, void *dev)
 358 {
 359         struct intel_ntb_dev *ndev = dev;
 360
 361         return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
 362 }
 363
 364 int ndev_init_isr(struct intel_ntb_dev *ndev,
 365                          int msix_min, int msix_max,
 366                          int msix_shift, int total_shift)
 367 {
 368         struct pci_dev *pdev;
 369         int rc, i, msix_count, node;
 370
 371         pdev = ndev->ntb.pdev;
 372
 373         node = dev_to_node(&pdev->dev);
 374
 375         /* Mask all doorbell interrupts */
 376         ndev->db_mask = ndev->db_valid_mask;
 377         ndev->reg->db_iowrite(ndev->db_mask,
 378                               ndev->self_mmio +
 379                               ndev->self_reg->db_mask);
 380
 381         /* Try to set up msix irq */
 382
 383         ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
 384                                  GFP_KERNEL, node);
 385         if (!ndev->vec)
 386                 goto err_msix_vec_alloc;
 387
 388         ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
 389                                   GFP_KERNEL, node);
 390         if (!ndev->msix)
 391                 goto err_msix_alloc;
 392
 393         for (i = 0; i < msix_max; ++i)
 394                 ndev->msix[i].entry = i;
 395
 396         msix_count = pci_enable_msix_range(pdev, ndev->msix,
 397                                            msix_min, msix_max);
 398         if (msix_count < 0)
 399                 goto err_msix_enable;
 400
 401         for (i = 0; i < msix_count; ++i) {
 402                 ndev->vec[i].ndev = ndev;
 403                 ndev->vec[i].num = i;
 404                 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
 405                                  "ndev_vec_isr", &ndev->vec[i]);
 406                 if (rc)
 407                         goto err_msix_request;
 408         }
 409
 410         dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
 411         ndev->db_vec_count = msix_count;
 412         ndev->db_vec_shift = msix_shift;
 413         return 0;
 414
 415 err_msix_request:
 416         while (i-- > 0)
 417                 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 418         pci_disable_msix(pdev);
 419 err_msix_enable:
 420         kfree(ndev->msix);
 421 err_msix_alloc:
 422         kfree(ndev->vec);
 423 err_msix_vec_alloc:
 424         ndev->msix = NULL;
 425         ndev->vec = NULL;
 426
 427         /* Try to set up msi irq */
 428
 429         rc = pci_enable_msi(pdev);
 430         if (rc)
 431                 goto err_msi_enable;
 432
 433         rc = request_irq(pdev->irq, ndev_irq_isr, 0,
 434                          "ndev_irq_isr", ndev);
 435         if (rc)
 436                 goto err_msi_request;
 437
 438         dev_dbg(&pdev->dev, "Using msi interrupts\n");
 439         ndev->db_vec_count = 1;
 440         ndev->db_vec_shift = total_shift;
 441         return 0;
 442
 443 err_msi_request:
 444         pci_disable_msi(pdev);
 445 err_msi_enable:
 446
 447         /* Try to set up intx irq */
 448
 449         pci_intx(pdev, 1);
 450
 451         rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
 452                          "ndev_irq_isr", ndev);
 453         if (rc)
 454                 goto err_intx_request;
 455
 456         dev_dbg(&pdev->dev, "Using intx interrupts\n");
 457         ndev->db_vec_count = 1;
 458         ndev->db_vec_shift = total_shift;
 459         return 0;
 460
 461 err_intx_request:
 462         return rc;
 463 }
 464
 465 static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
 466 {
 467         struct pci_dev *pdev;
 468         int i;
 469
 470         pdev = ndev->ntb.pdev;
 471
 472         /* Mask all doorbell interrupts */
 473         ndev->db_mask = ndev->db_valid_mask;
 474         ndev->reg->db_iowrite(ndev->db_mask,
 475                               ndev->self_mmio +
 476                               ndev->self_reg->db_mask);
 477
 478         if (ndev->msix) {
 479                 i = ndev->db_vec_count;
 480                 while (i--)
 481                         free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 482                 pci_disable_msix(pdev);
 483                 kfree(ndev->msix);
 484                 kfree(ndev->vec);
 485         } else {
 486                 free_irq(pdev->irq, ndev);
 487                 if (pci_dev_msi_enabled(pdev))
 488                         pci_disable_msi(pdev);
 489         }
 490 }
 491
 492 static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
 493                                      size_t count, loff_t *offp)
 494 {
 495         struct intel_ntb_dev *ndev;
 496         struct pci_dev *pdev;
 497         void __iomem *mmio;
 498         char *buf;
 499         size_t buf_size;
 500         ssize_t ret, off;
 501         union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
 502
 503         ndev = filp->private_data;
 504         pdev = ndev->ntb.pdev;
 505         mmio = ndev->self_mmio;
 506
 507         buf_size = min(count, 0x800ul);
 508
 509         buf = kmalloc(buf_size, GFP_KERNEL);
 510         if (!buf)
 511                 return -ENOMEM;
 512
 513         off = 0;
 514
 515         off += scnprintf(buf + off, buf_size - off,
 516                          "NTB Device Information:\n");
 517
 518         off += scnprintf(buf + off, buf_size - off,
 519                          "Connection Topology -\t%s\n",
 520                          ntb_topo_string(ndev->ntb.topo));
 521
 522         if (ndev->b2b_idx != UINT_MAX) {
 523                 off += scnprintf(buf + off, buf_size - off,
 524                                  "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
 525                 off += scnprintf(buf + off, buf_size - off,
 526                                  "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
 527         }
 528
 529         off += scnprintf(buf + off, buf_size - off,
 530                          "BAR4 Split -\t\t%s\n",
 531                          ndev->bar4_split ? "yes" : "no");
 532
 533         off += scnprintf(buf + off, buf_size - off,
 534                          "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
 535         off += scnprintf(buf + off, buf_size - off,
 536                          "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
 537
 538         if (!ndev->reg->link_is_up(ndev)) {
 539                 off += scnprintf(buf + off, buf_size - off,
 540                                  "Link Status -\t\tDown\n");
 541         } else {
 542                 off += scnprintf(buf + off, buf_size - off,
 543                                  "Link Status -\t\tUp\n");
 544                 off += scnprintf(buf + off, buf_size - off,
 545                                  "Link Speed -\t\tPCI-E Gen %u\n",
 546                                  NTB_LNK_STA_SPEED(ndev->lnk_sta));
 547                 off += scnprintf(buf + off, buf_size - off,
 548                                  "Link Width -\t\tx%u\n",
 549                                  NTB_LNK_STA_WIDTH(ndev->lnk_sta));
 550         }
 551
 552         off += scnprintf(buf + off, buf_size - off,
 553                          "Memory Window Count -\t%u\n", ndev->mw_count);
 554         off += scnprintf(buf + off, buf_size - off,
 555                          "Scratchpad Count -\t%u\n", ndev->spad_count);
 556         off += scnprintf(buf + off, buf_size - off,
 557                          "Doorbell Count -\t%u\n", ndev->db_count);
 558         off += scnprintf(buf + off, buf_size - off,
 559                          "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
 560         off += scnprintf(buf + off, buf_size - off,
 561                          "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
 562
 563         off += scnprintf(buf + off, buf_size - off,
 564                          "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
 565         off += scnprintf(buf + off, buf_size - off,
 566                          "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
 567         off += scnprintf(buf + off, buf_size - off,
 568                          "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
 569
 570         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
 571         off += scnprintf(buf + off, buf_size - off,
 572                          "Doorbell Mask -\t\t%#llx\n", u.v64);
 573
 574         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
 575         off += scnprintf(buf + off, buf_size - off,
 576                          "Doorbell Bell -\t\t%#llx\n", u.v64);
 577
 578         off += scnprintf(buf + off, buf_size - off,
 579                          "\nNTB Window Size:\n");
 580
 581         pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
 582         off += scnprintf(buf + off, buf_size - off,
 583                          "PBAR23SZ %hhu\n", u.v8);
 584         if (!ndev->bar4_split) {
 585                 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
 586                 off += scnprintf(buf + off, buf_size - off,
 587                                  "PBAR45SZ %hhu\n", u.v8);
 588         } else {
 589                 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
 590                 off += scnprintf(buf + off, buf_size - off,
 591                                  "PBAR4SZ %hhu\n", u.v8);
 592                 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
 593                 off += scnprintf(buf + off, buf_size - off,
 594                                  "PBAR5SZ %hhu\n", u.v8);
 595         }
 596
 597         pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
 598         off += scnprintf(buf + off, buf_size - off,
 599                          "SBAR23SZ %hhu\n", u.v8);
 600         if (!ndev->bar4_split) {
 601                 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
 602                 off += scnprintf(buf + off, buf_size - off,
 603                                  "SBAR45SZ %hhu\n", u.v8);
 604         } else {
 605                 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
 606                 off += scnprintf(buf + off, buf_size - off,
 607                                  "SBAR4SZ %hhu\n", u.v8);
 608                 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
 609                 off += scnprintf(buf + off, buf_size - off,
 610                                  "SBAR5SZ %hhu\n", u.v8);
 611         }
 612
 613         off += scnprintf(buf + off, buf_size - off,
 614                          "\nNTB Incoming XLAT:\n");
 615
 616         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
 617         off += scnprintf(buf + off, buf_size - off,
 618                          "XLAT23 -\t\t%#018llx\n", u.v64);
 619
 620         if (ndev->bar4_split) {
 621                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
 622                 off += scnprintf(buf + off, buf_size - off,
 623                                  "XLAT4 -\t\t\t%#06x\n", u.v32);
 624
 625                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
 626                 off += scnprintf(buf + off, buf_size - off,
 627                                  "XLAT5 -\t\t\t%#06x\n", u.v32);
 628         } else {
 629                 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
 630                 off += scnprintf(buf + off, buf_size - off,
 631                                  "XLAT45 -\t\t%#018llx\n", u.v64);
 632         }
 633
 634         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
 635         off += scnprintf(buf + off, buf_size - off,
 636                          "LMT23 -\t\t\t%#018llx\n", u.v64);
 637
 638         if (ndev->bar4_split) {
 639                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
 640                 off += scnprintf(buf + off, buf_size - off,
 641                                  "LMT4 -\t\t\t%#06x\n", u.v32);
 642                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
 643                 off += scnprintf(buf + off, buf_size - off,
 644                                  "LMT5 -\t\t\t%#06x\n", u.v32);
 645         } else {
 646                 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
 647                 off += scnprintf(buf + off, buf_size - off,
 648                                  "LMT45 -\t\t\t%#018llx\n", u.v64);
 649         }
 650
 651         if (pdev_is_gen1(pdev)) {
 652                 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
 653                         off += scnprintf(buf + off, buf_size - off,
 654                                          "\nNTB Outgoing B2B XLAT:\n");
 655
 656                         u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
 657                         off += scnprintf(buf + off, buf_size - off,
 658                                          "B2B XLAT23 -\t\t%#018llx\n", u.v64);
 659
 660                         if (ndev->bar4_split) {
 661                                 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
 662                                 off += scnprintf(buf + off, buf_size - off,
 663                                                  "B2B XLAT4 -\t\t%#06x\n",
 664                                                  u.v32);
 665                                 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
 666                                 off += scnprintf(buf + off, buf_size - off,
 667                                                  "B2B XLAT5 -\t\t%#06x\n",
 668                                                  u.v32);
 669                         } else {
 670                                 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
 671                                 off += scnprintf(buf + off, buf_size - off,
 672                                                  "B2B XLAT45 -\t\t%#018llx\n",
 673                                                  u.v64);
 674                         }
 675
 676                         u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
 677                         off += scnprintf(buf + off, buf_size - off,
 678                                          "B2B LMT23 -\t\t%#018llx\n", u.v64);
 679
 680                         if (ndev->bar4_split) {
 681                                 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
 682                                 off += scnprintf(buf + off, buf_size - off,
 683                                                  "B2B LMT4 -\t\t%#06x\n",
 684                                                  u.v32);
 685                                 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
 686                                 off += scnprintf(buf + off, buf_size - off,
 687                                                  "B2B LMT5 -\t\t%#06x\n",
 688                                                  u.v32);
 689                         } else {
 690                                 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
 691                                 off += scnprintf(buf + off, buf_size - off,
 692                                                  "B2B LMT45 -\t\t%#018llx\n",
 693                                                  u.v64);
 694                         }
 695
 696                         off += scnprintf(buf + off, buf_size - off,
 697                                          "\nNTB Secondary BAR:\n");
 698
 699                         u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
 700                         off += scnprintf(buf + off, buf_size - off,
 701                                          "SBAR01 -\t\t%#018llx\n", u.v64);
 702
 703                         u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
 704                         off += scnprintf(buf + off, buf_size - off,
 705                                          "SBAR23 -\t\t%#018llx\n", u.v64);
 706
 707                         if (ndev->bar4_split) {
 708                                 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
 709                                 off += scnprintf(buf + off, buf_size - off,
 710                                                  "SBAR4 -\t\t\t%#06x\n", u.v32);
 711                                 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
 712                                 off += scnprintf(buf + off, buf_size - off,
 713                                                  "SBAR5 -\t\t\t%#06x\n", u.v32);
 714                         } else {
 715                                 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
 716                                 off += scnprintf(buf + off, buf_size - off,
 717                                                  "SBAR45 -\t\t%#018llx\n",
 718                                                  u.v64);
 719                         }
 720                 }
 721
 722                 off += scnprintf(buf + off, buf_size - off,
 723                                  "\nXEON NTB Statistics:\n");
 724
 725                 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
 726                 off += scnprintf(buf + off, buf_size - off,
 727                                  "Upstream Memory Miss -\t%u\n", u.v16);
 728
 729                 off += scnprintf(buf + off, buf_size - off,
 730                                  "\nXEON NTB Hardware Errors:\n");
 731
 732                 if (!pci_read_config_word(pdev,
 733                                           XEON_DEVSTS_OFFSET, &u.v16))
 734                         off += scnprintf(buf + off, buf_size - off,
 735                                          "DEVSTS -\t\t%#06x\n", u.v16);
 736
 737                 if (!pci_read_config_word(pdev,
 738                                           XEON_LINK_STATUS_OFFSET, &u.v16))
 739                         off += scnprintf(buf + off, buf_size - off,
 740                                          "LNKSTS -\t\t%#06x\n", u.v16);
 741
 742                 if (!pci_read_config_dword(pdev,
 743                                            XEON_UNCERRSTS_OFFSET, &u.v32))
 744                         off += scnprintf(buf + off, buf_size - off,
 745                                          "UNCERRSTS -\t\t%#06x\n", u.v32);
 746
 747                 if (!pci_read_config_dword(pdev,
 748                                            XEON_CORERRSTS_OFFSET, &u.v32))
 749                         off += scnprintf(buf + off, buf_size - off,
 750                                          "CORERRSTS -\t\t%#06x\n", u.v32);
 751         }
 752
 753         ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
 754         kfree(buf);
 755         return ret;
 756 }
 757
 758 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
 759                                  size_t count, loff_t *offp)
 760 {
 761         struct intel_ntb_dev *ndev = filp->private_data;
 762
 763         if (pdev_is_gen1(ndev->ntb.pdev))
 764                 return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
 765         else if (pdev_is_gen3(ndev->ntb.pdev))
 766                 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
 767
 768         return -ENXIO;
 769 }
 770
 771 static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
 772 {
 773         if (!debugfs_dir) {
 774                 ndev->debugfs_dir = NULL;
 775                 ndev->debugfs_info = NULL;
 776         } else {
 777                 ndev->debugfs_dir =
 778                         debugfs_create_dir(pci_name(ndev->ntb.pdev),
 779                                            debugfs_dir);
 780                 if (!ndev->debugfs_dir)
 781                         ndev->debugfs_info = NULL;
 782                 else
 783                         ndev->debugfs_info =
 784                                 debugfs_create_file("info", S_IRUSR,
 785                                                     ndev->debugfs_dir, ndev,
 786                                                     &intel_ntb_debugfs_info);
 787         }
 788 }
 789
 790 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
 791 {
 792         debugfs_remove_recursive(ndev->debugfs_dir);
 793 }
 794
 795 int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 796 {
 797         if (pidx != NTB_DEF_PEER_IDX)
 798                 return -EINVAL;
 799
 800         return ntb_ndev(ntb)->mw_count;
 801 }
 802
 803 int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
 804                            resource_size_t *addr_align,
 805                            resource_size_t *size_align,
 806                            resource_size_t *size_max)
 807 {
 808         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 809         resource_size_t bar_size, mw_size;
 810         int bar;
 811
 812         if (pidx != NTB_DEF_PEER_IDX)
 813                 return -EINVAL;
 814
 815         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
 816                 idx += 1;
 817
 818         bar = ndev_mw_to_bar(ndev, idx);
 819         if (bar < 0)
 820                 return bar;
 821
 822         bar_size = pci_resource_len(ndev->ntb.pdev, bar);
 823
 824         if (idx == ndev->b2b_idx)
 825                 mw_size = bar_size - ndev->b2b_off;
 826         else
 827                 mw_size = bar_size;
 828
 829         if (addr_align)
 830                 *addr_align = pci_resource_len(ndev->ntb.pdev, bar);
 831
 832         if (size_align)
 833                 *size_align = 1;
 834
 835         if (size_max)
 836                 *size_max = mw_size;
 837
 838         return 0;
 839 }
 840
 841 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
 842                                   dma_addr_t addr, resource_size_t size)
 843 {
 844         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 845         unsigned long base_reg, xlat_reg, limit_reg;
 846         resource_size_t bar_size, mw_size;
 847         void __iomem *mmio;
 848         u64 base, limit, reg_val;
 849         int bar;
 850
 851         if (pidx != NTB_DEF_PEER_IDX)
 852                 return -EINVAL;
 853
 854         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
 855                 idx += 1;
 856
 857         bar = ndev_mw_to_bar(ndev, idx);
 858         if (bar < 0)
 859                 return bar;
 860
 861         bar_size = pci_resource_len(ndev->ntb.pdev, bar);
 862
 863         if (idx == ndev->b2b_idx)
 864                 mw_size = bar_size - ndev->b2b_off;
 865         else
 866                 mw_size = bar_size;
 867
 868         /* hardware requires that addr is aligned to bar size */
 869         if (addr & (bar_size - 1))
 870                 return -EINVAL;
 871
 872         /* make sure the range fits in the usable mw size */
 873         if (size > mw_size)
 874                 return -EINVAL;
 875
 876         mmio = ndev->self_mmio;
 877         base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
 878         xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
 879         limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
 880
 881         if (bar < 4 || !ndev->bar4_split) {
 882                 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
 883
 884                 /* Set the limit if supported, if size is not mw_size */
 885                 if (limit_reg && size != mw_size)
 886                         limit = base + size;
 887                 else
 888                         limit = 0;
 889
 890                 /* set and verify setting the translation address */
 891                 iowrite64(addr, mmio + xlat_reg);
 892                 reg_val = ioread64(mmio + xlat_reg);
 893                 if (reg_val != addr) {
 894                         iowrite64(0, mmio + xlat_reg);
 895                         return -EIO;
 896                 }
 897
 898                 /* set and verify setting the limit */
 899                 iowrite64(limit, mmio + limit_reg);
 900                 reg_val = ioread64(mmio + limit_reg);
 901                 if (reg_val != limit) {
 902                         iowrite64(base, mmio + limit_reg);
 903                         iowrite64(0, mmio + xlat_reg);
 904                         return -EIO;
 905                 }
 906         } else {
 907                 /* split bar addr range must all be 32 bit */
 908                 if (addr & (~0ull << 32))
 909                         return -EINVAL;
 910                 if ((addr + size) & (~0ull << 32))
 911                         return -EINVAL;
 912
 913                 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
 914
 915                 /* Set the limit if supported, if size is not mw_size */
 916                 if (limit_reg && size != mw_size)
 917                         limit = base + size;
 918                 else
 919                         limit = 0;
 920
 921                 /* set and verify setting the translation address */
 922                 iowrite32(addr, mmio + xlat_reg);
 923                 reg_val = ioread32(mmio + xlat_reg);
 924                 if (reg_val != addr) {
 925                         iowrite32(0, mmio + xlat_reg);
 926                         return -EIO;
 927                 }
 928
 929                 /* set and verify setting the limit */
 930                 iowrite32(limit, mmio + limit_reg);
 931                 reg_val = ioread32(mmio + limit_reg);
 932                 if (reg_val != limit) {
 933                         iowrite32(base, mmio + limit_reg);
 934                         iowrite32(0, mmio + xlat_reg);
 935                         return -EIO;
 936                 }
 937         }
 938
 939         return 0;
 940 }
 941
 942 u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed,
 943                          enum ntb_width *width)
 944 {
 945         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 946
 947         if (ndev->reg->link_is_up(ndev)) {
 948                 if (speed)
 949                         *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
 950                 if (width)
 951                         *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
 952                 return 1;
 953         } else {
 954                 /* TODO MAYBE: is it possible to observe the link speed and
 955                  * width while link is training? */
 956                 if (speed)
 957                         *speed = NTB_SPEED_NONE;
 958                 if (width)
 959                         *width = NTB_WIDTH_NONE;
 960                 return 0;
 961         }
 962 }
 963
 964 static int intel_ntb_link_enable(struct ntb_dev *ntb,
 965                                  enum ntb_speed max_speed,
 966                                  enum ntb_width max_width)
 967 {
 968         struct intel_ntb_dev *ndev;
 969         u32 ntb_ctl;
 970
 971         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
 972
 973         if (ndev->ntb.topo == NTB_TOPO_SEC)
 974                 return -EINVAL;
 975
 976         dev_dbg(&ntb->pdev->dev,
 977                 "Enabling link with max_speed %d max_width %d\n",
 978                 max_speed, max_width);
 979         if (max_speed != NTB_SPEED_AUTO)
 980                 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
 981         if (max_width != NTB_WIDTH_AUTO)
 982                 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
 983
 984         ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
 985         ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
 986         ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
 987         ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
 988         if (ndev->bar4_split)
 989                 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
 990         iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
 991
 992         return 0;
 993 }
 994
 995 int intel_ntb_link_disable(struct ntb_dev *ntb)
 996 {
 997         struct intel_ntb_dev *ndev;
 998         u32 ntb_cntl;
 999
1000         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1001
1002         if (ndev->ntb.topo == NTB_TOPO_SEC)
1003                 return -EINVAL;
1004
1005         dev_dbg(&ntb->pdev->dev, "Disabling link\n");
1006
1007         /* Bring NTB link down */
1008         ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1009         ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
1010         ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
1011         if (ndev->bar4_split)
1012                 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
1013         ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
1014         iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1015
1016         return 0;
1017 }
1018
1019 int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
1020 {
1021         /* Numbers of inbound and outbound memory windows match */
1022         return ntb_ndev(ntb)->mw_count;
1023 }
1024
1025 int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
1026                                phys_addr_t *base, resource_size_t *size)
1027 {
1028         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1029         int bar;
1030
1031         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1032                 idx += 1;
1033
1034         bar = ndev_mw_to_bar(ndev, idx);
1035         if (bar < 0)
1036                 return bar;
1037
1038         if (base)
1039                 *base = pci_resource_start(ndev->ntb.pdev, bar) +
1040                         (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1041
1042         if (size)
1043                 *size = pci_resource_len(ndev->ntb.pdev, bar) -
1044                         (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1045
1046         return 0;
1047 }
1048
1049 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1050 {
1051         return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1052 }
1053
1054 u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1055 {
1056         return ntb_ndev(ntb)->db_valid_mask;
1057 }
1058
1059 int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1060 {
1061         struct intel_ntb_dev *ndev;
1062
1063         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1064
1065         return ndev->db_vec_count;
1066 }
1067
1068 u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1069 {
1070         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1071
1072         if (db_vector < 0 || db_vector > ndev->db_vec_count)
1073                 return 0;
1074
1075         return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1076 }
1077
1078 static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1079 {
1080         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1081
1082         return ndev_db_read(ndev,
1083                             ndev->self_mmio +
1084                             ndev->self_reg->db_bell);
1085 }
1086
1087 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1088 {
1089         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1090
1091         return ndev_db_write(ndev, db_bits,
1092                              ndev->self_mmio +
1093                              ndev->self_reg->db_bell);
1094 }
1095
1096 int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1097 {
1098         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1099
1100         return ndev_db_set_mask(ndev, db_bits,
1101                                 ndev->self_mmio +
1102                                 ndev->self_reg->db_mask);
1103 }
1104
1105 int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1106 {
1107         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1108
1109         return ndev_db_clear_mask(ndev, db_bits,
1110                                   ndev->self_mmio +
1111                                   ndev->self_reg->db_mask);
1112 }
1113
1114 int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
1115                            resource_size_t *db_size)
1116 {
1117         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1118
1119         return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1120                             ndev->peer_reg->db_bell);
1121 }
1122
1123 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1124 {
1125         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1126
1127         return ndev_db_write(ndev, db_bits,
1128                              ndev->peer_mmio +
1129                              ndev->peer_reg->db_bell);
1130 }
1131
1132 int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1133 {
1134         return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1135 }
1136
1137 int intel_ntb_spad_count(struct ntb_dev *ntb)
1138 {
1139         struct intel_ntb_dev *ndev;
1140
1141         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1142
1143         return ndev->spad_count;
1144 }
1145
1146 u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1147 {
1148         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1149
1150         return ndev_spad_read(ndev, idx,
1151                               ndev->self_mmio +
1152                               ndev->self_reg->spad);
1153 }
1154
1155 int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
1156 {
1157         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1158
1159         return ndev_spad_write(ndev, idx, val,
1160                                ndev->self_mmio +
1161                                ndev->self_reg->spad);
1162 }
1163
1164 int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
1165                              phys_addr_t *spad_addr)
1166 {
1167         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1168
1169         return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
1170                               ndev->peer_reg->spad);
1171 }
1172
1173 u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
1174 {
1175         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1176
1177         return ndev_spad_read(ndev, sidx,
1178                               ndev->peer_mmio +
1179                               ndev->peer_reg->spad);
1180 }
1181
1182 int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx,
1183                               u32 val)
1184 {
1185         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1186
1187         return ndev_spad_write(ndev, sidx, val,
1188                                ndev->peer_mmio +
1189                                ndev->peer_reg->spad);
1190 }
1191
1192 static u64 xeon_db_ioread(void __iomem *mmio)
1193 {
1194         return (u64)ioread16(mmio);
1195 }
1196
1197 static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
1198 {
1199         iowrite16((u16)bits, mmio);
1200 }
1201
1202 static int xeon_poll_link(struct intel_ntb_dev *ndev)
1203 {
1204         u16 reg_val;
1205         int rc;
1206
1207         ndev->reg->db_iowrite(ndev->db_link_mask,
1208                               ndev->self_mmio +
1209                               ndev->self_reg->db_bell);
1210
1211         rc = pci_read_config_word(ndev->ntb.pdev,
1212                                   XEON_LINK_STATUS_OFFSET, &reg_val);
1213         if (rc)
1214                 return 0;
1215
1216         if (reg_val == ndev->lnk_sta)
1217                 return 0;
1218
1219         ndev->lnk_sta = reg_val;
1220
1221         return 1;
1222 }
1223
1224 int xeon_link_is_up(struct intel_ntb_dev *ndev)
1225 {
1226         if (ndev->ntb.topo == NTB_TOPO_SEC)
1227                 return 1;
1228
1229         return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
1230 }
1231
1232 enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1233 {
1234         switch (ppd & XEON_PPD_TOPO_MASK) {
1235         case XEON_PPD_TOPO_B2B_USD:
1236                 return NTB_TOPO_B2B_USD;
1237
1238         case XEON_PPD_TOPO_B2B_DSD:
1239                 return NTB_TOPO_B2B_DSD;
1240
1241         case XEON_PPD_TOPO_PRI_USD:
1242         case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1243                 return NTB_TOPO_PRI;
1244
1245         case XEON_PPD_TOPO_SEC_USD:
1246         case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1247                 return NTB_TOPO_SEC;
1248         }
1249
1250         return NTB_TOPO_NONE;
1251 }
1252
1253 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1254 {
1255         if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
1256                 dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
1257                 return 1;
1258         }
1259         return 0;
1260 }
1261
1262 static int xeon_init_isr(struct intel_ntb_dev *ndev)
1263 {
1264         return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
1265                              XEON_DB_MSIX_VECTOR_COUNT,
1266                              XEON_DB_MSIX_VECTOR_SHIFT,
1267                              XEON_DB_TOTAL_SHIFT);
1268 }
1269
1270 static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
1271 {
1272         ndev_deinit_isr(ndev);
1273 }
1274
1275 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
1276                              const struct intel_b2b_addr *addr,
1277                              const struct intel_b2b_addr *peer_addr)
1278 {
1279         struct pci_dev *pdev;
1280         void __iomem *mmio;
1281         resource_size_t bar_size;
1282         phys_addr_t bar_addr;
1283         int b2b_bar;
1284         u8 bar_sz;
1285
1286         pdev = ndev->ntb.pdev;
1287         mmio = ndev->self_mmio;
1288
1289         if (ndev->b2b_idx == UINT_MAX) {
1290                 dev_dbg(&pdev->dev, "not using b2b mw\n");
1291                 b2b_bar = 0;
1292                 ndev->b2b_off = 0;
1293         } else {
1294                 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1295                 if (b2b_bar < 0)
1296                         return -EIO;
1297
1298                 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
1299
1300                 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1301
1302                 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
1303
1304                 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
1305                         dev_dbg(&pdev->dev, "b2b using first half of bar\n");
1306                         ndev->b2b_off = bar_size >> 1;
1307                 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
1308                         dev_dbg(&pdev->dev, "b2b using whole bar\n");
1309                         ndev->b2b_off = 0;
1310                         --ndev->mw_count;
1311                 } else {
1312                         dev_dbg(&pdev->dev, "b2b bar size is too small\n");
1313                         return -EIO;
1314                 }
1315         }
1316
1317         /* Reset the secondary bar sizes to match the primary bar sizes,
1318          * except disable or halve the size of the b2b secondary bar.
1319          *
1320          * Note: code for each specific bar size register, because the register
1321          * offsets are not in a consistent order (bar5sz comes after ppd, odd).
1322          */
1323         pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
1324         dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
1325         if (b2b_bar == 2) {
1326                 if (ndev->b2b_off)
1327                         bar_sz -= 1;
1328                 else
1329                         bar_sz = 0;
1330         }
1331         pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
1332         pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
1333         dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
1334
1335         if (!ndev->bar4_split) {
1336                 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
1337                 dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
1338                 if (b2b_bar == 4) {
1339                         if (ndev->b2b_off)
1340                                 bar_sz -= 1;
1341                         else
1342                                 bar_sz = 0;
1343                 }
1344                 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
1345                 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
1346                 dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
1347         } else {
1348                 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
1349                 dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
1350                 if (b2b_bar == 4) {
1351                         if (ndev->b2b_off)
1352                                 bar_sz -= 1;
1353                         else
1354                                 bar_sz = 0;
1355                 }
1356                 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
1357                 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
1358                 dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
1359
1360                 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
1361                 dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
1362                 if (b2b_bar == 5) {
1363                         if (ndev->b2b_off)
1364                                 bar_sz -= 1;
1365                         else
1366                                 bar_sz = 0;
1367                 }
1368                 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
1369                 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
1370                 dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
1371         }
1372
1373         /* SBAR01 hit by first part of the b2b bar */
1374         if (b2b_bar == 0)
1375                 bar_addr = addr->bar0_addr;
1376         else if (b2b_bar == 2)
1377                 bar_addr = addr->bar2_addr64;
1378         else if (b2b_bar == 4 && !ndev->bar4_split)
1379                 bar_addr = addr->bar4_addr64;
1380         else if (b2b_bar == 4)
1381                 bar_addr = addr->bar4_addr32;
1382         else if (b2b_bar == 5)
1383                 bar_addr = addr->bar5_addr32;
1384         else
1385                 return -EIO;
1386
1387         dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
1388         iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
1389
1390         /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
1391          * The b2b bar is either disabled above, or configured half-size, and
1392          * it starts at the PBAR xlat + offset.
1393          */
1394
1395         bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1396         iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
1397         bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
1398         dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
1399
1400         if (!ndev->bar4_split) {
1401                 bar_addr = addr->bar4_addr64 +
1402                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1403                 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
1404                 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
1405                 dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
1406         } else {
1407                 bar_addr = addr->bar4_addr32 +
1408                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1409                 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
1410                 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
1411                 dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
1412
1413                 bar_addr = addr->bar5_addr32 +
1414                         (b2b_bar == 5 ? ndev->b2b_off : 0);
1415                 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
1416                 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
1417                 dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
1418         }
1419
1420         /* setup incoming bar limits == base addrs (zero length windows) */
1421
1422         bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1423         iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
1424         bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
1425         dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
1426
1427         if (!ndev->bar4_split) {
1428                 bar_addr = addr->bar4_addr64 +
1429                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1430                 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
1431                 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
1432                 dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
1433         } else {
1434                 bar_addr = addr->bar4_addr32 +
1435                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1436                 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
1437                 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
1438                 dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
1439
1440                 bar_addr = addr->bar5_addr32 +
1441                         (b2b_bar == 5 ? ndev->b2b_off : 0);
1442                 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
1443                 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
1444                 dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
1445         }
1446
1447         /* zero incoming translation addrs */
1448         iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
1449
1450         if (!ndev->bar4_split) {
1451                 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
1452         } else {
1453                 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
1454                 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
1455         }
1456
1457         /* zero outgoing translation limits (whole bar size windows) */
1458         iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
1459         if (!ndev->bar4_split) {
1460                 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
1461         } else {
1462                 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
1463                 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
1464         }
1465
1466         /* set outgoing translation offsets */
1467         bar_addr = peer_addr->bar2_addr64;
1468         iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
1469         bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
1470         dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
1471
1472         if (!ndev->bar4_split) {
1473                 bar_addr = peer_addr->bar4_addr64;
1474                 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
1475                 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
1476                 dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
1477         } else {
1478                 bar_addr = peer_addr->bar4_addr32;
1479                 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
1480                 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
1481                 dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
1482
1483                 bar_addr = peer_addr->bar5_addr32;
1484                 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
1485                 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
1486                 dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
1487         }
1488
1489         /* set the translation offset for b2b registers */
1490         if (b2b_bar == 0)
1491                 bar_addr = peer_addr->bar0_addr;
1492         else if (b2b_bar == 2)
1493                 bar_addr = peer_addr->bar2_addr64;
1494         else if (b2b_bar == 4 && !ndev->bar4_split)
1495                 bar_addr = peer_addr->bar4_addr64;
1496         else if (b2b_bar == 4)
1497                 bar_addr = peer_addr->bar4_addr32;
1498         else if (b2b_bar == 5)
1499                 bar_addr = peer_addr->bar5_addr32;
1500         else
1501                 return -EIO;
1502
1503         /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
1504         dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
1505         iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
1506         iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
1507
1508         if (b2b_bar) {
1509                 /* map peer ntb mmio config space registers */
1510                 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1511                                             XEON_B2B_MIN_SIZE);
1512                 if (!ndev->peer_mmio)
1513                         return -EIO;
1514
1515                 ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
1516         }
1517
1518         return 0;
1519 }
1520
1521 static int xeon_init_ntb(struct intel_ntb_dev *ndev)
1522 {
1523         struct device *dev = &ndev->ntb.pdev->dev;
1524         int rc;
1525         u32 ntb_ctl;
1526
1527         if (ndev->bar4_split)
1528                 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1529         else
1530                 ndev->mw_count = XEON_MW_COUNT;
1531
1532         ndev->spad_count = XEON_SPAD_COUNT;
1533         ndev->db_count = XEON_DB_COUNT;
1534         ndev->db_link_mask = XEON_DB_LINK_BIT;
1535
1536         switch (ndev->ntb.topo) {
1537         case NTB_TOPO_PRI:
1538                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1539                         dev_err(dev, "NTB Primary config disabled\n");
1540                         return -EINVAL;
1541                 }
1542
1543                 /* enable link to allow secondary side device to appear */
1544                 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1545                 ntb_ctl &= ~NTB_CTL_DISABLE;
1546                 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1547
1548                 /* use half the spads for the peer */
1549                 ndev->spad_count >>= 1;
1550                 ndev->self_reg = &xeon_pri_reg;
1551                 ndev->peer_reg = &xeon_sec_reg;
1552                 ndev->xlat_reg = &xeon_sec_xlat;
1553                 break;
1554
1555         case NTB_TOPO_SEC:
1556                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1557                         dev_err(dev, "NTB Secondary config disabled\n");
1558                         return -EINVAL;
1559                 }
1560                 /* use half the spads for the peer */
1561                 ndev->spad_count >>= 1;
1562                 ndev->self_reg = &xeon_sec_reg;
1563                 ndev->peer_reg = &xeon_pri_reg;
1564                 ndev->xlat_reg = &xeon_pri_xlat;
1565                 break;
1566
1567         case NTB_TOPO_B2B_USD:
1568         case NTB_TOPO_B2B_DSD:
1569                 ndev->self_reg = &xeon_pri_reg;
1570                 ndev->peer_reg = &xeon_b2b_reg;
1571                 ndev->xlat_reg = &xeon_sec_xlat;
1572
1573                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1574                         ndev->peer_reg = &xeon_pri_reg;
1575
1576                         if (b2b_mw_idx < 0)
1577                                 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
1578                         else
1579                                 ndev->b2b_idx = b2b_mw_idx;
1580
1581                         if (ndev->b2b_idx >= ndev->mw_count) {
1582                                 dev_dbg(dev,
1583                                         "b2b_mw_idx %d invalid for mw_count %u\n",
1584                                         b2b_mw_idx, ndev->mw_count);
1585                                 return -EINVAL;
1586                         }
1587
1588                         dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
1589                                 b2b_mw_idx, ndev->b2b_idx);
1590
1591                 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
1592                         dev_warn(dev, "Reduce doorbell count by 1\n");
1593                         ndev->db_count -= 1;
1594                 }
1595
1596                 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1597                         rc = xeon_setup_b2b_mw(ndev,
1598                                                &xeon_b2b_dsd_addr,
1599                                                &xeon_b2b_usd_addr);
1600                 } else {
1601                         rc = xeon_setup_b2b_mw(ndev,
1602                                                &xeon_b2b_usd_addr,
1603                                                &xeon_b2b_dsd_addr);
1604                 }
1605                 if (rc)
1606                         return rc;
1607
1608                 /* Enable Bus Master and Memory Space on the secondary side */
1609                 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1610                           ndev->self_mmio + XEON_SPCICMD_OFFSET);
1611
1612                 break;
1613
1614         default:
1615                 return -EINVAL;
1616         }
1617
1618         ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1619
1620         ndev->reg->db_iowrite(ndev->db_valid_mask,
1621                               ndev->self_mmio +
1622                               ndev->self_reg->db_mask);
1623
1624         return 0;
1625 }
1626
1627 static int xeon_init_dev(struct intel_ntb_dev *ndev)
1628 {
1629         struct pci_dev *pdev;
1630         u8 ppd;
1631         int rc, mem;
1632
1633         pdev = ndev->ntb.pdev;
1634
1635         switch (pdev->device) {
1636         /* There is a Xeon hardware errata related to writes to SDOORBELL or
1637          * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
1638          * which may hang the system.  To workaround this use the second memory
1639          * window to access the interrupt and scratch pad registers on the
1640          * remote system.
1641          */
1642         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1643         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1644         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1645         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1646         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1647         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1648         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1649         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1650         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1651         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1652         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1653         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1654         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1655         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1656         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1657                 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
1658                 break;
1659         }
1660
1661         switch (pdev->device) {
1662         /* There is a hardware errata related to accessing any register in
1663          * SB01BASE in the presence of bidirectional traffic crossing the NTB.
1664          */
1665         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1666         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1667         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1668         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1669         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1670         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1671         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1672         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1673         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1674                 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
1675                 break;
1676         }
1677
1678         switch (pdev->device) {
1679         /* HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
1680          * mirrored to the remote system.  Shrink the number of bits by one,
1681          * since bit 14 is the last bit.
1682          */
1683         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1684         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1685         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1686         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1687         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1688         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1689         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1690         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1691         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1692         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1693         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1694         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1695         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1696         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1697         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1698                 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
1699                 break;
1700         }
1701
1702         ndev->reg = &xeon_reg;
1703
1704         rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1705         if (rc)
1706                 return -EIO;
1707
1708         ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1709         dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
1710                 ntb_topo_string(ndev->ntb.topo));
1711         if (ndev->ntb.topo == NTB_TOPO_NONE)
1712                 return -EINVAL;
1713
1714         if (ndev->ntb.topo != NTB_TOPO_SEC) {
1715                 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
1716                 dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
1717                         ppd, ndev->bar4_split);
1718         } else {
1719                 /* This is a way for transparent BAR to figure out if we are
1720                  * doing split BAR or not. There is no way for the hw on the
1721                  * transparent side to know and set the PPD.
1722                  */
1723                 mem = pci_select_bars(pdev, IORESOURCE_MEM);
1724                 ndev->bar4_split = hweight32(mem) ==
1725                         HSX_SPLIT_BAR_MW_COUNT + 1;
1726                 dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
1727                         mem, ndev->bar4_split);
1728         }
1729
1730         rc = xeon_init_ntb(ndev);
1731         if (rc)
1732                 return rc;
1733
1734         return xeon_init_isr(ndev);
1735 }
1736
1737 static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
1738 {
1739         xeon_deinit_isr(ndev);
1740 }
1741
1742 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1743 {
1744         int rc;
1745
1746         pci_set_drvdata(pdev, ndev);
1747
1748         rc = pci_enable_device(pdev);
1749         if (rc)
1750                 goto err_pci_enable;
1751
1752         rc = pci_request_regions(pdev, NTB_NAME);
1753         if (rc)
1754                 goto err_pci_regions;
1755
1756         pci_set_master(pdev);
1757
1758         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1759         if (rc) {
1760                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1761                 if (rc)
1762                         goto err_dma_mask;
1763                 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1764         }
1765
1766         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1767         if (rc) {
1768                 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1769                 if (rc)
1770                         goto err_dma_mask;
1771                 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1772         }
1773         rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
1774                                           dma_get_mask(&pdev->dev));
1775         if (rc)
1776                 goto err_dma_mask;
1777
1778         ndev->self_mmio = pci_iomap(pdev, 0, 0);
1779         if (!ndev->self_mmio) {
1780                 rc = -EIO;
1781                 goto err_mmio;
1782         }
1783         ndev->peer_mmio = ndev->self_mmio;
1784         ndev->peer_addr = pci_resource_start(pdev, 0);
1785
1786         return 0;
1787
1788 err_mmio:
1789 err_dma_mask:
1790         pci_clear_master(pdev);
1791         pci_release_regions(pdev);
1792 err_pci_regions:
1793         pci_disable_device(pdev);
1794 err_pci_enable:
1795         pci_set_drvdata(pdev, NULL);
1796         return rc;
1797 }
1798
1799 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
1800 {
1801         struct pci_dev *pdev = ndev->ntb.pdev;
1802
1803         if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
1804                 pci_iounmap(pdev, ndev->peer_mmio);
1805         pci_iounmap(pdev, ndev->self_mmio);
1806
1807         pci_clear_master(pdev);
1808         pci_release_regions(pdev);
1809         pci_disable_device(pdev);
1810         pci_set_drvdata(pdev, NULL);
1811 }
1812
1813 static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
1814                                     struct pci_dev *pdev)
1815 {
1816         ndev->ntb.pdev = pdev;
1817         ndev->ntb.topo = NTB_TOPO_NONE;
1818         ndev->ntb.ops = &intel_ntb_ops;
1819
1820         ndev->b2b_off = 0;
1821         ndev->b2b_idx = UINT_MAX;
1822
1823         ndev->bar4_split = 0;
1824
1825         ndev->mw_count = 0;
1826         ndev->spad_count = 0;
1827         ndev->db_count = 0;
1828         ndev->db_vec_count = 0;
1829         ndev->db_vec_shift = 0;
1830
1831         ndev->ntb_ctl = 0;
1832         ndev->lnk_sta = 0;
1833
1834         ndev->db_valid_mask = 0;
1835         ndev->db_link_mask = 0;
1836         ndev->db_mask = 0;
1837
1838         spin_lock_init(&ndev->db_mask_lock);
1839 }
1840
1841 static int intel_ntb_pci_probe(struct pci_dev *pdev,
1842                                const struct pci_device_id *id)
1843 {
1844         struct intel_ntb_dev *ndev;
1845         int rc, node;
1846
1847         node = dev_to_node(&pdev->dev);
1848
1849         if (pdev_is_gen1(pdev)) {
1850                 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1851                 if (!ndev) {
1852                         rc = -ENOMEM;
1853                         goto err_ndev;
1854                 }
1855
1856                 ndev_init_struct(ndev, pdev);
1857
1858                 rc = intel_ntb_init_pci(ndev, pdev);
1859                 if (rc)
1860                         goto err_init_pci;
1861
1862                 rc = xeon_init_dev(ndev);
1863                 if (rc)
1864                         goto err_init_dev;
1865
1866         } else if (pdev_is_gen3(pdev)) {
1867                 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1868                 if (!ndev) {
1869                         rc = -ENOMEM;
1870                         goto err_ndev;
1871                 }
1872
1873                 ndev_init_struct(ndev, pdev);
1874                 ndev->ntb.ops = &intel_ntb3_ops;
1875
1876                 rc = intel_ntb_init_pci(ndev, pdev);
1877                 if (rc)
1878                         goto err_init_pci;
1879
1880                 rc = gen3_init_dev(ndev);
1881                 if (rc)
1882                         goto err_init_dev;
1883
1884         } else {
1885                 rc = -EINVAL;
1886                 goto err_ndev;
1887         }
1888
1889         ndev_reset_unsafe_flags(ndev);
1890
1891         ndev->reg->poll_link(ndev);
1892
1893         ndev_init_debugfs(ndev);
1894
1895         rc = ntb_register_device(&ndev->ntb);
1896         if (rc)
1897                 goto err_register;
1898
1899         dev_info(&pdev->dev, "NTB device registered.\n");
1900
1901         return 0;
1902
1903 err_register:
1904         ndev_deinit_debugfs(ndev);
1905         if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
1906                 xeon_deinit_dev(ndev);
1907 err_init_dev:
1908         intel_ntb_deinit_pci(ndev);
1909 err_init_pci:
1910         kfree(ndev);
1911 err_ndev:
1912         return rc;
1913 }
1914
1915 static void intel_ntb_pci_remove(struct pci_dev *pdev)
1916 {
1917         struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
1918
1919         ntb_unregister_device(&ndev->ntb);
1920         ndev_deinit_debugfs(ndev);
1921         if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
1922                 xeon_deinit_dev(ndev);
1923         intel_ntb_deinit_pci(ndev);
1924         kfree(ndev);
1925 }
1926
1927 static const struct intel_ntb_reg xeon_reg = {
1928         .poll_link              = xeon_poll_link,
1929         .link_is_up             = xeon_link_is_up,
1930         .db_ioread              = xeon_db_ioread,
1931         .db_iowrite             = xeon_db_iowrite,
1932         .db_size                = sizeof(u32),
1933         .ntb_ctl                = XEON_NTBCNTL_OFFSET,
1934         .mw_bar                 = {2, 4, 5},
1935 };
1936
1937 static const struct intel_ntb_alt_reg xeon_pri_reg = {
1938         .db_bell                = XEON_PDOORBELL_OFFSET,
1939         .db_mask                = XEON_PDBMSK_OFFSET,
1940         .spad                   = XEON_SPAD_OFFSET,
1941 };
1942
1943 static const struct intel_ntb_alt_reg xeon_sec_reg = {
1944         .db_bell                = XEON_SDOORBELL_OFFSET,
1945         .db_mask                = XEON_SDBMSK_OFFSET,
1946         /* second half of the scratchpads */
1947         .spad                   = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
1948 };
1949
1950 static const struct intel_ntb_alt_reg xeon_b2b_reg = {
1951         .db_bell                = XEON_B2B_DOORBELL_OFFSET,
1952         .spad                   = XEON_B2B_SPAD_OFFSET,
1953 };
1954
1955 static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
1956         /* Note: no primary .bar0_base visible to the secondary side.
1957          *
1958          * The secondary side cannot get the base address stored in primary
1959          * bars.  The base address is necessary to set the limit register to
1960          * any value other than zero, or unlimited.
1961          *
1962          * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
1963          * window by setting the limit equal to base, nor can it limit the size
1964          * of the memory window by setting the limit to base + size.
1965          */
1966         .bar2_limit             = XEON_PBAR23LMT_OFFSET,
1967         .bar2_xlat              = XEON_PBAR23XLAT_OFFSET,
1968 };
1969
1970 static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
1971         .bar0_base              = XEON_SBAR0BASE_OFFSET,
1972         .bar2_limit             = XEON_SBAR23LMT_OFFSET,
1973         .bar2_xlat              = XEON_SBAR23XLAT_OFFSET,
1974 };
1975
1976 struct intel_b2b_addr xeon_b2b_usd_addr = {
1977         .bar2_addr64            = XEON_B2B_BAR2_ADDR64,
1978         .bar4_addr64            = XEON_B2B_BAR4_ADDR64,
1979         .bar4_addr32            = XEON_B2B_BAR4_ADDR32,
1980         .bar5_addr32            = XEON_B2B_BAR5_ADDR32,
1981 };
1982
1983 struct intel_b2b_addr xeon_b2b_dsd_addr = {
1984         .bar2_addr64            = XEON_B2B_BAR2_ADDR64,
1985         .bar4_addr64            = XEON_B2B_BAR4_ADDR64,
1986         .bar4_addr32            = XEON_B2B_BAR4_ADDR32,
1987         .bar5_addr32            = XEON_B2B_BAR5_ADDR32,
1988 };
1989
1990 /* operations for primary side of local ntb */
1991 static const struct ntb_dev_ops intel_ntb_ops = {
1992         .mw_count               = intel_ntb_mw_count,
1993         .mw_get_align           = intel_ntb_mw_get_align,
1994         .mw_set_trans           = intel_ntb_mw_set_trans,
1995         .peer_mw_count          = intel_ntb_peer_mw_count,
1996         .peer_mw_get_addr       = intel_ntb_peer_mw_get_addr,
1997         .link_is_up             = intel_ntb_link_is_up,
1998         .link_enable            = intel_ntb_link_enable,
1999         .link_disable           = intel_ntb_link_disable,
2000         .db_is_unsafe           = intel_ntb_db_is_unsafe,
2001         .db_valid_mask          = intel_ntb_db_valid_mask,
2002         .db_vector_count        = intel_ntb_db_vector_count,
2003         .db_vector_mask         = intel_ntb_db_vector_mask,
2004         .db_read                = intel_ntb_db_read,
2005         .db_clear               = intel_ntb_db_clear,
2006         .db_set_mask            = intel_ntb_db_set_mask,
2007         .db_clear_mask          = intel_ntb_db_clear_mask,
2008         .peer_db_addr           = intel_ntb_peer_db_addr,
2009         .peer_db_set            = intel_ntb_peer_db_set,
2010         .spad_is_unsafe         = intel_ntb_spad_is_unsafe,
2011         .spad_count             = intel_ntb_spad_count,
2012         .spad_read              = intel_ntb_spad_read,
2013         .spad_write             = intel_ntb_spad_write,
2014         .peer_spad_addr         = intel_ntb_peer_spad_addr,
2015         .peer_spad_read         = intel_ntb_peer_spad_read,
2016         .peer_spad_write        = intel_ntb_peer_spad_write,
2017 };
2018
2019 static const struct file_operations intel_ntb_debugfs_info = {
2020         .owner = THIS_MODULE,
2021         .open = simple_open,
2022         .read = ndev_debugfs_read,
2023 };
2024
2025 static const struct pci_device_id intel_ntb_pci_tbl[] = {
2026         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2027         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2028         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2029         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2030         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
2031         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2032         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2033         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2034         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2035         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
2036         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2037         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2038         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2039         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2040         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
2041         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
2042         {0}
2043 };
2044 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2045
2046 static struct pci_driver intel_ntb_pci_driver = {
2047         .name = KBUILD_MODNAME,
2048         .id_table = intel_ntb_pci_tbl,
2049         .probe = intel_ntb_pci_probe,
2050         .remove = intel_ntb_pci_remove,
2051 };
2052
2053 static int __init intel_ntb_pci_driver_init(void)
2054 {
2055         pr_info("%s %s\n", NTB_DESC, NTB_VER);
2056
2057         if (debugfs_initialized())
2058                 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2059
2060         return pci_register_driver(&intel_ntb_pci_driver);
2061 }
2062 module_init(intel_ntb_pci_driver_init);
2063
2064 static void __exit intel_ntb_pci_driver_exit(void)
2065 {
2066         pci_unregister_driver(&intel_ntb_pci_driver);
2067
2068         debugfs_remove_recursive(debugfs_dir);
2069 }
2070 module_exit(intel_ntb_pci_driver_exit);