arch/powerpc/platforms/powernv/pci.c

   1 /*
   2  * Support PCI/PCIe on PowerNV platforms
   3  *
   4  * Currently supports only P5IOC2
   5  *
   6  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License
  10  * as published by the Free Software Foundation; either version
  11  * 2 of the License, or (at your option) any later version.
  12  */
  13
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/delay.h>
  17 #include <linux/string.h>
  18 #include <linux/init.h>
  19 #include <linux/bootmem.h>
  20 #include <linux/irq.h>
  21 #include <linux/io.h>
  22 #include <linux/msi.h>
  23 #include <linux/iommu.h>
  24
  25 #include <asm/sections.h>
  26 #include <asm/io.h>
  27 #include <asm/prom.h>
  28 #include <asm/pci-bridge.h>
  29 #include <asm/machdep.h>
  30 #include <asm/msi_bitmap.h>
  31 #include <asm/ppc-pci.h>
  32 #include <asm/opal.h>
  33 #include <asm/iommu.h>
  34 #include <asm/tce.h>
  35 #include <asm/firmware.h>
  36 #include <asm/eeh_event.h>
  37 #include <asm/eeh.h>
  38
  39 #include "powernv.h"
  40 #include "pci.h"
  41
  42 /* Delay in usec */
  43 #define PCI_RESET_DELAY_US      3000000
  44
  45 #define cfg_dbg(fmt...) do { } while(0)
  46 //#define cfg_dbg(fmt...)       printk(fmt)
  47
  48 #ifdef CONFIG_PCI_MSI
  49 static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
  50 {
  51         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  52         struct pnv_phb *phb = hose->private_data;
  53         struct pci_dn *pdn = pci_get_pdn(pdev);
  54         struct msi_desc *entry;
  55         struct msi_msg msg;
  56         int hwirq;
  57         unsigned int virq;
  58         int rc;
  59
  60         if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
  61                 return -ENODEV;
  62
  63         if (pdn && pdn->force_32bit_msi && !phb->msi32_support)
  64                 return -ENODEV;
  65
  66         list_for_each_entry(entry, &pdev->msi_list, list) {
  67                 if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
  68                         pr_warn("%s: Supports only 64-bit MSIs\n",
  69                                 pci_name(pdev));
  70                         return -ENXIO;
  71                 }
  72                 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
  73                 if (hwirq < 0) {
  74                         pr_warn("%s: Failed to find a free MSI\n",
  75                                 pci_name(pdev));
  76                         return -ENOSPC;
  77                 }
  78                 virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
  79                 if (virq == NO_IRQ) {
  80                         pr_warn("%s: Failed to map MSI to linux irq\n",
  81                                 pci_name(pdev));
  82                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
  83                         return -ENOMEM;
  84                 }
  85                 rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
  86                                     virq, entry->msi_attrib.is_64, &msg);
  87                 if (rc) {
  88                         pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
  89                         irq_dispose_mapping(virq);
  90                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
  91                         return rc;
  92                 }
  93                 irq_set_msi_desc(virq, entry);
  94                 write_msi_msg(virq, &msg);
  95         }
  96         return 0;
  97 }
  98
  99 static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
 100 {
 101         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 102         struct pnv_phb *phb = hose->private_data;
 103         struct msi_desc *entry;
 104
 105         if (WARN_ON(!phb))
 106                 return;
 107
 108         list_for_each_entry(entry, &pdev->msi_list, list) {
 109                 if (entry->irq == NO_IRQ)
 110                         continue;
 111                 irq_set_msi_desc(entry->irq, NULL);
 112                 msi_bitmap_free_hwirqs(&phb->msi_bmp,
 113                         virq_to_hw(entry->irq) - phb->msi_base, 1);
 114                 irq_dispose_mapping(entry->irq);
 115         }
 116 }
 117 #endif /* CONFIG_PCI_MSI */
 118
 119 static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
 120                                          struct OpalIoPhbErrorCommon *common)
 121 {
 122         struct OpalIoP7IOCPhbErrorData *data;
 123         int i;
 124
 125         data = (struct OpalIoP7IOCPhbErrorData *)common;
 126         pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
 127                 hose->global_number, be32_to_cpu(common->version));
 128
 129         if (data->brdgCtl)
 130                 pr_info("brdgCtl:     %08x\n",
 131                         be32_to_cpu(data->brdgCtl));
 132         if (data->portStatusReg || data->rootCmplxStatus ||
 133             data->busAgentStatus)
 134                 pr_info("UtlSts:      %08x %08x %08x\n",
 135                         be32_to_cpu(data->portStatusReg),
 136                         be32_to_cpu(data->rootCmplxStatus),
 137                         be32_to_cpu(data->busAgentStatus));
 138         if (data->deviceStatus || data->slotStatus   ||
 139             data->linkStatus   || data->devCmdStatus ||
 140             data->devSecStatus)
 141                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
 142                         be32_to_cpu(data->deviceStatus),
 143                         be32_to_cpu(data->slotStatus),
 144                         be32_to_cpu(data->linkStatus),
 145                         be32_to_cpu(data->devCmdStatus),
 146                         be32_to_cpu(data->devSecStatus));
 147         if (data->rootErrorStatus   || data->uncorrErrorStatus ||
 148             data->corrErrorStatus)
 149                 pr_info("RootErrSts:  %08x %08x %08x\n",
 150                         be32_to_cpu(data->rootErrorStatus),
 151                         be32_to_cpu(data->uncorrErrorStatus),
 152                         be32_to_cpu(data->corrErrorStatus));
 153         if (data->tlpHdr1 || data->tlpHdr2 ||
 154             data->tlpHdr3 || data->tlpHdr4)
 155                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
 156                         be32_to_cpu(data->tlpHdr1),
 157                         be32_to_cpu(data->tlpHdr2),
 158                         be32_to_cpu(data->tlpHdr3),
 159                         be32_to_cpu(data->tlpHdr4));
 160         if (data->sourceId || data->errorClass ||
 161             data->correlator)
 162                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
 163                         be32_to_cpu(data->sourceId),
 164                         be64_to_cpu(data->errorClass),
 165                         be64_to_cpu(data->correlator));
 166         if (data->p7iocPlssr || data->p7iocCsr)
 167                 pr_info("PhbSts:      %016llx %016llx\n",
 168                         be64_to_cpu(data->p7iocPlssr),
 169                         be64_to_cpu(data->p7iocCsr));
 170         if (data->lemFir)
 171                 pr_info("Lem:         %016llx %016llx %016llx\n",
 172                         be64_to_cpu(data->lemFir),
 173                         be64_to_cpu(data->lemErrorMask),
 174                         be64_to_cpu(data->lemWOF));
 175         if (data->phbErrorStatus)
 176                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
 177                         be64_to_cpu(data->phbErrorStatus),
 178                         be64_to_cpu(data->phbFirstErrorStatus),
 179                         be64_to_cpu(data->phbErrorLog0),
 180                         be64_to_cpu(data->phbErrorLog1));
 181         if (data->mmioErrorStatus)
 182                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
 183                         be64_to_cpu(data->mmioErrorStatus),
 184                         be64_to_cpu(data->mmioFirstErrorStatus),
 185                         be64_to_cpu(data->mmioErrorLog0),
 186                         be64_to_cpu(data->mmioErrorLog1));
 187         if (data->dma0ErrorStatus)
 188                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
 189                         be64_to_cpu(data->dma0ErrorStatus),
 190                         be64_to_cpu(data->dma0FirstErrorStatus),
 191                         be64_to_cpu(data->dma0ErrorLog0),
 192                         be64_to_cpu(data->dma0ErrorLog1));
 193         if (data->dma1ErrorStatus)
 194                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
 195                         be64_to_cpu(data->dma1ErrorStatus),
 196                         be64_to_cpu(data->dma1FirstErrorStatus),
 197                         be64_to_cpu(data->dma1ErrorLog0),
 198                         be64_to_cpu(data->dma1ErrorLog1));
 199
 200         for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
 201                 if ((data->pestA[i] >> 63) == 0 &&
 202                     (data->pestB[i] >> 63) == 0)
 203                         continue;
 204
 205                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
 206                         i, be64_to_cpu(data->pestA[i]),
 207                         be64_to_cpu(data->pestB[i]));
 208         }
 209 }
 210
 211 static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
 212                                         struct OpalIoPhbErrorCommon *common)
 213 {
 214         struct OpalIoPhb3ErrorData *data;
 215         int i;
 216
 217         data = (struct OpalIoPhb3ErrorData*)common;
 218         pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
 219                 hose->global_number, be32_to_cpu(common->version));
 220         if (data->brdgCtl)
 221                 pr_info("brdgCtl:     %08x\n",
 222                         be32_to_cpu(data->brdgCtl));
 223         if (data->portStatusReg || data->rootCmplxStatus ||
 224             data->busAgentStatus)
 225                 pr_info("UtlSts:      %08x %08x %08x\n",
 226                         be32_to_cpu(data->portStatusReg),
 227                         be32_to_cpu(data->rootCmplxStatus),
 228                         be32_to_cpu(data->busAgentStatus));
 229         if (data->deviceStatus || data->slotStatus   ||
 230             data->linkStatus   || data->devCmdStatus ||
 231             data->devSecStatus)
 232                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
 233                         be32_to_cpu(data->deviceStatus),
 234                         be32_to_cpu(data->slotStatus),
 235                         be32_to_cpu(data->linkStatus),
 236                         be32_to_cpu(data->devCmdStatus),
 237                         be32_to_cpu(data->devSecStatus));
 238         if (data->rootErrorStatus || data->uncorrErrorStatus ||
 239             data->corrErrorStatus)
 240                 pr_info("RootErrSts:  %08x %08x %08x\n",
 241                         be32_to_cpu(data->rootErrorStatus),
 242                         be32_to_cpu(data->uncorrErrorStatus),
 243                         be32_to_cpu(data->corrErrorStatus));
 244         if (data->tlpHdr1 || data->tlpHdr2 ||
 245             data->tlpHdr3 || data->tlpHdr4)
 246                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
 247                         be32_to_cpu(data->tlpHdr1),
 248                         be32_to_cpu(data->tlpHdr2),
 249                         be32_to_cpu(data->tlpHdr3),
 250                         be32_to_cpu(data->tlpHdr4));
 251         if (data->sourceId || data->errorClass ||
 252             data->correlator)
 253                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
 254                         be32_to_cpu(data->sourceId),
 255                         be64_to_cpu(data->errorClass),
 256                         be64_to_cpu(data->correlator));
 257         if (data->nFir)
 258                 pr_info("nFir:        %016llx %016llx %016llx\n",
 259                         be64_to_cpu(data->nFir),
 260                         be64_to_cpu(data->nFirMask),
 261                         be64_to_cpu(data->nFirWOF));
 262         if (data->phbPlssr || data->phbCsr)
 263                 pr_info("PhbSts:      %016llx %016llx\n",
 264                         be64_to_cpu(data->phbPlssr),
 265                         be64_to_cpu(data->phbCsr));
 266         if (data->lemFir)
 267                 pr_info("Lem:         %016llx %016llx %016llx\n",
 268                         be64_to_cpu(data->lemFir),
 269                         be64_to_cpu(data->lemErrorMask),
 270                         be64_to_cpu(data->lemWOF));
 271         if (data->phbErrorStatus)
 272                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
 273                         be64_to_cpu(data->phbErrorStatus),
 274                         be64_to_cpu(data->phbFirstErrorStatus),
 275                         be64_to_cpu(data->phbErrorLog0),
 276                         be64_to_cpu(data->phbErrorLog1));
 277         if (data->mmioErrorStatus)
 278                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
 279                         be64_to_cpu(data->mmioErrorStatus),
 280                         be64_to_cpu(data->mmioFirstErrorStatus),
 281                         be64_to_cpu(data->mmioErrorLog0),
 282                         be64_to_cpu(data->mmioErrorLog1));
 283         if (data->dma0ErrorStatus)
 284                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
 285                         be64_to_cpu(data->dma0ErrorStatus),
 286                         be64_to_cpu(data->dma0FirstErrorStatus),
 287                         be64_to_cpu(data->dma0ErrorLog0),
 288                         be64_to_cpu(data->dma0ErrorLog1));
 289         if (data->dma1ErrorStatus)
 290                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
 291                         be64_to_cpu(data->dma1ErrorStatus),
 292                         be64_to_cpu(data->dma1FirstErrorStatus),
 293                         be64_to_cpu(data->dma1ErrorLog0),
 294                         be64_to_cpu(data->dma1ErrorLog1));
 295
 296         for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
 297                 if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
 298                     (be64_to_cpu(data->pestB[i]) >> 63) == 0)
 299                         continue;
 300
 301                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
 302                                 i, be64_to_cpu(data->pestA[i]),
 303                                 be64_to_cpu(data->pestB[i]));
 304         }
 305 }
 306
 307 void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
 308                                 unsigned char *log_buff)
 309 {
 310         struct OpalIoPhbErrorCommon *common;
 311
 312         if (!hose || !log_buff)
 313                 return;
 314
 315         common = (struct OpalIoPhbErrorCommon *)log_buff;
 316         switch (be32_to_cpu(common->ioType)) {
 317         case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
 318                 pnv_pci_dump_p7ioc_diag_data(hose, common);
 319                 break;
 320         case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
 321                 pnv_pci_dump_phb3_diag_data(hose, common);
 322                 break;
 323         default:
 324                 pr_warn("%s: Unrecognized ioType %d\n",
 325                         __func__, be32_to_cpu(common->ioType));
 326         }
 327 }
 328
 329 static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
 330 {
 331         unsigned long flags, rc;
 332         int has_diag, ret = 0;
 333
 334         spin_lock_irqsave(&phb->lock, flags);
 335
 336         /* Fetch PHB diag-data */
 337         rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
 338                                          PNV_PCI_DIAG_BUF_SIZE);
 339         has_diag = (rc == OPAL_SUCCESS);
 340
 341         /* If PHB supports compound PE, to handle it */
 342         if (phb->unfreeze_pe) {
 343                 ret = phb->unfreeze_pe(phb,
 344                                        pe_no,
 345                                        OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 346         } else {
 347                 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
 348                                              pe_no,
 349                                              OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 350                 if (rc) {
 351                         pr_warn("%s: Failure %ld clearing frozen "
 352                                 "PHB#%x-PE#%x\n",
 353                                 __func__, rc, phb->hose->global_number,
 354                                 pe_no);
 355                         ret = -EIO;
 356                 }
 357         }
 358
 359         /*
 360          * For now, let's only display the diag buffer when we fail to clear
 361          * the EEH status. We'll do more sensible things later when we have
 362          * proper EEH support. We need to make sure we don't pollute ourselves
 363          * with the normal errors generated when probing empty slots
 364          */
 365         if (has_diag && ret)
 366                 pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);
 367
 368         spin_unlock_irqrestore(&phb->lock, flags);
 369 }
 370
 371 static void pnv_pci_config_check_eeh(struct pnv_phb *phb,
 372                                      struct device_node *dn)
 373 {
 374         u8      fstate;
 375         __be16  pcierr;
 376         int     pe_no;
 377         s64     rc;
 378
 379         /*
 380          * Get the PE#. During the PCI probe stage, we might not
 381          * setup that yet. So all ER errors should be mapped to
 382          * reserved PE.
 383          */
 384         pe_no = PCI_DN(dn)->pe_number;
 385         if (pe_no == IODA_INVALID_PE) {
 386                 if (phb->type == PNV_PHB_P5IOC2)
 387                         pe_no = 0;
 388                 else
 389                         pe_no = phb->ioda.reserved_pe;
 390         }
 391
 392         /*
 393          * Fetch frozen state. If the PHB support compound PE,
 394          * we need handle that case.
 395          */
 396         if (phb->get_pe_state) {
 397                 fstate = phb->get_pe_state(phb, pe_no);
 398         } else {
 399                 rc = opal_pci_eeh_freeze_status(phb->opal_id,
 400                                                 pe_no,
 401                                                 &fstate,
 402                                                 &pcierr,
 403                                                 NULL);
 404                 if (rc) {
 405                         pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
 406                                 __func__, rc, phb->hose->global_number, pe_no);
 407                         return;
 408                 }
 409         }
 410
 411         cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
 412                 (PCI_DN(dn)->busno << 8) | (PCI_DN(dn)->devfn),
 413                 pe_no, fstate);
 414
 415         /* Clear the frozen state if applicable */
 416         if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
 417             fstate == OPAL_EEH_STOPPED_DMA_FREEZE  ||
 418             fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
 419                 /*
 420                  * If PHB supports compound PE, freeze it for
 421                  * consistency.
 422                  */
 423                 if (phb->freeze_pe)
 424                         phb->freeze_pe(phb, pe_no);
 425
 426                 pnv_pci_handle_eeh_config(phb, pe_no);
 427         }
 428 }
 429
 430 int pnv_pci_cfg_read(struct device_node *dn,
 431                      int where, int size, u32 *val)
 432 {
 433         struct pci_dn *pdn = PCI_DN(dn);
 434         struct pnv_phb *phb = pdn->phb->private_data;
 435         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
 436         s64 rc;
 437
 438         switch (size) {
 439         case 1: {
 440                 u8 v8;
 441                 rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
 442                 *val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
 443                 break;
 444         }
 445         case 2: {
 446                 __be16 v16;
 447                 rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
 448                                                    &v16);
 449                 *val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
 450                 break;
 451         }
 452         case 4: {
 453                 __be32 v32;
 454                 rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
 455                 *val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
 456                 break;
 457         }
 458         default:
 459                 return PCIBIOS_FUNC_NOT_SUPPORTED;
 460         }
 461
 462         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
 463                 __func__, pdn->busno, pdn->devfn, where, size, *val);
 464         return PCIBIOS_SUCCESSFUL;
 465 }
 466
 467 int pnv_pci_cfg_write(struct device_node *dn,
 468                       int where, int size, u32 val)
 469 {
 470         struct pci_dn *pdn = PCI_DN(dn);
 471         struct pnv_phb *phb = pdn->phb->private_data;
 472         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
 473
 474         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
 475                 pdn->busno, pdn->devfn, where, size, val);
 476         switch (size) {
 477         case 1:
 478                 opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
 479                 break;
 480         case 2:
 481                 opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
 482                 break;
 483         case 4:
 484                 opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
 485                 break;
 486         default:
 487                 return PCIBIOS_FUNC_NOT_SUPPORTED;
 488         }
 489
 490         return PCIBIOS_SUCCESSFUL;
 491 }
 492
 493 #if CONFIG_EEH
 494 static bool pnv_pci_cfg_check(struct pci_controller *hose,
 495                               struct device_node *dn)
 496 {
 497         struct eeh_dev *edev = NULL;
 498         struct pnv_phb *phb = hose->private_data;
 499
 500         /* EEH not enabled ? */
 501         if (!(phb->flags & PNV_PHB_FLAG_EEH))
 502                 return true;
 503
 504         /* PE reset or device removed ? */
 505         edev = of_node_to_eeh_dev(dn);
 506         if (edev) {
 507                 if (edev->pe &&
 508                     (edev->pe->state & EEH_PE_CFG_BLOCKED))
 509                         return false;
 510
 511                 if (edev->mode & EEH_DEV_REMOVED)
 512                         return false;
 513         }
 514
 515         return true;
 516 }
 517 #else
 518 static inline pnv_pci_cfg_check(struct pci_controller *hose,
 519                                 struct device_node *dn)
 520 {
 521         return true;
 522 }
 523 #endif /* CONFIG_EEH */
 524
 525 static int pnv_pci_read_config(struct pci_bus *bus,
 526                                unsigned int devfn,
 527                                int where, int size, u32 *val)
 528 {
 529         struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
 530         struct pci_dn *pdn;
 531         struct pnv_phb *phb;
 532         bool found = false;
 533         int ret;
 534
 535         *val = 0xFFFFFFFF;
 536         for (dn = busdn->child; dn; dn = dn->sibling) {
 537                 pdn = PCI_DN(dn);
 538                 if (pdn && pdn->devfn == devfn) {
 539                         phb = pdn->phb->private_data;
 540                         found = true;
 541                         break;
 542                 }
 543         }
 544
 545         if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
 546                 return PCIBIOS_DEVICE_NOT_FOUND;
 547
 548         ret = pnv_pci_cfg_read(dn, where, size, val);
 549         if (phb->flags & PNV_PHB_FLAG_EEH) {
 550                 if (*val == EEH_IO_ERROR_VALUE(size) &&
 551                     eeh_dev_check_failure(of_node_to_eeh_dev(dn)))
 552                         return PCIBIOS_DEVICE_NOT_FOUND;
 553         } else {
 554                 pnv_pci_config_check_eeh(phb, dn);
 555         }
 556
 557         return ret;
 558 }
 559
 560 static int pnv_pci_write_config(struct pci_bus *bus,
 561                                 unsigned int devfn,
 562                                 int where, int size, u32 val)
 563 {
 564         struct device_node *dn, *busdn = pci_bus_to_OF_node(bus);
 565         struct pci_dn *pdn;
 566         struct pnv_phb *phb;
 567         bool found = false;
 568         int ret;
 569
 570         for (dn = busdn->child; dn; dn = dn->sibling) {
 571                 pdn = PCI_DN(dn);
 572                 if (pdn && pdn->devfn == devfn) {
 573                         phb = pdn->phb->private_data;
 574                         found = true;
 575                         break;
 576                 }
 577         }
 578
 579         if (!found || !pnv_pci_cfg_check(pdn->phb, dn))
 580                 return PCIBIOS_DEVICE_NOT_FOUND;
 581
 582         ret = pnv_pci_cfg_write(dn, where, size, val);
 583         if (!(phb->flags & PNV_PHB_FLAG_EEH))
 584                 pnv_pci_config_check_eeh(phb, dn);
 585
 586         return ret;
 587 }
 588
 589 struct pci_ops pnv_pci_ops = {
 590         .read  = pnv_pci_read_config,
 591         .write = pnv_pci_write_config,
 592 };
 593
 594 static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
 595                          unsigned long uaddr, enum dma_data_direction direction,
 596                          struct dma_attrs *attrs, bool rm)
 597 {
 598         u64 proto_tce;
 599         __be64 *tcep, *tces;
 600         u64 rpn;
 601
 602         proto_tce = TCE_PCI_READ; // Read allowed
 603
 604         if (direction != DMA_TO_DEVICE)
 605                 proto_tce |= TCE_PCI_WRITE;
 606
 607         tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
 608         rpn = __pa(uaddr) >> tbl->it_page_shift;
 609
 610         while (npages--)
 611                 *(tcep++) = cpu_to_be64(proto_tce |
 612                                 (rpn++ << tbl->it_page_shift));
 613
 614         /* Some implementations won't cache invalid TCEs and thus may not
 615          * need that flush. We'll probably turn it_type into a bit mask
 616          * of flags if that becomes the case
 617          */
 618         if (tbl->it_type & TCE_PCI_SWINV_CREATE)
 619                 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
 620
 621         return 0;
 622 }
 623
 624 static int pnv_tce_build_vm(struct iommu_table *tbl, long index, long npages,
 625                             unsigned long uaddr,
 626                             enum dma_data_direction direction,
 627                             struct dma_attrs *attrs)
 628 {
 629         return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs,
 630                         false);
 631 }
 632
 633 static void pnv_tce_free(struct iommu_table *tbl, long index, long npages,
 634                 bool rm)
 635 {
 636         __be64 *tcep, *tces;
 637
 638         tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
 639
 640         while (npages--)
 641                 *(tcep++) = cpu_to_be64(0);
 642
 643         if (tbl->it_type & TCE_PCI_SWINV_FREE)
 644                 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
 645 }
 646
 647 static void pnv_tce_free_vm(struct iommu_table *tbl, long index, long npages)
 648 {
 649         pnv_tce_free(tbl, index, npages, false);
 650 }
 651
 652 static unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
 653 {
 654         return ((u64 *)tbl->it_base)[index - tbl->it_offset];
 655 }
 656
 657 static int pnv_tce_build_rm(struct iommu_table *tbl, long index, long npages,
 658                             unsigned long uaddr,
 659                             enum dma_data_direction direction,
 660                             struct dma_attrs *attrs)
 661 {
 662         return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, true);
 663 }
 664
 665 static void pnv_tce_free_rm(struct iommu_table *tbl, long index, long npages)
 666 {
 667         pnv_tce_free(tbl, index, npages, true);
 668 }
 669
 670 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
 671                                void *tce_mem, u64 tce_size,
 672                                u64 dma_offset, unsigned page_shift)
 673 {
 674         tbl->it_blocksize = 16;
 675         tbl->it_base = (unsigned long)tce_mem;
 676         tbl->it_page_shift = page_shift;
 677         tbl->it_offset = dma_offset >> tbl->it_page_shift;
 678         tbl->it_index = 0;
 679         tbl->it_size = tce_size >> 3;
 680         tbl->it_busno = 0;
 681         tbl->it_type = TCE_PCI;
 682 }
 683
 684 static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose)
 685 {
 686         struct iommu_table *tbl;
 687         const __be64 *basep, *swinvp;
 688         const __be32 *sizep;
 689
 690         basep = of_get_property(hose->dn, "linux,tce-base", NULL);
 691         sizep = of_get_property(hose->dn, "linux,tce-size", NULL);
 692         if (basep == NULL || sizep == NULL) {
 693                 pr_err("PCI: %s has missing tce entries !\n",
 694                        hose->dn->full_name);
 695                 return NULL;
 696         }
 697         tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node);
 698         if (WARN_ON(!tbl))
 699                 return NULL;
 700         pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)),
 701                                   be32_to_cpup(sizep), 0, IOMMU_PAGE_SHIFT_4K);
 702         iommu_init_table(tbl, hose->node);
 703         iommu_register_group(tbl, pci_domain_nr(hose->bus), 0);
 704
 705         /* Deal with SW invalidated TCEs when needed (BML way) */
 706         swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info",
 707                                  NULL);
 708         if (swinvp) {
 709                 tbl->it_busno = be64_to_cpu(swinvp[1]);
 710                 tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8);
 711                 tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
 712         }
 713         return tbl;
 714 }
 715
 716 static void pnv_pci_dma_fallback_setup(struct pci_controller *hose,
 717                                        struct pci_dev *pdev)
 718 {
 719         struct device_node *np = pci_bus_to_OF_node(hose->bus);
 720         struct pci_dn *pdn;
 721
 722         if (np == NULL)
 723                 return;
 724         pdn = PCI_DN(np);
 725         if (!pdn->iommu_table)
 726                 pdn->iommu_table = pnv_pci_setup_bml_iommu(hose);
 727         if (!pdn->iommu_table)
 728                 return;
 729         set_iommu_table_base_and_group(&pdev->dev, pdn->iommu_table);
 730 }
 731
 732 static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
 733 {
 734         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 735         struct pnv_phb *phb = hose->private_data;
 736
 737         /* If we have no phb structure, try to setup a fallback based on
 738          * the device-tree (RTAS PCI for example)
 739          */
 740         if (phb && phb->dma_dev_setup)
 741                 phb->dma_dev_setup(phb, pdev);
 742         else
 743                 pnv_pci_dma_fallback_setup(hose, pdev);
 744 }
 745
 746 int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
 747 {
 748         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 749         struct pnv_phb *phb = hose->private_data;
 750
 751         if (phb && phb->dma_set_mask)
 752                 return phb->dma_set_mask(phb, pdev, dma_mask);
 753         return __dma_set_mask(&pdev->dev, dma_mask);
 754 }
 755
 756 u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev)
 757 {
 758         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 759         struct pnv_phb *phb = hose->private_data;
 760
 761         if (phb && phb->dma_get_required_mask)
 762                 return phb->dma_get_required_mask(phb, pdev);
 763
 764         return __dma_get_required_mask(&pdev->dev);
 765 }
 766
 767 void pnv_pci_shutdown(void)
 768 {
 769         struct pci_controller *hose;
 770
 771         list_for_each_entry(hose, &hose_list, list_node) {
 772                 struct pnv_phb *phb = hose->private_data;
 773
 774                 if (phb && phb->shutdown)
 775                         phb->shutdown(phb);
 776         }
 777 }
 778
 779 /* Fixup wrong class code in p7ioc and p8 root complex */
 780 static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
 781 {
 782         dev->class = PCI_CLASS_BRIDGE_PCI << 8;
 783 }
 784 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);
 785
 786 static int pnv_pci_probe_mode(struct pci_bus *bus)
 787 {
 788         struct pci_controller *hose = pci_bus_to_host(bus);
 789         const __be64 *tstamp;
 790         u64 now, target;
 791
 792
 793         /* We hijack this as a way to ensure we have waited long
 794          * enough since the reset was lifted on the PCI bus
 795          */
 796         if (bus != hose->bus)
 797                 return PCI_PROBE_NORMAL;
 798         tstamp = of_get_property(hose->dn, "reset-clear-timestamp", NULL);
 799         if (!tstamp || !*tstamp)
 800                 return PCI_PROBE_NORMAL;
 801
 802         now = mftb() / tb_ticks_per_usec;
 803         target = (be64_to_cpup(tstamp) / tb_ticks_per_usec)
 804                 + PCI_RESET_DELAY_US;
 805
 806         pr_devel("pci %04d: Reset target: 0x%llx now: 0x%llx\n",
 807                  hose->global_number, target, now);
 808
 809         if (now < target)
 810                 msleep((target - now + 999) / 1000);
 811
 812         return PCI_PROBE_NORMAL;
 813 }
 814
 815 void __init pnv_pci_init(void)
 816 {
 817         struct device_node *np;
 818
 819         pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
 820
 821         /* OPAL absent, try POPAL first then RTAS detection of PHBs */
 822         if (!firmware_has_feature(FW_FEATURE_OPAL)) {
 823 #ifdef CONFIG_PPC_POWERNV_RTAS
 824                 init_pci_config_tokens();
 825                 find_and_init_phbs();
 826 #endif /* CONFIG_PPC_POWERNV_RTAS */
 827         }
 828         /* OPAL is here, do our normal stuff */
 829         else {
 830                 int found_ioda = 0;
 831
 832                 /* Look for IODA IO-Hubs. We don't support mixing IODA
 833                  * and p5ioc2 due to the need to change some global
 834                  * probing flags
 835                  */
 836                 for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
 837                         pnv_pci_init_ioda_hub(np);
 838                         found_ioda = 1;
 839                 }
 840
 841                 /* Look for p5ioc2 IO-Hubs */
 842                 if (!found_ioda)
 843                         for_each_compatible_node(np, NULL, "ibm,p5ioc2")
 844                                 pnv_pci_init_p5ioc2_hub(np);
 845
 846                 /* Look for ioda2 built-in PHB3's */
 847                 for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
 848                         pnv_pci_init_ioda2_phb(np);
 849         }
 850
 851         /* Setup the linkage between OF nodes and PHBs */
 852         pci_devs_phb_init();
 853
 854         /* Configure IOMMU DMA hooks */
 855         ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup;
 856         ppc_md.tce_build = pnv_tce_build_vm;
 857         ppc_md.tce_free = pnv_tce_free_vm;
 858         ppc_md.tce_build_rm = pnv_tce_build_rm;
 859         ppc_md.tce_free_rm = pnv_tce_free_rm;
 860         ppc_md.tce_get = pnv_tce_get;
 861         ppc_md.pci_probe_mode = pnv_pci_probe_mode;
 862         set_pci_dma_ops(&dma_iommu_ops);
 863
 864         /* Configure MSIs */
 865 #ifdef CONFIG_PCI_MSI
 866         ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
 867         ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
 868 #endif
 869 }
 870
 871 static int tce_iommu_bus_notifier(struct notifier_block *nb,
 872                 unsigned long action, void *data)
 873 {
 874         struct device *dev = data;
 875
 876         switch (action) {
 877         case BUS_NOTIFY_ADD_DEVICE:
 878                 return iommu_add_device(dev);
 879         case BUS_NOTIFY_DEL_DEVICE:
 880                 if (dev->iommu_group)
 881                         iommu_del_device(dev);
 882                 return 0;
 883         default:
 884                 return 0;
 885         }
 886 }
 887
 888 static struct notifier_block tce_iommu_bus_nb = {
 889         .notifier_call = tce_iommu_bus_notifier,
 890 };
 891
 892 static int __init tce_iommu_bus_notifier_init(void)
 893 {
 894         bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
 895         return 0;
 896 }
 897 machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);