drivers/irqchip/irq-gic-v3.c

   1 /*
   2  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #define pr_fmt(fmt)     "GICv3: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/cpu.h>
  22 #include <linux/cpu_pm.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/irqdomain.h>
  26 #include <linux/of.h>
  27 #include <linux/of_address.h>
  28 #include <linux/of_irq.h>
  29 #include <linux/percpu.h>
  30 #include <linux/refcount.h>
  31 #include <linux/slab.h>
  32
  33 #include <linux/irqchip.h>
  34 #include <linux/irqchip/arm-gic-common.h>
  35 #include <linux/irqchip/arm-gic-v3.h>
  36 #include <linux/irqchip/irq-partition-percpu.h>
  37
  38 #include <asm/cputype.h>
  39 #include <asm/exception.h>
  40 #include <asm/smp_plat.h>
  41 #include <asm/virt.h>
  42
  43 #include "irq-gic-common.h"
  44
  45 #define GICD_INT_NMI_PRI        (GICD_INT_DEF_PRI & ~0x80)
  46
  47 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996     (1ULL << 0)
  48
  49 struct redist_region {
  50         void __iomem            *redist_base;
  51         phys_addr_t             phys_base;
  52         bool                    single_redist;
  53 };
  54
  55 struct gic_chip_data {
  56         struct fwnode_handle    *fwnode;
  57         void __iomem            *dist_base;
  58         struct redist_region    *redist_regions;
  59         struct rdists           rdists;
  60         struct irq_domain       *domain;
  61         u64                     redist_stride;
  62         u32                     nr_redist_regions;
  63         u64                     flags;
  64         bool                    has_rss;
  65         unsigned int            irq_nr;
  66         struct partition_desc   *ppi_descs[16];
  67 };
  68
  69 static struct gic_chip_data gic_data __read_mostly;
  70 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
  71
  72 /*
  73  * The behaviours of RPR and PMR registers differ depending on the value of
  74  * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
  75  * distributor and redistributors depends on whether security is enabled in the
  76  * GIC.
  77  *
  78  * When security is enabled, non-secure priority values from the (re)distributor
  79  * are presented to the GIC CPUIF as follow:
  80  *     (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
  81  *
  82  * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure
  83  * EL1 are subject to a similar operation thus matching the priorities presented
  84  * from the (re)distributor when security is enabled.
  85  *
  86  * see GICv3/GICv4 Architecture Specification (IHI0069D):
  87  * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
  88  *   priorities.
  89  * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
  90  *   interrupt.
  91  *
  92  * For now, we only support pseudo-NMIs if we have non-secure view of
  93  * priorities.
  94  */
  95 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
  96
  97 /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
  98 static refcount_t ppi_nmi_refs[16];
  99
 100 static struct gic_kvm_info gic_v3_kvm_info;
 101 static DEFINE_PER_CPU(bool, has_rss);
 102
 103 #define MPIDR_RS(mpidr)                 (((mpidr) & 0xF0UL) >> 4)
 104 #define gic_data_rdist()                (this_cpu_ptr(gic_data.rdists.rdist))
 105 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 106 #define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
 107
 108 /* Our default, arbitrary priority value. Linux only uses one anyway. */
 109 #define DEFAULT_PMR_VALUE       0xf0
 110
 111 static inline unsigned int gic_irq(struct irq_data *d)
 112 {
 113         return d->hwirq;
 114 }
 115
 116 static inline int gic_irq_in_rdist(struct irq_data *d)
 117 {
 118         return gic_irq(d) < 32;
 119 }
 120
 121 static inline void __iomem *gic_dist_base(struct irq_data *d)
 122 {
 123         if (gic_irq_in_rdist(d))        /* SGI+PPI -> SGI_base for this CPU */
 124                 return gic_data_rdist_sgi_base();
 125
 126         if (d->hwirq <= 1023)           /* SPI -> dist_base */
 127                 return gic_data.dist_base;
 128
 129         return NULL;
 130 }
 131
 132 static void gic_do_wait_for_rwp(void __iomem *base)
 133 {
 134         u32 count = 1000000;    /* 1s! */
 135
 136         while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
 137                 count--;
 138                 if (!count) {
 139                         pr_err_ratelimited("RWP timeout, gone fishing\n");
 140                         return;
 141                 }
 142                 cpu_relax();
 143                 udelay(1);
 144         };
 145 }
 146
 147 /* Wait for completion of a distributor change */
 148 static void gic_dist_wait_for_rwp(void)
 149 {
 150         gic_do_wait_for_rwp(gic_data.dist_base);
 151 }
 152
 153 /* Wait for completion of a redistributor change */
 154 static void gic_redist_wait_for_rwp(void)
 155 {
 156         gic_do_wait_for_rwp(gic_data_rdist_rd_base());
 157 }
 158
 159 #ifdef CONFIG_ARM64
 160
 161 static u64 __maybe_unused gic_read_iar(void)
 162 {
 163         if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154))
 164                 return gic_read_iar_cavium_thunderx();
 165         else
 166                 return gic_read_iar_common();
 167 }
 168 #endif
 169
 170 static void gic_enable_redist(bool enable)
 171 {
 172         void __iomem *rbase;
 173         u32 count = 1000000;    /* 1s! */
 174         u32 val;
 175
 176         if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
 177                 return;
 178
 179         rbase = gic_data_rdist_rd_base();
 180
 181         val = readl_relaxed(rbase + GICR_WAKER);
 182         if (enable)
 183                 /* Wake up this CPU redistributor */
 184                 val &= ~GICR_WAKER_ProcessorSleep;
 185         else
 186                 val |= GICR_WAKER_ProcessorSleep;
 187         writel_relaxed(val, rbase + GICR_WAKER);
 188
 189         if (!enable) {          /* Check that GICR_WAKER is writeable */
 190                 val = readl_relaxed(rbase + GICR_WAKER);
 191                 if (!(val & GICR_WAKER_ProcessorSleep))
 192                         return; /* No PM support in this redistributor */
 193         }
 194
 195         while (--count) {
 196                 val = readl_relaxed(rbase + GICR_WAKER);
 197                 if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep))
 198                         break;
 199                 cpu_relax();
 200                 udelay(1);
 201         };
 202         if (!count)
 203                 pr_err_ratelimited("redistributor failed to %s...\n",
 204                                    enable ? "wakeup" : "sleep");
 205 }
 206
 207 /*
 208  * Routines to disable, enable, EOI and route interrupts
 209  */
 210 static int gic_peek_irq(struct irq_data *d, u32 offset)
 211 {
 212         u32 mask = 1 << (gic_irq(d) % 32);
 213         void __iomem *base;
 214
 215         if (gic_irq_in_rdist(d))
 216                 base = gic_data_rdist_sgi_base();
 217         else
 218                 base = gic_data.dist_base;
 219
 220         return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask);
 221 }
 222
 223 static void gic_poke_irq(struct irq_data *d, u32 offset)
 224 {
 225         u32 mask = 1 << (gic_irq(d) % 32);
 226         void (*rwp_wait)(void);
 227         void __iomem *base;
 228
 229         if (gic_irq_in_rdist(d)) {
 230                 base = gic_data_rdist_sgi_base();
 231                 rwp_wait = gic_redist_wait_for_rwp;
 232         } else {
 233                 base = gic_data.dist_base;
 234                 rwp_wait = gic_dist_wait_for_rwp;
 235         }
 236
 237         writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4);
 238         rwp_wait();
 239 }
 240
 241 static void gic_mask_irq(struct irq_data *d)
 242 {
 243         gic_poke_irq(d, GICD_ICENABLER);
 244 }
 245
 246 static void gic_eoimode1_mask_irq(struct irq_data *d)
 247 {
 248         gic_mask_irq(d);
 249         /*
 250          * When masking a forwarded interrupt, make sure it is
 251          * deactivated as well.
 252          *
 253          * This ensures that an interrupt that is getting
 254          * disabled/masked will not get "stuck", because there is
 255          * noone to deactivate it (guest is being terminated).
 256          */
 257         if (irqd_is_forwarded_to_vcpu(d))
 258                 gic_poke_irq(d, GICD_ICACTIVER);
 259 }
 260
 261 static void gic_unmask_irq(struct irq_data *d)
 262 {
 263         gic_poke_irq(d, GICD_ISENABLER);
 264 }
 265
 266 static inline bool gic_supports_nmi(void)
 267 {
 268         return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
 269                static_branch_likely(&supports_pseudo_nmis);
 270 }
 271
 272 static int gic_irq_set_irqchip_state(struct irq_data *d,
 273                                      enum irqchip_irq_state which, bool val)
 274 {
 275         u32 reg;
 276
 277         if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
 278                 return -EINVAL;
 279
 280         switch (which) {
 281         case IRQCHIP_STATE_PENDING:
 282                 reg = val ? GICD_ISPENDR : GICD_ICPENDR;
 283                 break;
 284
 285         case IRQCHIP_STATE_ACTIVE:
 286                 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
 287                 break;
 288
 289         case IRQCHIP_STATE_MASKED:
 290                 reg = val ? GICD_ICENABLER : GICD_ISENABLER;
 291                 break;
 292
 293         default:
 294                 return -EINVAL;
 295         }
 296
 297         gic_poke_irq(d, reg);
 298         return 0;
 299 }
 300
 301 static int gic_irq_get_irqchip_state(struct irq_data *d,
 302                                      enum irqchip_irq_state which, bool *val)
 303 {
 304         if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
 305                 return -EINVAL;
 306
 307         switch (which) {
 308         case IRQCHIP_STATE_PENDING:
 309                 *val = gic_peek_irq(d, GICD_ISPENDR);
 310                 break;
 311
 312         case IRQCHIP_STATE_ACTIVE:
 313                 *val = gic_peek_irq(d, GICD_ISACTIVER);
 314                 break;
 315
 316         case IRQCHIP_STATE_MASKED:
 317                 *val = !gic_peek_irq(d, GICD_ISENABLER);
 318                 break;
 319
 320         default:
 321                 return -EINVAL;
 322         }
 323
 324         return 0;
 325 }
 326
 327 static void gic_irq_set_prio(struct irq_data *d, u8 prio)
 328 {
 329         void __iomem *base = gic_dist_base(d);
 330
 331         writeb_relaxed(prio, base + GICD_IPRIORITYR + gic_irq(d));
 332 }
 333
 334 static int gic_irq_nmi_setup(struct irq_data *d)
 335 {
 336         struct irq_desc *desc = irq_to_desc(d->irq);
 337
 338         if (!gic_supports_nmi())
 339                 return -EINVAL;
 340
 341         if (gic_peek_irq(d, GICD_ISENABLER)) {
 342                 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
 343                 return -EINVAL;
 344         }
 345
 346         /*
 347          * A secondary irq_chip should be in charge of LPI request,
 348          * it should not be possible to get there
 349          */
 350         if (WARN_ON(gic_irq(d) >= 8192))
 351                 return -EINVAL;
 352
 353         /* desc lock should already be held */
 354         if (gic_irq(d) < 32) {
 355                 /* Setting up PPI as NMI, only switch handler for first NMI */
 356                 if (!refcount_inc_not_zero(&ppi_nmi_refs[gic_irq(d) - 16])) {
 357                         refcount_set(&ppi_nmi_refs[gic_irq(d) - 16], 1);
 358                         desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
 359                 }
 360         } else {
 361                 desc->handle_irq = handle_fasteoi_nmi;
 362         }
 363
 364         gic_irq_set_prio(d, GICD_INT_NMI_PRI);
 365
 366         return 0;
 367 }
 368
 369 static void gic_irq_nmi_teardown(struct irq_data *d)
 370 {
 371         struct irq_desc *desc = irq_to_desc(d->irq);
 372
 373         if (WARN_ON(!gic_supports_nmi()))
 374                 return;
 375
 376         if (gic_peek_irq(d, GICD_ISENABLER)) {
 377                 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
 378                 return;
 379         }
 380
 381         /*
 382          * A secondary irq_chip should be in charge of LPI request,
 383          * it should not be possible to get there
 384          */
 385         if (WARN_ON(gic_irq(d) >= 8192))
 386                 return;
 387
 388         /* desc lock should already be held */
 389         if (gic_irq(d) < 32) {
 390                 /* Tearing down NMI, only switch handler for last NMI */
 391                 if (refcount_dec_and_test(&ppi_nmi_refs[gic_irq(d) - 16]))
 392                         desc->handle_irq = handle_percpu_devid_irq;
 393         } else {
 394                 desc->handle_irq = handle_fasteoi_irq;
 395         }
 396
 397         gic_irq_set_prio(d, GICD_INT_DEF_PRI);
 398 }
 399
 400 static void gic_eoi_irq(struct irq_data *d)
 401 {
 402         gic_write_eoir(gic_irq(d));
 403 }
 404
 405 static void gic_eoimode1_eoi_irq(struct irq_data *d)
 406 {
 407         /*
 408          * No need to deactivate an LPI, or an interrupt that
 409          * is is getting forwarded to a vcpu.
 410          */
 411         if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
 412                 return;
 413         gic_write_dir(gic_irq(d));
 414 }
 415
 416 static int gic_set_type(struct irq_data *d, unsigned int type)
 417 {
 418         unsigned int irq = gic_irq(d);
 419         void (*rwp_wait)(void);
 420         void __iomem *base;
 421
 422         /* Interrupt configuration for SGIs can't be changed */
 423         if (irq < 16)
 424                 return -EINVAL;
 425
 426         /* SPIs have restrictions on the supported types */
 427         if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
 428                          type != IRQ_TYPE_EDGE_RISING)
 429                 return -EINVAL;
 430
 431         if (gic_irq_in_rdist(d)) {
 432                 base = gic_data_rdist_sgi_base();
 433                 rwp_wait = gic_redist_wait_for_rwp;
 434         } else {
 435                 base = gic_data.dist_base;
 436                 rwp_wait = gic_dist_wait_for_rwp;
 437         }
 438
 439         return gic_configure_irq(irq, type, base, rwp_wait);
 440 }
 441
 442 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
 443 {
 444         if (vcpu)
 445                 irqd_set_forwarded_to_vcpu(d);
 446         else
 447                 irqd_clr_forwarded_to_vcpu(d);
 448         return 0;
 449 }
 450
 451 static u64 gic_mpidr_to_affinity(unsigned long mpidr)
 452 {
 453         u64 aff;
 454
 455         aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
 456                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
 457                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
 458                MPIDR_AFFINITY_LEVEL(mpidr, 0));
 459
 460         return aff;
 461 }
 462
 463 static void gic_deactivate_unhandled(u32 irqnr)
 464 {
 465         if (static_branch_likely(&supports_deactivate_key)) {
 466                 if (irqnr < 8192)
 467                         gic_write_dir(irqnr);
 468         } else {
 469                 gic_write_eoir(irqnr);
 470         }
 471 }
 472
 473 static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
 474 {
 475         int err;
 476
 477         if (static_branch_likely(&supports_deactivate_key))
 478                 gic_write_eoir(irqnr);
 479         /*
 480          * Leave the PSR.I bit set to prevent other NMIs to be
 481          * received while handling this one.
 482          * PSR.I will be restored when we ERET to the
 483          * interrupted context.
 484          */
 485         err = handle_domain_nmi(gic_data.domain, irqnr, regs);
 486         if (err)
 487                 gic_deactivate_unhandled(irqnr);
 488 }
 489
 490 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
 491 {
 492         u32 irqnr;
 493
 494         irqnr = gic_read_iar();
 495
 496         if (gic_supports_nmi() &&
 497             unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) {
 498                 gic_handle_nmi(irqnr, regs);
 499                 return;
 500         }
 501
 502         if (gic_prio_masking_enabled()) {
 503                 gic_pmr_mask_irqs();
 504                 gic_arch_enable_irqs();
 505         }
 506
 507         if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) {
 508                 int err;
 509
 510                 if (static_branch_likely(&supports_deactivate_key))
 511                         gic_write_eoir(irqnr);
 512                 else
 513                         isb();
 514
 515                 err = handle_domain_irq(gic_data.domain, irqnr, regs);
 516                 if (err) {
 517                         WARN_ONCE(true, "Unexpected interrupt received!\n");
 518                         gic_deactivate_unhandled(irqnr);
 519                 }
 520                 return;
 521         }
 522         if (irqnr < 16) {
 523                 gic_write_eoir(irqnr);
 524                 if (static_branch_likely(&supports_deactivate_key))
 525                         gic_write_dir(irqnr);
 526 #ifdef CONFIG_SMP
 527                 /*
 528                  * Unlike GICv2, we don't need an smp_rmb() here.
 529                  * The control dependency from gic_read_iar to
 530                  * the ISB in gic_write_eoir is enough to ensure
 531                  * that any shared data read by handle_IPI will
 532                  * be read after the ACK.
 533                  */
 534                 handle_IPI(irqnr, regs);
 535 #else
 536                 WARN_ONCE(true, "Unexpected SGI received!\n");
 537 #endif
 538         }
 539 }
 540
 541 static u32 gic_get_pribits(void)
 542 {
 543         u32 pribits;
 544
 545         pribits = gic_read_ctlr();
 546         pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
 547         pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
 548         pribits++;
 549
 550         return pribits;
 551 }
 552
 553 static bool gic_has_group0(void)
 554 {
 555         u32 val;
 556         u32 old_pmr;
 557
 558         old_pmr = gic_read_pmr();
 559
 560         /*
 561          * Let's find out if Group0 is under control of EL3 or not by
 562          * setting the highest possible, non-zero priority in PMR.
 563          *
 564          * If SCR_EL3.FIQ is set, the priority gets shifted down in
 565          * order for the CPU interface to set bit 7, and keep the
 566          * actual priority in the non-secure range. In the process, it
 567          * looses the least significant bit and the actual priority
 568          * becomes 0x80. Reading it back returns 0, indicating that
 569          * we're don't have access to Group0.
 570          */
 571         gic_write_pmr(BIT(8 - gic_get_pribits()));
 572         val = gic_read_pmr();
 573
 574         gic_write_pmr(old_pmr);
 575
 576         return val != 0;
 577 }
 578
 579 static void __init gic_dist_init(void)
 580 {
 581         unsigned int i;
 582         u64 affinity;
 583         void __iomem *base = gic_data.dist_base;
 584
 585         /* Disable the distributor */
 586         writel_relaxed(0, base + GICD_CTLR);
 587         gic_dist_wait_for_rwp();
 588
 589         /*
 590          * Configure SPIs as non-secure Group-1. This will only matter
 591          * if the GIC only has a single security state. This will not
 592          * do the right thing if the kernel is running in secure mode,
 593          * but that's not the intended use case anyway.
 594          */
 595         for (i = 32; i < gic_data.irq_nr; i += 32)
 596                 writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
 597
 598         gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
 599
 600         /* Enable distributor with ARE, Group1 */
 601         writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
 602                        base + GICD_CTLR);
 603
 604         /*
 605          * Set all global interrupts to the boot CPU only. ARE must be
 606          * enabled.
 607          */
 608         affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
 609         for (i = 32; i < gic_data.irq_nr; i++)
 610                 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
 611 }
 612
 613 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
 614 {
 615         int ret = -ENODEV;
 616         int i;
 617
 618         for (i = 0; i < gic_data.nr_redist_regions; i++) {
 619                 void __iomem *ptr = gic_data.redist_regions[i].redist_base;
 620                 u64 typer;
 621                 u32 reg;
 622
 623                 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
 624                 if (reg != GIC_PIDR2_ARCH_GICv3 &&
 625                     reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
 626                         pr_warn("No redistributor present @%p\n", ptr);
 627                         break;
 628                 }
 629
 630                 do {
 631                         typer = gic_read_typer(ptr + GICR_TYPER);
 632                         ret = fn(gic_data.redist_regions + i, ptr);
 633                         if (!ret)
 634                                 return 0;
 635
 636                         if (gic_data.redist_regions[i].single_redist)
 637                                 break;
 638
 639                         if (gic_data.redist_stride) {
 640                                 ptr += gic_data.redist_stride;
 641                         } else {
 642                                 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
 643                                 if (typer & GICR_TYPER_VLPIS)
 644                                         ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
 645                         }
 646                 } while (!(typer & GICR_TYPER_LAST));
 647         }
 648
 649         return ret ? -ENODEV : 0;
 650 }
 651
 652 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
 653 {
 654         unsigned long mpidr = cpu_logical_map(smp_processor_id());
 655         u64 typer;
 656         u32 aff;
 657
 658         /*
 659          * Convert affinity to a 32bit value that can be matched to
 660          * GICR_TYPER bits [63:32].
 661          */
 662         aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
 663                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
 664                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
 665                MPIDR_AFFINITY_LEVEL(mpidr, 0));
 666
 667         typer = gic_read_typer(ptr + GICR_TYPER);
 668         if ((typer >> 32) == aff) {
 669                 u64 offset = ptr - region->redist_base;
 670                 gic_data_rdist_rd_base() = ptr;
 671                 gic_data_rdist()->phys_base = region->phys_base + offset;
 672
 673                 pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
 674                         smp_processor_id(), mpidr,
 675                         (int)(region - gic_data.redist_regions),
 676                         &gic_data_rdist()->phys_base);
 677                 return 0;
 678         }
 679
 680         /* Try next one */
 681         return 1;
 682 }
 683
 684 static int gic_populate_rdist(void)
 685 {
 686         if (gic_iterate_rdists(__gic_populate_rdist) == 0)
 687                 return 0;
 688
 689         /* We couldn't even deal with ourselves... */
 690         WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
 691              smp_processor_id(),
 692              (unsigned long)cpu_logical_map(smp_processor_id()));
 693         return -ENODEV;
 694 }
 695
 696 static int __gic_update_vlpi_properties(struct redist_region *region,
 697                                         void __iomem *ptr)
 698 {
 699         u64 typer = gic_read_typer(ptr + GICR_TYPER);
 700         gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
 701         gic_data.rdists.has_direct_lpi &= !!(typer & GICR_TYPER_DirectLPIS);
 702
 703         return 1;
 704 }
 705
 706 static void gic_update_vlpi_properties(void)
 707 {
 708         gic_iterate_rdists(__gic_update_vlpi_properties);
 709         pr_info("%sVLPI support, %sdirect LPI support\n",
 710                 !gic_data.rdists.has_vlpis ? "no " : "",
 711                 !gic_data.rdists.has_direct_lpi ? "no " : "");
 712 }
 713
 714 /* Check whether it's single security state view */
 715 static inline bool gic_dist_security_disabled(void)
 716 {
 717         return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
 718 }
 719
 720 static void gic_cpu_sys_reg_init(void)
 721 {
 722         int i, cpu = smp_processor_id();
 723         u64 mpidr = cpu_logical_map(cpu);
 724         u64 need_rss = MPIDR_RS(mpidr);
 725         bool group0;
 726         u32 pribits;
 727
 728         /*
 729          * Need to check that the SRE bit has actually been set. If
 730          * not, it means that SRE is disabled at EL2. We're going to
 731          * die painfully, and there is nothing we can do about it.
 732          *
 733          * Kindly inform the luser.
 734          */
 735         if (!gic_enable_sre())
 736                 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
 737
 738         pribits = gic_get_pribits();
 739
 740         group0 = gic_has_group0();
 741
 742         /* Set priority mask register */
 743         if (!gic_prio_masking_enabled()) {
 744                 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
 745         } else {
 746                 /*
 747                  * Mismatch configuration with boot CPU, the system is likely
 748                  * to die as interrupt masking will not work properly on all
 749                  * CPUs
 750                  */
 751                 WARN_ON(gic_supports_nmi() && group0 &&
 752                         !gic_dist_security_disabled());
 753         }
 754
 755         /*
 756          * Some firmwares hand over to the kernel with the BPR changed from
 757          * its reset value (and with a value large enough to prevent
 758          * any pre-emptive interrupts from working at all). Writing a zero
 759          * to BPR restores is reset value.
 760          */
 761         gic_write_bpr1(0);
 762
 763         if (static_branch_likely(&supports_deactivate_key)) {
 764                 /* EOI drops priority only (mode 1) */
 765                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
 766         } else {
 767                 /* EOI deactivates interrupt too (mode 0) */
 768                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
 769         }
 770
 771         /* Always whack Group0 before Group1 */
 772         if (group0) {
 773                 switch(pribits) {
 774                 case 8:
 775                 case 7:
 776                         write_gicreg(0, ICC_AP0R3_EL1);
 777                         write_gicreg(0, ICC_AP0R2_EL1);
 778                 case 6:
 779                         write_gicreg(0, ICC_AP0R1_EL1);
 780                 case 5:
 781                 case 4:
 782                         write_gicreg(0, ICC_AP0R0_EL1);
 783                 }
 784
 785                 isb();
 786         }
 787
 788         switch(pribits) {
 789         case 8:
 790         case 7:
 791                 write_gicreg(0, ICC_AP1R3_EL1);
 792                 write_gicreg(0, ICC_AP1R2_EL1);
 793         case 6:
 794                 write_gicreg(0, ICC_AP1R1_EL1);
 795         case 5:
 796         case 4:
 797                 write_gicreg(0, ICC_AP1R0_EL1);
 798         }
 799
 800         isb();
 801
 802         /* ... and let's hit the road... */
 803         gic_write_grpen1(1);
 804
 805         /* Keep the RSS capability status in per_cpu variable */
 806         per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS);
 807
 808         /* Check all the CPUs have capable of sending SGIs to other CPUs */
 809         for_each_online_cpu(i) {
 810                 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu);
 811
 812                 need_rss |= MPIDR_RS(cpu_logical_map(i));
 813                 if (need_rss && (!have_rss))
 814                         pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
 815                                 cpu, (unsigned long)mpidr,
 816                                 i, (unsigned long)cpu_logical_map(i));
 817         }
 818
 819         /**
 820          * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
 821          * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
 822          * UNPREDICTABLE choice of :
 823          *   - The write is ignored.
 824          *   - The RS field is treated as 0.
 825          */
 826         if (need_rss && (!gic_data.has_rss))
 827                 pr_crit_once("RSS is required but GICD doesn't support it\n");
 828 }
 829
 830 static bool gicv3_nolpi;
 831
 832 static int __init gicv3_nolpi_cfg(char *buf)
 833 {
 834         return strtobool(buf, &gicv3_nolpi);
 835 }
 836 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg);
 837
 838 static int gic_dist_supports_lpis(void)
 839 {
 840         return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
 841                 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
 842                 !gicv3_nolpi);
 843 }
 844
 845 static void gic_cpu_init(void)
 846 {
 847         void __iomem *rbase;
 848
 849         /* Register ourselves with the rest of the world */
 850         if (gic_populate_rdist())
 851                 return;
 852
 853         gic_enable_redist(true);
 854
 855         rbase = gic_data_rdist_sgi_base();
 856
 857         /* Configure SGIs/PPIs as non-secure Group-1 */
 858         writel_relaxed(~0, rbase + GICR_IGROUPR0);
 859
 860         gic_cpu_config(rbase, gic_redist_wait_for_rwp);
 861
 862         /* initialise system registers */
 863         gic_cpu_sys_reg_init();
 864 }
 865
 866 #ifdef CONFIG_SMP
 867
 868 #define MPIDR_TO_SGI_RS(mpidr)  (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
 869 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr)  ((mpidr) & ~0xFUL)
 870
 871 static int gic_starting_cpu(unsigned int cpu)
 872 {
 873         gic_cpu_init();
 874
 875         if (gic_dist_supports_lpis())
 876                 its_cpu_init();
 877
 878         return 0;
 879 }
 880
 881 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
 882                                    unsigned long cluster_id)
 883 {
 884         int next_cpu, cpu = *base_cpu;
 885         unsigned long mpidr = cpu_logical_map(cpu);
 886         u16 tlist = 0;
 887
 888         while (cpu < nr_cpu_ids) {
 889                 tlist |= 1 << (mpidr & 0xf);
 890
 891                 next_cpu = cpumask_next(cpu, mask);
 892                 if (next_cpu >= nr_cpu_ids)
 893                         goto out;
 894                 cpu = next_cpu;
 895
 896                 mpidr = cpu_logical_map(cpu);
 897
 898                 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) {
 899                         cpu--;
 900                         goto out;
 901                 }
 902         }
 903 out:
 904         *base_cpu = cpu;
 905         return tlist;
 906 }
 907
 908 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
 909         (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
 910                 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
 911
 912 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
 913 {
 914         u64 val;
 915
 916         val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)     |
 917                MPIDR_TO_SGI_AFFINITY(cluster_id, 2)     |
 918                irq << ICC_SGI1R_SGI_ID_SHIFT            |
 919                MPIDR_TO_SGI_AFFINITY(cluster_id, 1)     |
 920                MPIDR_TO_SGI_RS(cluster_id)              |
 921                tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
 922
 923         pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
 924         gic_write_sgi1r(val);
 925 }
 926
 927 static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
 928 {
 929         int cpu;
 930
 931         if (WARN_ON(irq >= 16))
 932                 return;
 933
 934         /*
 935          * Ensure that stores to Normal memory are visible to the
 936          * other CPUs before issuing the IPI.
 937          */
 938         wmb();
 939
 940         for_each_cpu(cpu, mask) {
 941                 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu));
 942                 u16 tlist;
 943
 944                 tlist = gic_compute_target_list(&cpu, mask, cluster_id);
 945                 gic_send_sgi(cluster_id, tlist, irq);
 946         }
 947
 948         /* Force the above writes to ICC_SGI1R_EL1 to be executed */
 949         isb();
 950 }
 951
 952 static void gic_smp_init(void)
 953 {
 954         set_smp_cross_call(gic_raise_softirq);
 955         cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
 956                                   "irqchip/arm/gicv3:starting",
 957                                   gic_starting_cpu, NULL);
 958 }
 959
 960 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 961                             bool force)
 962 {
 963         unsigned int cpu;
 964         void __iomem *reg;
 965         int enabled;
 966         u64 val;
 967
 968         if (force)
 969                 cpu = cpumask_first(mask_val);
 970         else
 971                 cpu = cpumask_any_and(mask_val, cpu_online_mask);
 972
 973         if (cpu >= nr_cpu_ids)
 974                 return -EINVAL;
 975
 976         if (gic_irq_in_rdist(d))
 977                 return -EINVAL;
 978
 979         /* If interrupt was enabled, disable it first */
 980         enabled = gic_peek_irq(d, GICD_ISENABLER);
 981         if (enabled)
 982                 gic_mask_irq(d);
 983
 984         reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8);
 985         val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
 986
 987         gic_write_irouter(val, reg);
 988
 989         /*
 990          * If the interrupt was enabled, enabled it again. Otherwise,
 991          * just wait for the distributor to have digested our changes.
 992          */
 993         if (enabled)
 994                 gic_unmask_irq(d);
 995         else
 996                 gic_dist_wait_for_rwp();
 997
 998         irq_data_update_effective_affinity(d, cpumask_of(cpu));
 999
1000         return IRQ_SET_MASK_OK_DONE;
1001 }
1002 #else
1003 #define gic_set_affinity        NULL
1004 #define gic_smp_init()          do { } while(0)
1005 #endif
1006
1007 #ifdef CONFIG_CPU_PM
1008 static int gic_cpu_pm_notifier(struct notifier_block *self,
1009                                unsigned long cmd, void *v)
1010 {
1011         if (cmd == CPU_PM_EXIT) {
1012                 if (gic_dist_security_disabled())
1013                         gic_enable_redist(true);
1014                 gic_cpu_sys_reg_init();
1015         } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
1016                 gic_write_grpen1(0);
1017                 gic_enable_redist(false);
1018         }
1019         return NOTIFY_OK;
1020 }
1021
1022 static struct notifier_block gic_cpu_pm_notifier_block = {
1023         .notifier_call = gic_cpu_pm_notifier,
1024 };
1025
1026 static void gic_cpu_pm_init(void)
1027 {
1028         cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
1029 }
1030
1031 #else
1032 static inline void gic_cpu_pm_init(void) { }
1033 #endif /* CONFIG_CPU_PM */
1034
1035 static struct irq_chip gic_chip = {
1036         .name                   = "GICv3",
1037         .irq_mask               = gic_mask_irq,
1038         .irq_unmask             = gic_unmask_irq,
1039         .irq_eoi                = gic_eoi_irq,
1040         .irq_set_type           = gic_set_type,
1041         .irq_set_affinity       = gic_set_affinity,
1042         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
1043         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
1044         .irq_nmi_setup          = gic_irq_nmi_setup,
1045         .irq_nmi_teardown       = gic_irq_nmi_teardown,
1046         .flags                  = IRQCHIP_SET_TYPE_MASKED |
1047                                   IRQCHIP_SKIP_SET_WAKE |
1048                                   IRQCHIP_MASK_ON_SUSPEND,
1049 };
1050
1051 static struct irq_chip gic_eoimode1_chip = {
1052         .name                   = "GICv3",
1053         .irq_mask               = gic_eoimode1_mask_irq,
1054         .irq_unmask             = gic_unmask_irq,
1055         .irq_eoi                = gic_eoimode1_eoi_irq,
1056         .irq_set_type           = gic_set_type,
1057         .irq_set_affinity       = gic_set_affinity,
1058         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
1059         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
1060         .irq_set_vcpu_affinity  = gic_irq_set_vcpu_affinity,
1061         .irq_nmi_setup          = gic_irq_nmi_setup,
1062         .irq_nmi_teardown       = gic_irq_nmi_teardown,
1063         .flags                  = IRQCHIP_SET_TYPE_MASKED |
1064                                   IRQCHIP_SKIP_SET_WAKE |
1065                                   IRQCHIP_MASK_ON_SUSPEND,
1066 };
1067
1068 #define GIC_ID_NR       (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
1069
1070 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
1071                               irq_hw_number_t hw)
1072 {
1073         struct irq_chip *chip = &gic_chip;
1074
1075         if (static_branch_likely(&supports_deactivate_key))
1076                 chip = &gic_eoimode1_chip;
1077
1078         /* SGIs are private to the core kernel */
1079         if (hw < 16)
1080                 return -EPERM;
1081         /* Nothing here */
1082         if (hw >= gic_data.irq_nr && hw < 8192)
1083                 return -EPERM;
1084         /* Off limits */
1085         if (hw >= GIC_ID_NR)
1086                 return -EPERM;
1087
1088         /* PPIs */
1089         if (hw < 32) {
1090                 irq_set_percpu_devid(irq);
1091                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1092                                     handle_percpu_devid_irq, NULL, NULL);
1093                 irq_set_status_flags(irq, IRQ_NOAUTOEN);
1094         }
1095         /* SPIs */
1096         if (hw >= 32 && hw < gic_data.irq_nr) {
1097                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1098                                     handle_fasteoi_irq, NULL, NULL);
1099                 irq_set_probe(irq);
1100                 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
1101         }
1102         /* LPIs */
1103         if (hw >= 8192 && hw < GIC_ID_NR) {
1104                 if (!gic_dist_supports_lpis())
1105                         return -EPERM;
1106                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1107                                     handle_fasteoi_irq, NULL, NULL);
1108         }
1109
1110         return 0;
1111 }
1112
1113 #define GIC_IRQ_TYPE_PARTITION  (GIC_IRQ_TYPE_LPI + 1)
1114
1115 static int gic_irq_domain_translate(struct irq_domain *d,
1116                                     struct irq_fwspec *fwspec,
1117                                     unsigned long *hwirq,
1118                                     unsigned int *type)
1119 {
1120         if (is_of_node(fwspec->fwnode)) {
1121                 if (fwspec->param_count < 3)
1122                         return -EINVAL;
1123
1124                 switch (fwspec->param[0]) {
1125                 case 0:                 /* SPI */
1126                         *hwirq = fwspec->param[1] + 32;
1127                         break;
1128                 case 1:                 /* PPI */
1129                 case GIC_IRQ_TYPE_PARTITION:
1130                         *hwirq = fwspec->param[1] + 16;
1131                         break;
1132                 case GIC_IRQ_TYPE_LPI:  /* LPI */
1133                         *hwirq = fwspec->param[1];
1134                         break;
1135                 default:
1136                         return -EINVAL;
1137                 }
1138
1139                 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1140
1141                 /*
1142                  * Make it clear that broken DTs are... broken.
1143                  * Partitionned PPIs are an unfortunate exception.
1144                  */
1145                 WARN_ON(*type == IRQ_TYPE_NONE &&
1146                         fwspec->param[0] != GIC_IRQ_TYPE_PARTITION);
1147                 return 0;
1148         }
1149
1150         if (is_fwnode_irqchip(fwspec->fwnode)) {
1151                 if(fwspec->param_count != 2)
1152                         return -EINVAL;
1153
1154                 *hwirq = fwspec->param[0];
1155                 *type = fwspec->param[1];
1156
1157                 WARN_ON(*type == IRQ_TYPE_NONE);
1158                 return 0;
1159         }
1160
1161         return -EINVAL;
1162 }
1163
1164 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1165                                 unsigned int nr_irqs, void *arg)
1166 {
1167         int i, ret;
1168         irq_hw_number_t hwirq;
1169         unsigned int type = IRQ_TYPE_NONE;
1170         struct irq_fwspec *fwspec = arg;
1171
1172         ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
1173         if (ret)
1174                 return ret;
1175
1176         for (i = 0; i < nr_irqs; i++) {
1177                 ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
1178                 if (ret)
1179                         return ret;
1180         }
1181
1182         return 0;
1183 }
1184
1185 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1186                                 unsigned int nr_irqs)
1187 {
1188         int i;
1189
1190         for (i = 0; i < nr_irqs; i++) {
1191                 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
1192                 irq_set_handler(virq + i, NULL);
1193                 irq_domain_reset_irq_data(d);
1194         }
1195 }
1196
1197 static int gic_irq_domain_select(struct irq_domain *d,
1198                                  struct irq_fwspec *fwspec,
1199                                  enum irq_domain_bus_token bus_token)
1200 {
1201         /* Not for us */
1202         if (fwspec->fwnode != d->fwnode)
1203                 return 0;
1204
1205         /* If this is not DT, then we have a single domain */
1206         if (!is_of_node(fwspec->fwnode))
1207                 return 1;
1208
1209         /*
1210          * If this is a PPI and we have a 4th (non-null) parameter,
1211          * then we need to match the partition domain.
1212          */
1213         if (fwspec->param_count >= 4 &&
1214             fwspec->param[0] == 1 && fwspec->param[3] != 0)
1215                 return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]);
1216
1217         return d == gic_data.domain;
1218 }
1219
1220 static const struct irq_domain_ops gic_irq_domain_ops = {
1221         .translate = gic_irq_domain_translate,
1222         .alloc = gic_irq_domain_alloc,
1223         .free = gic_irq_domain_free,
1224         .select = gic_irq_domain_select,
1225 };
1226
1227 static int partition_domain_translate(struct irq_domain *d,
1228                                       struct irq_fwspec *fwspec,
1229                                       unsigned long *hwirq,
1230                                       unsigned int *type)
1231 {
1232         struct device_node *np;
1233         int ret;
1234
1235         np = of_find_node_by_phandle(fwspec->param[3]);
1236         if (WARN_ON(!np))
1237                 return -EINVAL;
1238
1239         ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]],
1240                                      of_node_to_fwnode(np));
1241         if (ret < 0)
1242                 return ret;
1243
1244         *hwirq = ret;
1245         *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1246
1247         return 0;
1248 }
1249
1250 static const struct irq_domain_ops partition_domain_ops = {
1251         .translate = partition_domain_translate,
1252         .select = gic_irq_domain_select,
1253 };
1254
1255 static bool gic_enable_quirk_msm8996(void *data)
1256 {
1257         struct gic_chip_data *d = data;
1258
1259         d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;
1260
1261         return true;
1262 }
1263
1264 static void gic_enable_nmi_support(void)
1265 {
1266         int i;
1267
1268         for (i = 0; i < 16; i++)
1269                 refcount_set(&ppi_nmi_refs[i], 0);
1270
1271         static_branch_enable(&supports_pseudo_nmis);
1272
1273         if (static_branch_likely(&supports_deactivate_key))
1274                 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1275         else
1276                 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1277 }
1278
1279 static int __init gic_init_bases(void __iomem *dist_base,
1280                                  struct redist_region *rdist_regs,
1281                                  u32 nr_redist_regions,
1282                                  u64 redist_stride,
1283                                  struct fwnode_handle *handle)
1284 {
1285         u32 typer;
1286         int gic_irqs;
1287         int err;
1288
1289         if (!is_hyp_mode_available())
1290                 static_branch_disable(&supports_deactivate_key);
1291
1292         if (static_branch_likely(&supports_deactivate_key))
1293                 pr_info("GIC: Using split EOI/Deactivate mode\n");
1294
1295         gic_data.fwnode = handle;
1296         gic_data.dist_base = dist_base;
1297         gic_data.redist_regions = rdist_regs;
1298         gic_data.nr_redist_regions = nr_redist_regions;
1299         gic_data.redist_stride = redist_stride;
1300
1301         /*
1302          * Find out how many interrupts are supported.
1303          * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
1304          */
1305         typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
1306         gic_data.rdists.gicd_typer = typer;
1307         gic_irqs = GICD_TYPER_IRQS(typer);
1308         if (gic_irqs > 1020)
1309                 gic_irqs = 1020;
1310         gic_data.irq_nr = gic_irqs;
1311
1312         gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
1313                                                  &gic_data);
1314         irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
1315         gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
1316         gic_data.rdists.has_vlpis = true;
1317         gic_data.rdists.has_direct_lpi = true;
1318
1319         if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
1320                 err = -ENOMEM;
1321                 goto out_free;
1322         }
1323
1324         gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
1325         pr_info("Distributor has %sRange Selector support\n",
1326                 gic_data.has_rss ? "" : "no ");
1327
1328         if (typer & GICD_TYPER_MBIS) {
1329                 err = mbi_init(handle, gic_data.domain);
1330                 if (err)
1331                         pr_err("Failed to initialize MBIs\n");
1332         }
1333
1334         set_handle_irq(gic_handle_irq);
1335
1336         gic_update_vlpi_properties();
1337
1338         gic_smp_init();
1339         gic_dist_init();
1340         gic_cpu_init();
1341         gic_cpu_pm_init();
1342
1343         if (gic_dist_supports_lpis()) {
1344                 its_init(handle, &gic_data.rdists, gic_data.domain);
1345                 its_cpu_init();
1346         }
1347
1348         if (gic_prio_masking_enabled()) {
1349                 if (!gic_has_group0() || gic_dist_security_disabled())
1350                         gic_enable_nmi_support();
1351                 else
1352                         pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n");
1353         }
1354
1355         return 0;
1356
1357 out_free:
1358         if (gic_data.domain)
1359                 irq_domain_remove(gic_data.domain);
1360         free_percpu(gic_data.rdists.rdist);
1361         return err;
1362 }
1363
1364 static int __init gic_validate_dist_version(void __iomem *dist_base)
1365 {
1366         u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
1367
1368         if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
1369                 return -ENODEV;
1370
1371         return 0;
1372 }
1373
1374 /* Create all possible partitions at boot time */
1375 static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
1376 {
1377         struct device_node *parts_node, *child_part;
1378         int part_idx = 0, i;
1379         int nr_parts;
1380         struct partition_affinity *parts;
1381
1382         parts_node = of_get_child_by_name(gic_node, "ppi-partitions");
1383         if (!parts_node)
1384                 return;
1385
1386         nr_parts = of_get_child_count(parts_node);
1387
1388         if (!nr_parts)
1389                 goto out_put_node;
1390
1391         parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL);
1392         if (WARN_ON(!parts))
1393                 goto out_put_node;
1394
1395         for_each_child_of_node(parts_node, child_part) {
1396                 struct partition_affinity *part;
1397                 int n;
1398
1399                 part = &parts[part_idx];
1400
1401                 part->partition_id = of_node_to_fwnode(child_part);
1402
1403                 pr_info("GIC: PPI partition %pOFn[%d] { ",
1404                         child_part, part_idx);
1405
1406                 n = of_property_count_elems_of_size(child_part, "affinity",
1407                                                     sizeof(u32));
1408                 WARN_ON(n <= 0);
1409
1410                 for (i = 0; i < n; i++) {
1411                         int err, cpu;
1412                         u32 cpu_phandle;
1413                         struct device_node *cpu_node;
1414
1415                         err = of_property_read_u32_index(child_part, "affinity",
1416                                                          i, &cpu_phandle);
1417                         if (WARN_ON(err))
1418                                 continue;
1419
1420                         cpu_node = of_find_node_by_phandle(cpu_phandle);
1421                         if (WARN_ON(!cpu_node))
1422                                 continue;
1423
1424                         cpu = of_cpu_node_to_id(cpu_node);
1425                         if (WARN_ON(cpu < 0))
1426                                 continue;
1427
1428                         pr_cont("%pOF[%d] ", cpu_node, cpu);
1429
1430                         cpumask_set_cpu(cpu, &part->mask);
1431                 }
1432
1433                 pr_cont("}\n");
1434                 part_idx++;
1435         }
1436
1437         for (i = 0; i < 16; i++) {
1438                 unsigned int irq;
1439                 struct partition_desc *desc;
1440                 struct irq_fwspec ppi_fwspec = {
1441                         .fwnode         = gic_data.fwnode,
1442                         .param_count    = 3,
1443                         .param          = {
1444                                 [0]     = GIC_IRQ_TYPE_PARTITION,
1445                                 [1]     = i,
1446                                 [2]     = IRQ_TYPE_NONE,
1447                         },
1448                 };
1449
1450                 irq = irq_create_fwspec_mapping(&ppi_fwspec);
1451                 if (WARN_ON(!irq))
1452                         continue;
1453                 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
1454                                              irq, &partition_domain_ops);
1455                 if (WARN_ON(!desc))
1456                         continue;
1457
1458                 gic_data.ppi_descs[i] = desc;
1459         }
1460
1461 out_put_node:
1462         of_node_put(parts_node);
1463 }
1464
1465 static void __init gic_of_setup_kvm_info(struct device_node *node)
1466 {
1467         int ret;
1468         struct resource r;
1469         u32 gicv_idx;
1470
1471         gic_v3_kvm_info.type = GIC_V3;
1472
1473         gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
1474         if (!gic_v3_kvm_info.maint_irq)
1475                 return;
1476
1477         if (of_property_read_u32(node, "#redistributor-regions",
1478                                  &gicv_idx))
1479                 gicv_idx = 1;
1480
1481         gicv_idx += 3;  /* Also skip GICD, GICC, GICH */
1482         ret = of_address_to_resource(node, gicv_idx, &r);
1483         if (!ret)
1484                 gic_v3_kvm_info.vcpu = r;
1485
1486         gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
1487         gic_set_kvm_info(&gic_v3_kvm_info);
1488 }
1489
1490 static const struct gic_quirk gic_quirks[] = {
1491         {
1492                 .desc   = "GICv3: Qualcomm MSM8996 broken firmware",
1493                 .compatible = "qcom,msm8996-gic-v3",
1494                 .init   = gic_enable_quirk_msm8996,
1495         },
1496         {
1497         }
1498 };
1499
1500 static int __init gic_of_init(struct device_node *node, struct device_node *parent)
1501 {
1502         void __iomem *dist_base;
1503         struct redist_region *rdist_regs;
1504         u64 redist_stride;
1505         u32 nr_redist_regions;
1506         int err, i;
1507
1508         dist_base = of_iomap(node, 0);
1509         if (!dist_base) {
1510                 pr_err("%pOF: unable to map gic dist registers\n", node);
1511                 return -ENXIO;
1512         }
1513
1514         err = gic_validate_dist_version(dist_base);
1515         if (err) {
1516                 pr_err("%pOF: no distributor detected, giving up\n", node);
1517                 goto out_unmap_dist;
1518         }
1519
1520         if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
1521                 nr_redist_regions = 1;
1522
1523         rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs),
1524                              GFP_KERNEL);
1525         if (!rdist_regs) {
1526                 err = -ENOMEM;
1527                 goto out_unmap_dist;
1528         }
1529
1530         for (i = 0; i < nr_redist_regions; i++) {
1531                 struct resource res;
1532                 int ret;
1533
1534                 ret = of_address_to_resource(node, 1 + i, &res);
1535                 rdist_regs[i].redist_base = of_iomap(node, 1 + i);
1536                 if (ret || !rdist_regs[i].redist_base) {
1537                         pr_err("%pOF: couldn't map region %d\n", node, i);
1538                         err = -ENODEV;
1539                         goto out_unmap_rdist;
1540                 }
1541                 rdist_regs[i].phys_base = res.start;
1542         }
1543
1544         if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
1545                 redist_stride = 0;
1546
1547         gic_enable_of_quirks(node, gic_quirks, &gic_data);
1548
1549         err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
1550                              redist_stride, &node->fwnode);
1551         if (err)
1552                 goto out_unmap_rdist;
1553
1554         gic_populate_ppi_partitions(node);
1555
1556         if (static_branch_likely(&supports_deactivate_key))
1557                 gic_of_setup_kvm_info(node);
1558         return 0;
1559
1560 out_unmap_rdist:
1561         for (i = 0; i < nr_redist_regions; i++)
1562                 if (rdist_regs[i].redist_base)
1563                         iounmap(rdist_regs[i].redist_base);
1564         kfree(rdist_regs);
1565 out_unmap_dist:
1566         iounmap(dist_base);
1567         return err;
1568 }
1569
1570 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);
1571
1572 #ifdef CONFIG_ACPI
1573 static struct
1574 {
1575         void __iomem *dist_base;
1576         struct redist_region *redist_regs;
1577         u32 nr_redist_regions;
1578         bool single_redist;
1579         u32 maint_irq;
1580         int maint_irq_mode;
1581         phys_addr_t vcpu_base;
1582 } acpi_data __initdata;
1583
1584 static void __init
1585 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base)
1586 {
1587         static int count = 0;
1588
1589         acpi_data.redist_regs[count].phys_base = phys_base;
1590         acpi_data.redist_regs[count].redist_base = redist_base;
1591         acpi_data.redist_regs[count].single_redist = acpi_data.single_redist;
1592         count++;
1593 }
1594
1595 static int __init
1596 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
1597                            const unsigned long end)
1598 {
1599         struct acpi_madt_generic_redistributor *redist =
1600                         (struct acpi_madt_generic_redistributor *)header;
1601         void __iomem *redist_base;
1602
1603         redist_base = ioremap(redist->base_address, redist->length);
1604         if (!redist_base) {
1605                 pr_err("Couldn't map GICR region @%llx\n", redist->base_address);
1606                 return -ENOMEM;
1607         }
1608
1609         gic_acpi_register_redist(redist->base_address, redist_base);
1610         return 0;
1611 }
1612
1613 static int __init
1614 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
1615                          const unsigned long end)
1616 {
1617         struct acpi_madt_generic_interrupt *gicc =
1618                                 (struct acpi_madt_generic_interrupt *)header;
1619         u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
1620         u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
1621         void __iomem *redist_base;
1622
1623         /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */
1624         if (!(gicc->flags & ACPI_MADT_ENABLED))
1625                 return 0;
1626
1627         redist_base = ioremap(gicc->gicr_base_address, size);
1628         if (!redist_base)
1629                 return -ENOMEM;
1630
1631         gic_acpi_register_redist(gicc->gicr_base_address, redist_base);
1632         return 0;
1633 }
1634
1635 static int __init gic_acpi_collect_gicr_base(void)
1636 {
1637         acpi_tbl_entry_handler redist_parser;
1638         enum acpi_madt_type type;
1639
1640         if (acpi_data.single_redist) {
1641                 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT;
1642                 redist_parser = gic_acpi_parse_madt_gicc;
1643         } else {
1644                 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR;
1645                 redist_parser = gic_acpi_parse_madt_redist;
1646         }
1647
1648         /* Collect redistributor base addresses in GICR entries */
1649         if (acpi_table_parse_madt(type, redist_parser, 0) > 0)
1650                 return 0;
1651
1652         pr_info("No valid GICR entries exist\n");
1653         return -ENODEV;
1654 }
1655
1656 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
1657                                   const unsigned long end)
1658 {
1659         /* Subtable presence means that redist exists, that's it */
1660         return 0;
1661 }
1662
1663 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
1664                                       const unsigned long end)
1665 {
1666         struct acpi_madt_generic_interrupt *gicc =
1667                                 (struct acpi_madt_generic_interrupt *)header;
1668
1669         /*
1670          * If GICC is enabled and has valid gicr base address, then it means
1671          * GICR base is presented via GICC
1672          */
1673         if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address)
1674                 return 0;
1675
1676         /*
1677          * It's perfectly valid firmware can pass disabled GICC entry, driver
1678          * should not treat as errors, skip the entry instead of probe fail.
1679          */
1680         if (!(gicc->flags & ACPI_MADT_ENABLED))
1681                 return 0;
1682
1683         return -ENODEV;
1684 }
1685
1686 static int __init gic_acpi_count_gicr_regions(void)
1687 {
1688         int count;
1689
1690         /*
1691          * Count how many redistributor regions we have. It is not allowed
1692          * to mix redistributor description, GICR and GICC subtables have to be
1693          * mutually exclusive.
1694          */
1695         count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
1696                                       gic_acpi_match_gicr, 0);
1697         if (count > 0) {
1698                 acpi_data.single_redist = false;
1699                 return count;
1700         }
1701
1702         count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
1703                                       gic_acpi_match_gicc, 0);
1704         if (count > 0)
1705                 acpi_data.single_redist = true;
1706
1707         return count;
1708 }
1709
1710 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
1711                                            struct acpi_probe_entry *ape)
1712 {
1713         struct acpi_madt_generic_distributor *dist;
1714         int count;
1715
1716         dist = (struct acpi_madt_generic_distributor *)header;
1717         if (dist->version != ape->driver_data)
1718                 return false;
1719
1720         /* We need to do that exercise anyway, the sooner the better */
1721         count = gic_acpi_count_gicr_regions();
1722         if (count <= 0)
1723                 return false;
1724
1725         acpi_data.nr_redist_regions = count;
1726         return true;
1727 }
1728
1729 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
1730                                                 const unsigned long end)
1731 {
1732         struct acpi_madt_generic_interrupt *gicc =
1733                 (struct acpi_madt_generic_interrupt *)header;
1734         int maint_irq_mode;
1735         static int first_madt = true;
1736
1737         /* Skip unusable CPUs */
1738         if (!(gicc->flags & ACPI_MADT_ENABLED))
1739                 return 0;
1740
1741         maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
1742                 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
1743
1744         if (first_madt) {
1745                 first_madt = false;
1746
1747                 acpi_data.maint_irq = gicc->vgic_interrupt;
1748                 acpi_data.maint_irq_mode = maint_irq_mode;
1749                 acpi_data.vcpu_base = gicc->gicv_base_address;
1750
1751                 return 0;
1752         }
1753
1754         /*
1755          * The maintenance interrupt and GICV should be the same for every CPU
1756          */
1757         if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
1758             (acpi_data.maint_irq_mode != maint_irq_mode) ||
1759             (acpi_data.vcpu_base != gicc->gicv_base_address))
1760                 return -EINVAL;
1761
1762         return 0;
1763 }
1764
1765 static bool __init gic_acpi_collect_virt_info(void)
1766 {
1767         int count;
1768
1769         count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
1770                                       gic_acpi_parse_virt_madt_gicc, 0);
1771
1772         return (count > 0);
1773 }
1774
1775 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
1776 #define ACPI_GICV2_VCTRL_MEM_SIZE       (SZ_4K)
1777 #define ACPI_GICV2_VCPU_MEM_SIZE        (SZ_8K)
1778
1779 static void __init gic_acpi_setup_kvm_info(void)
1780 {
1781         int irq;
1782
1783         if (!gic_acpi_collect_virt_info()) {
1784                 pr_warn("Unable to get hardware information used for virtualization\n");
1785                 return;
1786         }
1787
1788         gic_v3_kvm_info.type = GIC_V3;
1789
1790         irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
1791                                 acpi_data.maint_irq_mode,
1792                                 ACPI_ACTIVE_HIGH);
1793         if (irq <= 0)
1794                 return;
1795
1796         gic_v3_kvm_info.maint_irq = irq;
1797
1798         if (acpi_data.vcpu_base) {
1799                 struct resource *vcpu = &gic_v3_kvm_info.vcpu;
1800
1801                 vcpu->flags = IORESOURCE_MEM;
1802                 vcpu->start = acpi_data.vcpu_base;
1803                 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
1804         }
1805
1806         gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
1807         gic_set_kvm_info(&gic_v3_kvm_info);
1808 }
1809
1810 static int __init
1811 gic_acpi_init(struct acpi_subtable_header *header, const unsigned long end)
1812 {
1813         struct acpi_madt_generic_distributor *dist;
1814         struct fwnode_handle *domain_handle;
1815         size_t size;
1816         int i, err;
1817
1818         /* Get distributor base address */
1819         dist = (struct acpi_madt_generic_distributor *)header;
1820         acpi_data.dist_base = ioremap(dist->base_address,
1821                                       ACPI_GICV3_DIST_MEM_SIZE);
1822         if (!acpi_data.dist_base) {
1823                 pr_err("Unable to map GICD registers\n");
1824                 return -ENOMEM;
1825         }
1826
1827         err = gic_validate_dist_version(acpi_data.dist_base);
1828         if (err) {
1829                 pr_err("No distributor detected at @%p, giving up\n",
1830                        acpi_data.dist_base);
1831                 goto out_dist_unmap;
1832         }
1833
1834         size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions;
1835         acpi_data.redist_regs = kzalloc(size, GFP_KERNEL);
1836         if (!acpi_data.redist_regs) {
1837                 err = -ENOMEM;
1838                 goto out_dist_unmap;
1839         }
1840
1841         err = gic_acpi_collect_gicr_base();
1842         if (err)
1843                 goto out_redist_unmap;
1844
1845         domain_handle = irq_domain_alloc_fwnode(acpi_data.dist_base);
1846         if (!domain_handle) {
1847                 err = -ENOMEM;
1848                 goto out_redist_unmap;
1849         }
1850
1851         err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs,
1852                              acpi_data.nr_redist_regions, 0, domain_handle);
1853         if (err)
1854                 goto out_fwhandle_free;
1855
1856         acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle);
1857
1858         if (static_branch_likely(&supports_deactivate_key))
1859                 gic_acpi_setup_kvm_info();
1860
1861         return 0;
1862
1863 out_fwhandle_free:
1864         irq_domain_free_fwnode(domain_handle);
1865 out_redist_unmap:
1866         for (i = 0; i < acpi_data.nr_redist_regions; i++)
1867                 if (acpi_data.redist_regs[i].redist_base)
1868                         iounmap(acpi_data.redist_regs[i].redist_base);
1869         kfree(acpi_data.redist_regs);
1870 out_dist_unmap:
1871         iounmap(acpi_data.dist_base);
1872         return err;
1873 }
1874 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
1875                      acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3,
1876                      gic_acpi_init);
1877 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
1878                      acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4,
1879                      gic_acpi_init);
1880 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
1881                      acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE,
1882                      gic_acpi_init);
1883 #endif