arch/powerpc/kvm/mpic.c

   1 /*
   2  * OpenPIC emulation
   3  *
   4  * Copyright (c) 2004 Jocelyn Mayer
   5  *               2011 Alexander Graf
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a copy
   8  * of this software and associated documentation files (the "Software"), to deal
   9  * in the Software without restriction, including without limitation the rights
  10  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  11  * copies of the Software, and to permit persons to whom the Software is
  12  * furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice shall be included in
  15  * all copies or substantial portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23  * THE SOFTWARE.
  24  */
  25
  26 #include <linux/slab.h>
  27 #include <linux/mutex.h>
  28 #include <linux/kvm_host.h>
  29 #include <linux/errno.h>
  30 #include <linux/fs.h>
  31 #include <linux/anon_inodes.h>
  32 #include <asm/uaccess.h>
  33 #include <asm/mpic.h>
  34 #include <asm/kvm_para.h>
  35 #include <asm/kvm_host.h>
  36 #include <asm/kvm_ppc.h>
  37 #include "iodev.h"
  38
  39 #define MAX_CPU     32
  40 #define MAX_SRC     256
  41 #define MAX_TMR     4
  42 #define MAX_IPI     4
  43 #define MAX_MSI     8
  44 #define MAX_IRQ     (MAX_SRC + MAX_IPI + MAX_TMR)
  45 #define VID         0x03        /* MPIC version ID */
  46
  47 /* OpenPIC capability flags */
  48 #define OPENPIC_FLAG_IDR_CRIT     (1 << 0)
  49 #define OPENPIC_FLAG_ILR          (2 << 0)
  50
  51 /* OpenPIC address map */
  52 #define OPENPIC_REG_SIZE             0x40000
  53 #define OPENPIC_GLB_REG_START        0x0
  54 #define OPENPIC_GLB_REG_SIZE         0x10F0
  55 #define OPENPIC_TMR_REG_START        0x10F0
  56 #define OPENPIC_TMR_REG_SIZE         0x220
  57 #define OPENPIC_MSI_REG_START        0x1600
  58 #define OPENPIC_MSI_REG_SIZE         0x200
  59 #define OPENPIC_SUMMARY_REG_START    0x3800
  60 #define OPENPIC_SUMMARY_REG_SIZE     0x800
  61 #define OPENPIC_SRC_REG_START        0x10000
  62 #define OPENPIC_SRC_REG_SIZE         (MAX_SRC * 0x20)
  63 #define OPENPIC_CPU_REG_START        0x20000
  64 #define OPENPIC_CPU_REG_SIZE         (0x100 + ((MAX_CPU - 1) * 0x1000))
  65
  66 struct fsl_mpic_info {
  67         int max_ext;
  68 };
  69
  70 static struct fsl_mpic_info fsl_mpic_20 = {
  71         .max_ext = 12,
  72 };
  73
  74 static struct fsl_mpic_info fsl_mpic_42 = {
  75         .max_ext = 12,
  76 };
  77
  78 #define FRR_NIRQ_SHIFT    16
  79 #define FRR_NCPU_SHIFT     8
  80 #define FRR_VID_SHIFT      0
  81
  82 #define VID_REVISION_1_2   2
  83 #define VID_REVISION_1_3   3
  84
  85 #define VIR_GENERIC      0x00000000     /* Generic Vendor ID */
  86
  87 #define GCR_RESET        0x80000000
  88 #define GCR_MODE_PASS    0x00000000
  89 #define GCR_MODE_MIXED   0x20000000
  90 #define GCR_MODE_PROXY   0x60000000
  91
  92 #define TBCR_CI           0x80000000    /* count inhibit */
  93 #define TCCR_TOG          0x80000000    /* toggles when decrement to zero */
  94
  95 #define IDR_EP_SHIFT      31
  96 #define IDR_EP_MASK       (1 << IDR_EP_SHIFT)
  97 #define IDR_CI0_SHIFT     30
  98 #define IDR_CI1_SHIFT     29
  99 #define IDR_P1_SHIFT      1
 100 #define IDR_P0_SHIFT      0
 101
 102 #define ILR_INTTGT_MASK   0x000000ff
 103 #define ILR_INTTGT_INT    0x00
 104 #define ILR_INTTGT_CINT   0x01  /* critical */
 105 #define ILR_INTTGT_MCP    0x02  /* machine check */
 106 #define NUM_OUTPUTS       3
 107
 108 #define MSIIR_OFFSET       0x140
 109 #define MSIIR_SRS_SHIFT    29
 110 #define MSIIR_SRS_MASK     (0x7 << MSIIR_SRS_SHIFT)
 111 #define MSIIR_IBS_SHIFT    24
 112 #define MSIIR_IBS_MASK     (0x1f << MSIIR_IBS_SHIFT)
 113
 114 static int get_current_cpu(void)
 115 {
 116 #if defined(CONFIG_KVM) && defined(CONFIG_BOOKE)
 117         struct kvm_vcpu *vcpu = current->thread.kvm_vcpu;
 118         return vcpu ? vcpu->arch.irq_cpu_id : -1;
 119 #else
 120         /* XXX */
 121         return -1;
 122 #endif
 123 }
 124
 125 static int openpic_cpu_write_internal(void *opaque, gpa_t addr,
 126                                       u32 val, int idx);
 127 static int openpic_cpu_read_internal(void *opaque, gpa_t addr,
 128                                      u32 *ptr, int idx);
 129 static inline void write_IRQreg_idr(struct openpic *opp, int n_IRQ,
 130                                     uint32_t val);
 131
 132 enum irq_type {
 133         IRQ_TYPE_NORMAL = 0,
 134         IRQ_TYPE_FSLINT,        /* FSL internal interrupt -- level only */
 135         IRQ_TYPE_FSLSPECIAL,    /* FSL timer/IPI interrupt, edge, no polarity */
 136 };
 137
 138 struct irq_queue {
 139         /* Round up to the nearest 64 IRQs so that the queue length
 140          * won't change when moving between 32 and 64 bit hosts.
 141          */
 142         unsigned long queue[BITS_TO_LONGS((MAX_IRQ + 63) & ~63)];
 143         int next;
 144         int priority;
 145 };
 146
 147 struct irq_source {
 148         uint32_t ivpr;          /* IRQ vector/priority register */
 149         uint32_t idr;           /* IRQ destination register */
 150         uint32_t destmask;      /* bitmap of CPU destinations */
 151         int last_cpu;
 152         int output;             /* IRQ level, e.g. ILR_INTTGT_INT */
 153         int pending;            /* TRUE if IRQ is pending */
 154         enum irq_type type;
 155         bool level:1;           /* level-triggered */
 156         bool nomask:1;  /* critical interrupts ignore mask on some FSL MPICs */
 157 };
 158
 159 #define IVPR_MASK_SHIFT       31
 160 #define IVPR_MASK_MASK        (1 << IVPR_MASK_SHIFT)
 161 #define IVPR_ACTIVITY_SHIFT   30
 162 #define IVPR_ACTIVITY_MASK    (1 << IVPR_ACTIVITY_SHIFT)
 163 #define IVPR_MODE_SHIFT       29
 164 #define IVPR_MODE_MASK        (1 << IVPR_MODE_SHIFT)
 165 #define IVPR_POLARITY_SHIFT   23
 166 #define IVPR_POLARITY_MASK    (1 << IVPR_POLARITY_SHIFT)
 167 #define IVPR_SENSE_SHIFT      22
 168 #define IVPR_SENSE_MASK       (1 << IVPR_SENSE_SHIFT)
 169
 170 #define IVPR_PRIORITY_MASK     (0xF << 16)
 171 #define IVPR_PRIORITY(_ivprr_) ((int)(((_ivprr_) & IVPR_PRIORITY_MASK) >> 16))
 172 #define IVPR_VECTOR(opp, _ivprr_) ((_ivprr_) & (opp)->vector_mask)
 173
 174 /* IDR[EP/CI] are only for FSL MPIC prior to v4.0 */
 175 #define IDR_EP      0x80000000  /* external pin */
 176 #define IDR_CI      0x40000000  /* critical interrupt */
 177
 178 struct irq_dest {
 179         struct kvm_vcpu *vcpu;
 180
 181         int32_t ctpr;           /* CPU current task priority */
 182         struct irq_queue raised;
 183         struct irq_queue servicing;
 184
 185         /* Count of IRQ sources asserting on non-INT outputs */
 186         uint32_t outputs_active[NUM_OUTPUTS];
 187 };
 188
 189 #define MAX_MMIO_REGIONS 10
 190
 191 struct openpic {
 192         struct kvm *kvm;
 193         struct kvm_device *dev;
 194         struct kvm_io_device mmio;
 195         const struct mem_reg *mmio_regions[MAX_MMIO_REGIONS];
 196         int num_mmio_regions;
 197
 198         gpa_t reg_base;
 199         spinlock_t lock;
 200
 201         /* Behavior control */
 202         struct fsl_mpic_info *fsl;
 203         uint32_t model;
 204         uint32_t flags;
 205         uint32_t nb_irqs;
 206         uint32_t vid;
 207         uint32_t vir;           /* Vendor identification register */
 208         uint32_t vector_mask;
 209         uint32_t tfrr_reset;
 210         uint32_t ivpr_reset;
 211         uint32_t idr_reset;
 212         uint32_t brr1;
 213         uint32_t mpic_mode_mask;
 214
 215         /* Global registers */
 216         uint32_t frr;           /* Feature reporting register */
 217         uint32_t gcr;           /* Global configuration register  */
 218         uint32_t pir;           /* Processor initialization register */
 219         uint32_t spve;          /* Spurious vector register */
 220         uint32_t tfrr;          /* Timer frequency reporting register */
 221         /* Source registers */
 222         struct irq_source src[MAX_IRQ];
 223         /* Local registers per output pin */
 224         struct irq_dest dst[MAX_CPU];
 225         uint32_t nb_cpus;
 226         /* Timer registers */
 227         struct {
 228                 uint32_t tccr;  /* Global timer current count register */
 229                 uint32_t tbcr;  /* Global timer base count register */
 230         } timers[MAX_TMR];
 231         /* Shared MSI registers */
 232         struct {
 233                 uint32_t msir;  /* Shared Message Signaled Interrupt Register */
 234         } msi[MAX_MSI];
 235         uint32_t max_irq;
 236         uint32_t irq_ipi0;
 237         uint32_t irq_tim0;
 238         uint32_t irq_msi;
 239 };
 240
 241
 242 static void mpic_irq_raise(struct openpic *opp, struct irq_dest *dst,
 243                            int output)
 244 {
 245         struct kvm_interrupt irq = {
 246                 .irq = KVM_INTERRUPT_SET_LEVEL,
 247         };
 248
 249         if (!dst->vcpu) {
 250                 pr_debug("%s: destination cpu %d does not exist\n",
 251                          __func__, (int)(dst - &opp->dst[0]));
 252                 return;
 253         }
 254
 255         pr_debug("%s: cpu %d output %d\n", __func__, dst->vcpu->arch.irq_cpu_id,
 256                 output);
 257
 258         if (output != ILR_INTTGT_INT)   /* TODO */
 259                 return;
 260
 261         kvm_vcpu_ioctl_interrupt(dst->vcpu, &irq);
 262 }
 263
 264 static void mpic_irq_lower(struct openpic *opp, struct irq_dest *dst,
 265                            int output)
 266 {
 267         if (!dst->vcpu) {
 268                 pr_debug("%s: destination cpu %d does not exist\n",
 269                          __func__, (int)(dst - &opp->dst[0]));
 270                 return;
 271         }
 272
 273         pr_debug("%s: cpu %d output %d\n", __func__, dst->vcpu->arch.irq_cpu_id,
 274                 output);
 275
 276         if (output != ILR_INTTGT_INT)   /* TODO */
 277                 return;
 278
 279         kvmppc_core_dequeue_external(dst->vcpu);
 280 }
 281
 282 static inline void IRQ_setbit(struct irq_queue *q, int n_IRQ)
 283 {
 284         set_bit(n_IRQ, q->queue);
 285 }
 286
 287 static inline void IRQ_resetbit(struct irq_queue *q, int n_IRQ)
 288 {
 289         clear_bit(n_IRQ, q->queue);
 290 }
 291
 292 static inline int IRQ_testbit(struct irq_queue *q, int n_IRQ)
 293 {
 294         return test_bit(n_IRQ, q->queue);
 295 }
 296
 297 static void IRQ_check(struct openpic *opp, struct irq_queue *q)
 298 {
 299         int irq = -1;
 300         int next = -1;
 301         int priority = -1;
 302
 303         for (;;) {
 304                 irq = find_next_bit(q->queue, opp->max_irq, irq + 1);
 305                 if (irq == opp->max_irq)
 306                         break;
 307
 308                 pr_debug("IRQ_check: irq %d set ivpr_pr=%d pr=%d\n",
 309                         irq, IVPR_PRIORITY(opp->src[irq].ivpr), priority);
 310
 311                 if (IVPR_PRIORITY(opp->src[irq].ivpr) > priority) {
 312                         next = irq;
 313                         priority = IVPR_PRIORITY(opp->src[irq].ivpr);
 314                 }
 315         }
 316
 317         q->next = next;
 318         q->priority = priority;
 319 }
 320
 321 static int IRQ_get_next(struct openpic *opp, struct irq_queue *q)
 322 {
 323         /* XXX: optimize */
 324         IRQ_check(opp, q);
 325
 326         return q->next;
 327 }
 328
 329 static void IRQ_local_pipe(struct openpic *opp, int n_CPU, int n_IRQ,
 330                            bool active, bool was_active)
 331 {
 332         struct irq_dest *dst;
 333         struct irq_source *src;
 334         int priority;
 335
 336         dst = &opp->dst[n_CPU];
 337         src = &opp->src[n_IRQ];
 338
 339         pr_debug("%s: IRQ %d active %d was %d\n",
 340                 __func__, n_IRQ, active, was_active);
 341
 342         if (src->output != ILR_INTTGT_INT) {
 343                 pr_debug("%s: output %d irq %d active %d was %d count %d\n",
 344                         __func__, src->output, n_IRQ, active, was_active,
 345                         dst->outputs_active[src->output]);
 346
 347                 /* On Freescale MPIC, critical interrupts ignore priority,
 348                  * IACK, EOI, etc.  Before MPIC v4.1 they also ignore
 349                  * masking.
 350                  */
 351                 if (active) {
 352                         if (!was_active &&
 353                             dst->outputs_active[src->output]++ == 0) {
 354                                 pr_debug("%s: Raise OpenPIC output %d cpu %d irq %d\n",
 355                                         __func__, src->output, n_CPU, n_IRQ);
 356                                 mpic_irq_raise(opp, dst, src->output);
 357                         }
 358                 } else {
 359                         if (was_active &&
 360                             --dst->outputs_active[src->output] == 0) {
 361                                 pr_debug("%s: Lower OpenPIC output %d cpu %d irq %d\n",
 362                                         __func__, src->output, n_CPU, n_IRQ);
 363                                 mpic_irq_lower(opp, dst, src->output);
 364                         }
 365                 }
 366
 367                 return;
 368         }
 369
 370         priority = IVPR_PRIORITY(src->ivpr);
 371
 372         /* Even if the interrupt doesn't have enough priority,
 373          * it is still raised, in case ctpr is lowered later.
 374          */
 375         if (active)
 376                 IRQ_setbit(&dst->raised, n_IRQ);
 377         else
 378                 IRQ_resetbit(&dst->raised, n_IRQ);
 379
 380         IRQ_check(opp, &dst->raised);
 381
 382         if (active && priority <= dst->ctpr) {
 383                 pr_debug("%s: IRQ %d priority %d too low for ctpr %d on CPU %d\n",
 384                         __func__, n_IRQ, priority, dst->ctpr, n_CPU);
 385                 active = 0;
 386         }
 387
 388         if (active) {
 389                 if (IRQ_get_next(opp, &dst->servicing) >= 0 &&
 390                     priority <= dst->servicing.priority) {
 391                         pr_debug("%s: IRQ %d is hidden by servicing IRQ %d on CPU %d\n",
 392                                 __func__, n_IRQ, dst->servicing.next, n_CPU);
 393                 } else {
 394                         pr_debug("%s: Raise OpenPIC INT output cpu %d irq %d/%d\n",
 395                                 __func__, n_CPU, n_IRQ, dst->raised.next);
 396                         mpic_irq_raise(opp, dst, ILR_INTTGT_INT);
 397                 }
 398         } else {
 399                 IRQ_get_next(opp, &dst->servicing);
 400                 if (dst->raised.priority > dst->ctpr &&
 401                     dst->raised.priority > dst->servicing.priority) {
 402                         pr_debug("%s: IRQ %d inactive, IRQ %d prio %d above %d/%d, CPU %d\n",
 403                                 __func__, n_IRQ, dst->raised.next,
 404                                 dst->raised.priority, dst->ctpr,
 405                                 dst->servicing.priority, n_CPU);
 406                         /* IRQ line stays asserted */
 407                 } else {
 408                         pr_debug("%s: IRQ %d inactive, current prio %d/%d, CPU %d\n",
 409                                 __func__, n_IRQ, dst->ctpr,
 410                                 dst->servicing.priority, n_CPU);
 411                         mpic_irq_lower(opp, dst, ILR_INTTGT_INT);
 412                 }
 413         }
 414 }
 415
 416 /* update pic state because registers for n_IRQ have changed value */
 417 static void openpic_update_irq(struct openpic *opp, int n_IRQ)
 418 {
 419         struct irq_source *src;
 420         bool active, was_active;
 421         int i;
 422
 423         src = &opp->src[n_IRQ];
 424         active = src->pending;
 425
 426         if ((src->ivpr & IVPR_MASK_MASK) && !src->nomask) {
 427                 /* Interrupt source is disabled */
 428                 pr_debug("%s: IRQ %d is disabled\n", __func__, n_IRQ);
 429                 active = false;
 430         }
 431
 432         was_active = !!(src->ivpr & IVPR_ACTIVITY_MASK);
 433
 434         /*
 435          * We don't have a similar check for already-active because
 436          * ctpr may have changed and we need to withdraw the interrupt.
 437          */
 438         if (!active && !was_active) {
 439                 pr_debug("%s: IRQ %d is already inactive\n", __func__, n_IRQ);
 440                 return;
 441         }
 442
 443         if (active)
 444                 src->ivpr |= IVPR_ACTIVITY_MASK;
 445         else
 446                 src->ivpr &= ~IVPR_ACTIVITY_MASK;
 447
 448         if (src->destmask == 0) {
 449                 /* No target */
 450                 pr_debug("%s: IRQ %d has no target\n", __func__, n_IRQ);
 451                 return;
 452         }
 453
 454         if (src->destmask == (1 << src->last_cpu)) {
 455                 /* Only one CPU is allowed to receive this IRQ */
 456                 IRQ_local_pipe(opp, src->last_cpu, n_IRQ, active, was_active);
 457         } else if (!(src->ivpr & IVPR_MODE_MASK)) {
 458                 /* Directed delivery mode */
 459                 for (i = 0; i < opp->nb_cpus; i++) {
 460                         if (src->destmask & (1 << i)) {
 461                                 IRQ_local_pipe(opp, i, n_IRQ, active,
 462                                                was_active);
 463                         }
 464                 }
 465         } else {
 466                 /* Distributed delivery mode */
 467                 for (i = src->last_cpu + 1; i != src->last_cpu; i++) {
 468                         if (i == opp->nb_cpus)
 469                                 i = 0;
 470
 471                         if (src->destmask & (1 << i)) {
 472                                 IRQ_local_pipe(opp, i, n_IRQ, active,
 473                                                was_active);
 474                                 src->last_cpu = i;
 475                                 break;
 476                         }
 477                 }
 478         }
 479 }
 480
 481 static void openpic_set_irq(void *opaque, int n_IRQ, int level)
 482 {
 483         struct openpic *opp = opaque;
 484         struct irq_source *src;
 485
 486         if (n_IRQ >= MAX_IRQ) {
 487                 WARN_ONCE(1, "%s: IRQ %d out of range\n", __func__, n_IRQ);
 488                 return;
 489         }
 490
 491         src = &opp->src[n_IRQ];
 492         pr_debug("openpic: set irq %d = %d ivpr=0x%08x\n",
 493                 n_IRQ, level, src->ivpr);
 494         if (src->level) {
 495                 /* level-sensitive irq */
 496                 src->pending = level;
 497                 openpic_update_irq(opp, n_IRQ);
 498         } else {
 499                 /* edge-sensitive irq */
 500                 if (level) {
 501                         src->pending = 1;
 502                         openpic_update_irq(opp, n_IRQ);
 503                 }
 504
 505                 if (src->output != ILR_INTTGT_INT) {
 506                         /* Edge-triggered interrupts shouldn't be used
 507                          * with non-INT delivery, but just in case,
 508                          * try to make it do something sane rather than
 509                          * cause an interrupt storm.  This is close to
 510                          * what you'd probably see happen in real hardware.
 511                          */
 512                         src->pending = 0;
 513                         openpic_update_irq(opp, n_IRQ);
 514                 }
 515         }
 516 }
 517
 518 static void openpic_reset(struct openpic *opp)
 519 {
 520         int i;
 521
 522         opp->gcr = GCR_RESET;
 523         /* Initialise controller registers */
 524         opp->frr = ((opp->nb_irqs - 1) << FRR_NIRQ_SHIFT) |
 525             (opp->vid << FRR_VID_SHIFT);
 526
 527         opp->pir = 0;
 528         opp->spve = -1 & opp->vector_mask;
 529         opp->tfrr = opp->tfrr_reset;
 530         /* Initialise IRQ sources */
 531         for (i = 0; i < opp->max_irq; i++) {
 532                 opp->src[i].ivpr = opp->ivpr_reset;
 533
 534                 switch (opp->src[i].type) {
 535                 case IRQ_TYPE_NORMAL:
 536                         opp->src[i].level =
 537                             !!(opp->ivpr_reset & IVPR_SENSE_MASK);
 538                         break;
 539
 540                 case IRQ_TYPE_FSLINT:
 541                         opp->src[i].ivpr |= IVPR_POLARITY_MASK;
 542                         break;
 543
 544                 case IRQ_TYPE_FSLSPECIAL:
 545                         break;
 546                 }
 547
 548                 write_IRQreg_idr(opp, i, opp->idr_reset);
 549         }
 550         /* Initialise IRQ destinations */
 551         for (i = 0; i < MAX_CPU; i++) {
 552                 opp->dst[i].ctpr = 15;
 553                 memset(&opp->dst[i].raised, 0, sizeof(struct irq_queue));
 554                 opp->dst[i].raised.next = -1;
 555                 memset(&opp->dst[i].servicing, 0, sizeof(struct irq_queue));
 556                 opp->dst[i].servicing.next = -1;
 557         }
 558         /* Initialise timers */
 559         for (i = 0; i < MAX_TMR; i++) {
 560                 opp->timers[i].tccr = 0;
 561                 opp->timers[i].tbcr = TBCR_CI;
 562         }
 563         /* Go out of RESET state */
 564         opp->gcr = 0;
 565 }
 566
 567 static inline uint32_t read_IRQreg_idr(struct openpic *opp, int n_IRQ)
 568 {
 569         return opp->src[n_IRQ].idr;
 570 }
 571
 572 static inline uint32_t read_IRQreg_ilr(struct openpic *opp, int n_IRQ)
 573 {
 574         if (opp->flags & OPENPIC_FLAG_ILR)
 575                 return opp->src[n_IRQ].output;
 576
 577         return 0xffffffff;
 578 }
 579
 580 static inline uint32_t read_IRQreg_ivpr(struct openpic *opp, int n_IRQ)
 581 {
 582         return opp->src[n_IRQ].ivpr;
 583 }
 584
 585 static inline void write_IRQreg_idr(struct openpic *opp, int n_IRQ,
 586                                     uint32_t val)
 587 {
 588         struct irq_source *src = &opp->src[n_IRQ];
 589         uint32_t normal_mask = (1UL << opp->nb_cpus) - 1;
 590         uint32_t crit_mask = 0;
 591         uint32_t mask = normal_mask;
 592         int crit_shift = IDR_EP_SHIFT - opp->nb_cpus;
 593         int i;
 594
 595         if (opp->flags & OPENPIC_FLAG_IDR_CRIT) {
 596                 crit_mask = mask << crit_shift;
 597                 mask |= crit_mask | IDR_EP;
 598         }
 599
 600         src->idr = val & mask;
 601         pr_debug("Set IDR %d to 0x%08x\n", n_IRQ, src->idr);
 602
 603         if (opp->flags & OPENPIC_FLAG_IDR_CRIT) {
 604                 if (src->idr & crit_mask) {
 605                         if (src->idr & normal_mask) {
 606                                 pr_debug("%s: IRQ configured for multiple output types, using critical\n",
 607                                         __func__);
 608                         }
 609
 610                         src->output = ILR_INTTGT_CINT;
 611                         src->nomask = true;
 612                         src->destmask = 0;
 613
 614                         for (i = 0; i < opp->nb_cpus; i++) {
 615                                 int n_ci = IDR_CI0_SHIFT - i;
 616
 617                                 if (src->idr & (1UL << n_ci))
 618                                         src->destmask |= 1UL << i;
 619                         }
 620                 } else {
 621                         src->output = ILR_INTTGT_INT;
 622                         src->nomask = false;
 623                         src->destmask = src->idr & normal_mask;
 624                 }
 625         } else {
 626                 src->destmask = src->idr;
 627         }
 628 }
 629
 630 static inline void write_IRQreg_ilr(struct openpic *opp, int n_IRQ,
 631                                     uint32_t val)
 632 {
 633         if (opp->flags & OPENPIC_FLAG_ILR) {
 634                 struct irq_source *src = &opp->src[n_IRQ];
 635
 636                 src->output = val & ILR_INTTGT_MASK;
 637                 pr_debug("Set ILR %d to 0x%08x, output %d\n", n_IRQ, src->idr,
 638                         src->output);
 639
 640                 /* TODO: on MPIC v4.0 only, set nomask for non-INT */
 641         }
 642 }
 643
 644 static inline void write_IRQreg_ivpr(struct openpic *opp, int n_IRQ,
 645                                      uint32_t val)
 646 {
 647         uint32_t mask;
 648
 649         /* NOTE when implementing newer FSL MPIC models: starting with v4.0,
 650          * the polarity bit is read-only on internal interrupts.
 651          */
 652         mask = IVPR_MASK_MASK | IVPR_PRIORITY_MASK | IVPR_SENSE_MASK |
 653             IVPR_POLARITY_MASK | opp->vector_mask;
 654
 655         /* ACTIVITY bit is read-only */
 656         opp->src[n_IRQ].ivpr =
 657             (opp->src[n_IRQ].ivpr & IVPR_ACTIVITY_MASK) | (val & mask);
 658
 659         /* For FSL internal interrupts, The sense bit is reserved and zero,
 660          * and the interrupt is always level-triggered.  Timers and IPIs
 661          * have no sense or polarity bits, and are edge-triggered.
 662          */
 663         switch (opp->src[n_IRQ].type) {
 664         case IRQ_TYPE_NORMAL:
 665                 opp->src[n_IRQ].level =
 666                     !!(opp->src[n_IRQ].ivpr & IVPR_SENSE_MASK);
 667                 break;
 668
 669         case IRQ_TYPE_FSLINT:
 670                 opp->src[n_IRQ].ivpr &= ~IVPR_SENSE_MASK;
 671                 break;
 672
 673         case IRQ_TYPE_FSLSPECIAL:
 674                 opp->src[n_IRQ].ivpr &= ~(IVPR_POLARITY_MASK | IVPR_SENSE_MASK);
 675                 break;
 676         }
 677
 678         openpic_update_irq(opp, n_IRQ);
 679         pr_debug("Set IVPR %d to 0x%08x -> 0x%08x\n", n_IRQ, val,
 680                 opp->src[n_IRQ].ivpr);
 681 }
 682
 683 static void openpic_gcr_write(struct openpic *opp, uint64_t val)
 684 {
 685         if (val & GCR_RESET) {
 686                 openpic_reset(opp);
 687                 return;
 688         }
 689
 690         opp->gcr &= ~opp->mpic_mode_mask;
 691         opp->gcr |= val & opp->mpic_mode_mask;
 692 }
 693
 694 static int openpic_gbl_write(void *opaque, gpa_t addr, u32 val)
 695 {
 696         struct openpic *opp = opaque;
 697         int err = 0;
 698
 699         pr_debug("%s: addr %#llx <= %08x\n", __func__, addr, val);
 700         if (addr & 0xF)
 701                 return 0;
 702
 703         switch (addr) {
 704         case 0x00:      /* Block Revision Register1 (BRR1) is Readonly */
 705                 break;
 706         case 0x40:
 707         case 0x50:
 708         case 0x60:
 709         case 0x70:
 710         case 0x80:
 711         case 0x90:
 712         case 0xA0:
 713         case 0xB0:
 714                 err = openpic_cpu_write_internal(opp, addr, val,
 715                                                  get_current_cpu());
 716                 break;
 717         case 0x1000:            /* FRR */
 718                 break;
 719         case 0x1020:            /* GCR */
 720                 openpic_gcr_write(opp, val);
 721                 break;
 722         case 0x1080:            /* VIR */
 723                 break;
 724         case 0x1090:            /* PIR */
 725                 /*
 726                  * This register is used to reset a CPU core --
 727                  * let userspace handle it.
 728                  */
 729                 err = -ENXIO;
 730                 break;
 731         case 0x10A0:            /* IPI_IVPR */
 732         case 0x10B0:
 733         case 0x10C0:
 734         case 0x10D0: {
 735                 int idx;
 736                 idx = (addr - 0x10A0) >> 4;
 737                 write_IRQreg_ivpr(opp, opp->irq_ipi0 + idx, val);
 738                 break;
 739         }
 740         case 0x10E0:            /* SPVE */
 741                 opp->spve = val & opp->vector_mask;
 742                 break;
 743         default:
 744                 break;
 745         }
 746
 747         return err;
 748 }
 749
 750 static int openpic_gbl_read(void *opaque, gpa_t addr, u32 *ptr)
 751 {
 752         struct openpic *opp = opaque;
 753         u32 retval;
 754         int err = 0;
 755
 756         pr_debug("%s: addr %#llx\n", __func__, addr);
 757         retval = 0xFFFFFFFF;
 758         if (addr & 0xF)
 759                 goto out;
 760
 761         switch (addr) {
 762         case 0x1000:            /* FRR */
 763                 retval = opp->frr;
 764                 retval |= (opp->nb_cpus - 1) << FRR_NCPU_SHIFT;
 765                 break;
 766         case 0x1020:            /* GCR */
 767                 retval = opp->gcr;
 768                 break;
 769         case 0x1080:            /* VIR */
 770                 retval = opp->vir;
 771                 break;
 772         case 0x1090:            /* PIR */
 773                 retval = 0x00000000;
 774                 break;
 775         case 0x00:              /* Block Revision Register1 (BRR1) */
 776                 retval = opp->brr1;
 777                 break;
 778         case 0x40:
 779         case 0x50:
 780         case 0x60:
 781         case 0x70:
 782         case 0x80:
 783         case 0x90:
 784         case 0xA0:
 785         case 0xB0:
 786                 err = openpic_cpu_read_internal(opp, addr,
 787                         &retval, get_current_cpu());
 788                 break;
 789         case 0x10A0:            /* IPI_IVPR */
 790         case 0x10B0:
 791         case 0x10C0:
 792         case 0x10D0:
 793                 {
 794                         int idx;
 795                         idx = (addr - 0x10A0) >> 4;
 796                         retval = read_IRQreg_ivpr(opp, opp->irq_ipi0 + idx);
 797                 }
 798                 break;
 799         case 0x10E0:            /* SPVE */
 800                 retval = opp->spve;
 801                 break;
 802         default:
 803                 break;
 804         }
 805
 806 out:
 807         pr_debug("%s: => 0x%08x\n", __func__, retval);
 808         *ptr = retval;
 809         return err;
 810 }
 811
 812 static int openpic_tmr_write(void *opaque, gpa_t addr, u32 val)
 813 {
 814         struct openpic *opp = opaque;
 815         int idx;
 816
 817         addr += 0x10f0;
 818
 819         pr_debug("%s: addr %#llx <= %08x\n", __func__, addr, val);
 820         if (addr & 0xF)
 821                 return 0;
 822
 823         if (addr == 0x10f0) {
 824                 /* TFRR */
 825                 opp->tfrr = val;
 826                 return 0;
 827         }
 828
 829         idx = (addr >> 6) & 0x3;
 830         addr = addr & 0x30;
 831
 832         switch (addr & 0x30) {
 833         case 0x00:              /* TCCR */
 834                 break;
 835         case 0x10:              /* TBCR */
 836                 if ((opp->timers[idx].tccr & TCCR_TOG) != 0 &&
 837                     (val & TBCR_CI) == 0 &&
 838                     (opp->timers[idx].tbcr & TBCR_CI) != 0)
 839                         opp->timers[idx].tccr &= ~TCCR_TOG;
 840
 841                 opp->timers[idx].tbcr = val;
 842                 break;
 843         case 0x20:              /* TVPR */
 844                 write_IRQreg_ivpr(opp, opp->irq_tim0 + idx, val);
 845                 break;
 846         case 0x30:              /* TDR */
 847                 write_IRQreg_idr(opp, opp->irq_tim0 + idx, val);
 848                 break;
 849         }
 850
 851         return 0;
 852 }
 853
 854 static int openpic_tmr_read(void *opaque, gpa_t addr, u32 *ptr)
 855 {
 856         struct openpic *opp = opaque;
 857         uint32_t retval = -1;
 858         int idx;
 859
 860         pr_debug("%s: addr %#llx\n", __func__, addr);
 861         if (addr & 0xF)
 862                 goto out;
 863
 864         idx = (addr >> 6) & 0x3;
 865         if (addr == 0x0) {
 866                 /* TFRR */
 867                 retval = opp->tfrr;
 868                 goto out;
 869         }
 870
 871         switch (addr & 0x30) {
 872         case 0x00:              /* TCCR */
 873                 retval = opp->timers[idx].tccr;
 874                 break;
 875         case 0x10:              /* TBCR */
 876                 retval = opp->timers[idx].tbcr;
 877                 break;
 878         case 0x20:              /* TIPV */
 879                 retval = read_IRQreg_ivpr(opp, opp->irq_tim0 + idx);
 880                 break;
 881         case 0x30:              /* TIDE (TIDR) */
 882                 retval = read_IRQreg_idr(opp, opp->irq_tim0 + idx);
 883                 break;
 884         }
 885
 886 out:
 887         pr_debug("%s: => 0x%08x\n", __func__, retval);
 888         *ptr = retval;
 889         return 0;
 890 }
 891
 892 static int openpic_src_write(void *opaque, gpa_t addr, u32 val)
 893 {
 894         struct openpic *opp = opaque;
 895         int idx;
 896
 897         pr_debug("%s: addr %#llx <= %08x\n", __func__, addr, val);
 898
 899         addr = addr & 0xffff;
 900         idx = addr >> 5;
 901
 902         switch (addr & 0x1f) {
 903         case 0x00:
 904                 write_IRQreg_ivpr(opp, idx, val);
 905                 break;
 906         case 0x10:
 907                 write_IRQreg_idr(opp, idx, val);
 908                 break;
 909         case 0x18:
 910                 write_IRQreg_ilr(opp, idx, val);
 911                 break;
 912         }
 913
 914         return 0;
 915 }
 916
 917 static int openpic_src_read(void *opaque, gpa_t addr, u32 *ptr)
 918 {
 919         struct openpic *opp = opaque;
 920         uint32_t retval;
 921         int idx;
 922
 923         pr_debug("%s: addr %#llx\n", __func__, addr);
 924         retval = 0xFFFFFFFF;
 925
 926         addr = addr & 0xffff;
 927         idx = addr >> 5;
 928
 929         switch (addr & 0x1f) {
 930         case 0x00:
 931                 retval = read_IRQreg_ivpr(opp, idx);
 932                 break;
 933         case 0x10:
 934                 retval = read_IRQreg_idr(opp, idx);
 935                 break;
 936         case 0x18:
 937                 retval = read_IRQreg_ilr(opp, idx);
 938                 break;
 939         }
 940
 941         pr_debug("%s: => 0x%08x\n", __func__, retval);
 942         *ptr = retval;
 943         return 0;
 944 }
 945
 946 static int openpic_msi_write(void *opaque, gpa_t addr, u32 val)
 947 {
 948         struct openpic *opp = opaque;
 949         int idx = opp->irq_msi;
 950         int srs, ibs;
 951
 952         pr_debug("%s: addr %#llx <= 0x%08x\n", __func__, addr, val);
 953         if (addr & 0xF)
 954                 return 0;
 955
 956         switch (addr) {
 957         case MSIIR_OFFSET:
 958                 srs = val >> MSIIR_SRS_SHIFT;
 959                 idx += srs;
 960                 ibs = (val & MSIIR_IBS_MASK) >> MSIIR_IBS_SHIFT;
 961                 opp->msi[srs].msir |= 1 << ibs;
 962                 openpic_set_irq(opp, idx, 1);
 963                 break;
 964         default:
 965                 /* most registers are read-only, thus ignored */
 966                 break;
 967         }
 968
 969         return 0;
 970 }
 971
 972 static int openpic_msi_read(void *opaque, gpa_t addr, u32 *ptr)
 973 {
 974         struct openpic *opp = opaque;
 975         uint32_t r = 0;
 976         int i, srs;
 977
 978         pr_debug("%s: addr %#llx\n", __func__, addr);
 979         if (addr & 0xF)
 980                 return -ENXIO;
 981
 982         srs = addr >> 4;
 983
 984         switch (addr) {
 985         case 0x00:
 986         case 0x10:
 987         case 0x20:
 988         case 0x30:
 989         case 0x40:
 990         case 0x50:
 991         case 0x60:
 992         case 0x70:              /* MSIRs */
 993                 r = opp->msi[srs].msir;
 994                 /* Clear on read */
 995                 opp->msi[srs].msir = 0;
 996                 openpic_set_irq(opp, opp->irq_msi + srs, 0);
 997                 break;
 998         case 0x120:             /* MSISR */
 999                 for (i = 0; i < MAX_MSI; i++)
1000                         r |= (opp->msi[i].msir ? 1 : 0) << i;
1001                 break;
1002         }
1003
1004         pr_debug("%s: => 0x%08x\n", __func__, r);
1005         *ptr = r;
1006         return 0;
1007 }
1008
1009 static int openpic_summary_read(void *opaque, gpa_t addr, u32 *ptr)
1010 {
1011         uint32_t r = 0;
1012
1013         pr_debug("%s: addr %#llx\n", __func__, addr);
1014
1015         /* TODO: EISR/EIMR */
1016
1017         *ptr = r;
1018         return 0;
1019 }
1020
1021 static int openpic_summary_write(void *opaque, gpa_t addr, u32 val)
1022 {
1023         pr_debug("%s: addr %#llx <= 0x%08x\n", __func__, addr, val);
1024
1025         /* TODO: EISR/EIMR */
1026         return 0;
1027 }
1028
1029 static int openpic_cpu_write_internal(void *opaque, gpa_t addr,
1030                                       u32 val, int idx)
1031 {
1032         struct openpic *opp = opaque;
1033         struct irq_source *src;
1034         struct irq_dest *dst;
1035         int s_IRQ, n_IRQ;
1036
1037         pr_debug("%s: cpu %d addr %#llx <= 0x%08x\n", __func__, idx,
1038                 addr, val);
1039
1040         if (idx < 0)
1041                 return 0;
1042
1043         if (addr & 0xF)
1044                 return 0;
1045
1046         dst = &opp->dst[idx];
1047         addr &= 0xFF0;
1048         switch (addr) {
1049         case 0x40:              /* IPIDR */
1050         case 0x50:
1051         case 0x60:
1052         case 0x70:
1053                 idx = (addr - 0x40) >> 4;
1054                 /* we use IDE as mask which CPUs to deliver the IPI to still. */
1055                 opp->src[opp->irq_ipi0 + idx].destmask |= val;
1056                 openpic_set_irq(opp, opp->irq_ipi0 + idx, 1);
1057                 openpic_set_irq(opp, opp->irq_ipi0 + idx, 0);
1058                 break;
1059         case 0x80:              /* CTPR */
1060                 dst->ctpr = val & 0x0000000F;
1061
1062                 pr_debug("%s: set CPU %d ctpr to %d, raised %d servicing %d\n",
1063                         __func__, idx, dst->ctpr, dst->raised.priority,
1064                         dst->servicing.priority);
1065
1066                 if (dst->raised.priority <= dst->ctpr) {
1067                         pr_debug("%s: Lower OpenPIC INT output cpu %d due to ctpr\n",
1068                                 __func__, idx);
1069                         mpic_irq_lower(opp, dst, ILR_INTTGT_INT);
1070                 } else if (dst->raised.priority > dst->servicing.priority) {
1071                         pr_debug("%s: Raise OpenPIC INT output cpu %d irq %d\n",
1072                                 __func__, idx, dst->raised.next);
1073                         mpic_irq_raise(opp, dst, ILR_INTTGT_INT);
1074                 }
1075
1076                 break;
1077         case 0x90:              /* WHOAMI */
1078                 /* Read-only register */
1079                 break;
1080         case 0xA0:              /* IACK */
1081                 /* Read-only register */
1082                 break;
1083         case 0xB0: {            /* EOI */
1084                 int notify_eoi;
1085
1086                 pr_debug("EOI\n");
1087                 s_IRQ = IRQ_get_next(opp, &dst->servicing);
1088
1089                 if (s_IRQ < 0) {
1090                         pr_debug("%s: EOI with no interrupt in service\n",
1091                                 __func__);
1092                         break;
1093                 }
1094
1095                 IRQ_resetbit(&dst->servicing, s_IRQ);
1096                 /* Notify listeners that the IRQ is over */
1097                 notify_eoi = s_IRQ;
1098                 /* Set up next servicing IRQ */
1099                 s_IRQ = IRQ_get_next(opp, &dst->servicing);
1100                 /* Check queued interrupts. */
1101                 n_IRQ = IRQ_get_next(opp, &dst->raised);
1102                 src = &opp->src[n_IRQ];
1103                 if (n_IRQ != -1 &&
1104                     (s_IRQ == -1 ||
1105                      IVPR_PRIORITY(src->ivpr) > dst->servicing.priority)) {
1106                         pr_debug("Raise OpenPIC INT output cpu %d irq %d\n",
1107                                 idx, n_IRQ);
1108                         mpic_irq_raise(opp, dst, ILR_INTTGT_INT);
1109                 }
1110
1111                 spin_unlock(&opp->lock);
1112                 kvm_notify_acked_irq(opp->kvm, 0, notify_eoi);
1113                 spin_lock(&opp->lock);
1114
1115                 break;
1116         }
1117         default:
1118                 break;
1119         }
1120
1121         return 0;
1122 }
1123
1124 static int openpic_cpu_write(void *opaque, gpa_t addr, u32 val)
1125 {
1126         struct openpic *opp = opaque;
1127
1128         return openpic_cpu_write_internal(opp, addr, val,
1129                                          (addr & 0x1f000) >> 12);
1130 }
1131
1132 static uint32_t openpic_iack(struct openpic *opp, struct irq_dest *dst,
1133                              int cpu)
1134 {
1135         struct irq_source *src;
1136         int retval, irq;
1137
1138         pr_debug("Lower OpenPIC INT output\n");
1139         mpic_irq_lower(opp, dst, ILR_INTTGT_INT);
1140
1141         irq = IRQ_get_next(opp, &dst->raised);
1142         pr_debug("IACK: irq=%d\n", irq);
1143
1144         if (irq == -1)
1145                 /* No more interrupt pending */
1146                 return opp->spve;
1147
1148         src = &opp->src[irq];
1149         if (!(src->ivpr & IVPR_ACTIVITY_MASK) ||
1150             !(IVPR_PRIORITY(src->ivpr) > dst->ctpr)) {
1151                 pr_err("%s: bad raised IRQ %d ctpr %d ivpr 0x%08x\n",
1152                         __func__, irq, dst->ctpr, src->ivpr);
1153                 openpic_update_irq(opp, irq);
1154                 retval = opp->spve;
1155         } else {
1156                 /* IRQ enter servicing state */
1157                 IRQ_setbit(&dst->servicing, irq);
1158                 retval = IVPR_VECTOR(opp, src->ivpr);
1159         }
1160
1161         if (!src->level) {
1162                 /* edge-sensitive IRQ */
1163                 src->ivpr &= ~IVPR_ACTIVITY_MASK;
1164                 src->pending = 0;
1165                 IRQ_resetbit(&dst->raised, irq);
1166         }
1167
1168         if ((irq >= opp->irq_ipi0) && (irq < (opp->irq_ipi0 + MAX_IPI))) {
1169                 src->destmask &= ~(1 << cpu);
1170                 if (src->destmask && !src->level) {
1171                         /* trigger on CPUs that didn't know about it yet */
1172                         openpic_set_irq(opp, irq, 1);
1173                         openpic_set_irq(opp, irq, 0);
1174                         /* if all CPUs knew about it, set active bit again */
1175                         src->ivpr |= IVPR_ACTIVITY_MASK;
1176                 }
1177         }
1178
1179         return retval;
1180 }
1181
1182 void kvmppc_mpic_set_epr(struct kvm_vcpu *vcpu)
1183 {
1184         struct openpic *opp = vcpu->arch.mpic;
1185         int cpu = vcpu->arch.irq_cpu_id;
1186         unsigned long flags;
1187
1188         spin_lock_irqsave(&opp->lock, flags);
1189
1190         if ((opp->gcr & opp->mpic_mode_mask) == GCR_MODE_PROXY)
1191                 kvmppc_set_epr(vcpu, openpic_iack(opp, &opp->dst[cpu], cpu));
1192
1193         spin_unlock_irqrestore(&opp->lock, flags);
1194 }
1195
1196 static int openpic_cpu_read_internal(void *opaque, gpa_t addr,
1197                                      u32 *ptr, int idx)
1198 {
1199         struct openpic *opp = opaque;
1200         struct irq_dest *dst;
1201         uint32_t retval;
1202
1203         pr_debug("%s: cpu %d addr %#llx\n", __func__, idx, addr);
1204         retval = 0xFFFFFFFF;
1205
1206         if (idx < 0)
1207                 goto out;
1208
1209         if (addr & 0xF)
1210                 goto out;
1211
1212         dst = &opp->dst[idx];
1213         addr &= 0xFF0;
1214         switch (addr) {
1215         case 0x80:              /* CTPR */
1216                 retval = dst->ctpr;
1217                 break;
1218         case 0x90:              /* WHOAMI */
1219                 retval = idx;
1220                 break;
1221         case 0xA0:              /* IACK */
1222                 retval = openpic_iack(opp, dst, idx);
1223                 break;
1224         case 0xB0:              /* EOI */
1225                 retval = 0;
1226                 break;
1227         default:
1228                 break;
1229         }
1230         pr_debug("%s: => 0x%08x\n", __func__, retval);
1231
1232 out:
1233         *ptr = retval;
1234         return 0;
1235 }
1236
1237 static int openpic_cpu_read(void *opaque, gpa_t addr, u32 *ptr)
1238 {
1239         struct openpic *opp = opaque;
1240
1241         return openpic_cpu_read_internal(opp, addr, ptr,
1242                                          (addr & 0x1f000) >> 12);
1243 }
1244
1245 struct mem_reg {
1246         int (*read)(void *opaque, gpa_t addr, u32 *ptr);
1247         int (*write)(void *opaque, gpa_t addr, u32 val);
1248         gpa_t start_addr;
1249         int size;
1250 };
1251
1252 static const struct mem_reg openpic_gbl_mmio = {
1253         .write = openpic_gbl_write,
1254         .read = openpic_gbl_read,
1255         .start_addr = OPENPIC_GLB_REG_START,
1256         .size = OPENPIC_GLB_REG_SIZE,
1257 };
1258
1259 static const struct mem_reg openpic_tmr_mmio = {
1260         .write = openpic_tmr_write,
1261         .read = openpic_tmr_read,
1262         .start_addr = OPENPIC_TMR_REG_START,
1263         .size = OPENPIC_TMR_REG_SIZE,
1264 };
1265
1266 static const struct mem_reg openpic_cpu_mmio = {
1267         .write = openpic_cpu_write,
1268         .read = openpic_cpu_read,
1269         .start_addr = OPENPIC_CPU_REG_START,
1270         .size = OPENPIC_CPU_REG_SIZE,
1271 };
1272
1273 static const struct mem_reg openpic_src_mmio = {
1274         .write = openpic_src_write,
1275         .read = openpic_src_read,
1276         .start_addr = OPENPIC_SRC_REG_START,
1277         .size = OPENPIC_SRC_REG_SIZE,
1278 };
1279
1280 static const struct mem_reg openpic_msi_mmio = {
1281         .read = openpic_msi_read,
1282         .write = openpic_msi_write,
1283         .start_addr = OPENPIC_MSI_REG_START,
1284         .size = OPENPIC_MSI_REG_SIZE,
1285 };
1286
1287 static const struct mem_reg openpic_summary_mmio = {
1288         .read = openpic_summary_read,
1289         .write = openpic_summary_write,
1290         .start_addr = OPENPIC_SUMMARY_REG_START,
1291         .size = OPENPIC_SUMMARY_REG_SIZE,
1292 };
1293
1294 static void add_mmio_region(struct openpic *opp, const struct mem_reg *mr)
1295 {
1296         if (opp->num_mmio_regions >= MAX_MMIO_REGIONS) {
1297                 WARN(1, "kvm mpic: too many mmio regions\n");
1298                 return;
1299         }
1300
1301         opp->mmio_regions[opp->num_mmio_regions++] = mr;
1302 }
1303
1304 static void fsl_common_init(struct openpic *opp)
1305 {
1306         int i;
1307         int virq = MAX_SRC;
1308
1309         add_mmio_region(opp, &openpic_msi_mmio);
1310         add_mmio_region(opp, &openpic_summary_mmio);
1311
1312         opp->vid = VID_REVISION_1_2;
1313         opp->vir = VIR_GENERIC;
1314         opp->vector_mask = 0xFFFF;
1315         opp->tfrr_reset = 0;
1316         opp->ivpr_reset = IVPR_MASK_MASK;
1317         opp->idr_reset = 1 << 0;
1318         opp->max_irq = MAX_IRQ;
1319
1320         opp->irq_ipi0 = virq;
1321         virq += MAX_IPI;
1322         opp->irq_tim0 = virq;
1323         virq += MAX_TMR;
1324
1325         BUG_ON(virq > MAX_IRQ);
1326
1327         opp->irq_msi = 224;
1328
1329         for (i = 0; i < opp->fsl->max_ext; i++)
1330                 opp->src[i].level = false;
1331
1332         /* Internal interrupts, including message and MSI */
1333         for (i = 16; i < MAX_SRC; i++) {
1334                 opp->src[i].type = IRQ_TYPE_FSLINT;
1335                 opp->src[i].level = true;
1336         }
1337
1338         /* timers and IPIs */
1339         for (i = MAX_SRC; i < virq; i++) {
1340                 opp->src[i].type = IRQ_TYPE_FSLSPECIAL;
1341                 opp->src[i].level = false;
1342         }
1343 }
1344
1345 static int kvm_mpic_read_internal(struct openpic *opp, gpa_t addr, u32 *ptr)
1346 {
1347         int i;
1348
1349         for (i = 0; i < opp->num_mmio_regions; i++) {
1350                 const struct mem_reg *mr = opp->mmio_regions[i];
1351
1352                 if (mr->start_addr > addr || addr >= mr->start_addr + mr->size)
1353                         continue;
1354
1355                 return mr->read(opp, addr - mr->start_addr, ptr);
1356         }
1357
1358         return -ENXIO;
1359 }
1360
1361 static int kvm_mpic_write_internal(struct openpic *opp, gpa_t addr, u32 val)
1362 {
1363         int i;
1364
1365         for (i = 0; i < opp->num_mmio_regions; i++) {
1366                 const struct mem_reg *mr = opp->mmio_regions[i];
1367
1368                 if (mr->start_addr > addr || addr >= mr->start_addr + mr->size)
1369                         continue;
1370
1371                 return mr->write(opp, addr - mr->start_addr, val);
1372         }
1373
1374         return -ENXIO;
1375 }
1376
1377 static int kvm_mpic_read(struct kvm_io_device *this, gpa_t addr,
1378                          int len, void *ptr)
1379 {
1380         struct openpic *opp = container_of(this, struct openpic, mmio);
1381         int ret;
1382         union {
1383                 u32 val;
1384                 u8 bytes[4];
1385         } u;
1386
1387         if (addr & (len - 1)) {
1388                 pr_debug("%s: bad alignment %llx/%d\n",
1389                          __func__, addr, len);
1390                 return -EINVAL;
1391         }
1392
1393         spin_lock_irq(&opp->lock);
1394         ret = kvm_mpic_read_internal(opp, addr - opp->reg_base, &u.val);
1395         spin_unlock_irq(&opp->lock);
1396
1397         /*
1398          * Technically only 32-bit accesses are allowed, but be nice to
1399          * people dumping registers a byte at a time -- it works in real
1400          * hardware (reads only, not writes).
1401          */
1402         if (len == 4) {
1403                 *(u32 *)ptr = u.val;
1404                 pr_debug("%s: addr %llx ret %d len 4 val %x\n",
1405                          __func__, addr, ret, u.val);
1406         } else if (len == 1) {
1407                 *(u8 *)ptr = u.bytes[addr & 3];
1408                 pr_debug("%s: addr %llx ret %d len 1 val %x\n",
1409                          __func__, addr, ret, u.bytes[addr & 3]);
1410         } else {
1411                 pr_debug("%s: bad length %d\n", __func__, len);
1412                 return -EINVAL;
1413         }
1414
1415         return ret;
1416 }
1417
1418 static int kvm_mpic_write(struct kvm_io_device *this, gpa_t addr,
1419                           int len, const void *ptr)
1420 {
1421         struct openpic *opp = container_of(this, struct openpic, mmio);
1422         int ret;
1423
1424         if (len != 4) {
1425                 pr_debug("%s: bad length %d\n", __func__, len);
1426                 return -EOPNOTSUPP;
1427         }
1428         if (addr & 3) {
1429                 pr_debug("%s: bad alignment %llx/%d\n", __func__, addr, len);
1430                 return -EOPNOTSUPP;
1431         }
1432
1433         spin_lock_irq(&opp->lock);
1434         ret = kvm_mpic_write_internal(opp, addr - opp->reg_base,
1435                                       *(const u32 *)ptr);
1436         spin_unlock_irq(&opp->lock);
1437
1438         pr_debug("%s: addr %llx ret %d val %x\n",
1439                  __func__, addr, ret, *(const u32 *)ptr);
1440
1441         return ret;
1442 }
1443
1444 static const struct kvm_io_device_ops mpic_mmio_ops = {
1445         .read = kvm_mpic_read,
1446         .write = kvm_mpic_write,
1447 };
1448
1449 static void map_mmio(struct openpic *opp)
1450 {
1451         kvm_iodevice_init(&opp->mmio, &mpic_mmio_ops);
1452
1453         kvm_io_bus_register_dev(opp->kvm, KVM_MMIO_BUS,
1454                                 opp->reg_base, OPENPIC_REG_SIZE,
1455                                 &opp->mmio);
1456 }
1457
1458 static void unmap_mmio(struct openpic *opp)
1459 {
1460         kvm_io_bus_unregister_dev(opp->kvm, KVM_MMIO_BUS, &opp->mmio);
1461 }
1462
1463 static int set_base_addr(struct openpic *opp, struct kvm_device_attr *attr)
1464 {
1465         u64 base;
1466
1467         if (copy_from_user(&base, (u64 __user *)(long)attr->addr, sizeof(u64)))
1468                 return -EFAULT;
1469
1470         if (base & 0x3ffff) {
1471                 pr_debug("kvm mpic %s: KVM_DEV_MPIC_BASE_ADDR %08llx not aligned\n",
1472                          __func__, base);
1473                 return -EINVAL;
1474         }
1475
1476         if (base == opp->reg_base)
1477                 return 0;
1478
1479         mutex_lock(&opp->kvm->slots_lock);
1480
1481         unmap_mmio(opp);
1482         opp->reg_base = base;
1483
1484         pr_debug("kvm mpic %s: KVM_DEV_MPIC_BASE_ADDR %08llx\n",
1485                  __func__, base);
1486
1487         if (base == 0)
1488                 goto out;
1489
1490         map_mmio(opp);
1491
1492 out:
1493         mutex_unlock(&opp->kvm->slots_lock);
1494         return 0;
1495 }
1496
1497 #define ATTR_SET                0
1498 #define ATTR_GET                1
1499
1500 static int access_reg(struct openpic *opp, gpa_t addr, u32 *val, int type)
1501 {
1502         int ret;
1503
1504         if (addr & 3)
1505                 return -ENXIO;
1506
1507         spin_lock_irq(&opp->lock);
1508
1509         if (type == ATTR_SET)
1510                 ret = kvm_mpic_write_internal(opp, addr, *val);
1511         else
1512                 ret = kvm_mpic_read_internal(opp, addr, val);
1513
1514         spin_unlock_irq(&opp->lock);
1515
1516         pr_debug("%s: type %d addr %llx val %x\n", __func__, type, addr, *val);
1517
1518         return ret;
1519 }
1520
1521 static int mpic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1522 {
1523         struct openpic *opp = dev->private;
1524         u32 attr32;
1525
1526         switch (attr->group) {
1527         case KVM_DEV_MPIC_GRP_MISC:
1528                 switch (attr->attr) {
1529                 case KVM_DEV_MPIC_BASE_ADDR:
1530                         return set_base_addr(opp, attr);
1531                 }
1532
1533                 break;
1534
1535         case KVM_DEV_MPIC_GRP_REGISTER:
1536                 if (get_user(attr32, (u32 __user *)(long)attr->addr))
1537                         return -EFAULT;
1538
1539                 return access_reg(opp, attr->attr, &attr32, ATTR_SET);
1540
1541         case KVM_DEV_MPIC_GRP_IRQ_ACTIVE:
1542                 if (attr->attr > MAX_SRC)
1543                         return -EINVAL;
1544
1545                 if (get_user(attr32, (u32 __user *)(long)attr->addr))
1546                         return -EFAULT;
1547
1548                 if (attr32 != 0 && attr32 != 1)
1549                         return -EINVAL;
1550
1551                 spin_lock_irq(&opp->lock);
1552                 openpic_set_irq(opp, attr->attr, attr32);
1553                 spin_unlock_irq(&opp->lock);
1554                 return 0;
1555         }
1556
1557         return -ENXIO;
1558 }
1559
1560 static int mpic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1561 {
1562         struct openpic *opp = dev->private;
1563         u64 attr64;
1564         u32 attr32;
1565         int ret;
1566
1567         switch (attr->group) {
1568         case KVM_DEV_MPIC_GRP_MISC:
1569                 switch (attr->attr) {
1570                 case KVM_DEV_MPIC_BASE_ADDR:
1571                         mutex_lock(&opp->kvm->slots_lock);
1572                         attr64 = opp->reg_base;
1573                         mutex_unlock(&opp->kvm->slots_lock);
1574
1575                         if (copy_to_user((u64 __user *)(long)attr->addr,
1576                                          &attr64, sizeof(u64)))
1577                                 return -EFAULT;
1578
1579                         return 0;
1580                 }
1581
1582                 break;
1583
1584         case KVM_DEV_MPIC_GRP_REGISTER:
1585                 ret = access_reg(opp, attr->attr, &attr32, ATTR_GET);
1586                 if (ret)
1587                         return ret;
1588
1589                 if (put_user(attr32, (u32 __user *)(long)attr->addr))
1590                         return -EFAULT;
1591
1592                 return 0;
1593
1594         case KVM_DEV_MPIC_GRP_IRQ_ACTIVE:
1595                 if (attr->attr > MAX_SRC)
1596                         return -EINVAL;
1597
1598                 spin_lock_irq(&opp->lock);
1599                 attr32 = opp->src[attr->attr].pending;
1600                 spin_unlock_irq(&opp->lock);
1601
1602                 if (put_user(attr32, (u32 __user *)(long)attr->addr))
1603                         return -EFAULT;
1604
1605                 return 0;
1606         }
1607
1608         return -ENXIO;
1609 }
1610
1611 static int mpic_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1612 {
1613         switch (attr->group) {
1614         case KVM_DEV_MPIC_GRP_MISC:
1615                 switch (attr->attr) {
1616                 case KVM_DEV_MPIC_BASE_ADDR:
1617                         return 0;
1618                 }
1619
1620                 break;
1621
1622         case KVM_DEV_MPIC_GRP_REGISTER:
1623                 return 0;
1624
1625         case KVM_DEV_MPIC_GRP_IRQ_ACTIVE:
1626                 if (attr->attr > MAX_SRC)
1627                         break;
1628
1629                 return 0;
1630         }
1631
1632         return -ENXIO;
1633 }
1634
1635 static void mpic_destroy(struct kvm_device *dev)
1636 {
1637         struct openpic *opp = dev->private;
1638
1639         dev->kvm->arch.mpic = NULL;
1640         kfree(opp);
1641         kfree(dev);
1642 }
1643
1644 static int mpic_set_default_irq_routing(struct openpic *opp)
1645 {
1646         struct kvm_irq_routing_entry *routing;
1647
1648         /* Create a nop default map, so that dereferencing it still works */
1649         routing = kzalloc((sizeof(*routing)), GFP_KERNEL);
1650         if (!routing)
1651                 return -ENOMEM;
1652
1653         kvm_set_irq_routing(opp->kvm, routing, 0, 0);
1654
1655         kfree(routing);
1656         return 0;
1657 }
1658
1659 static int mpic_create(struct kvm_device *dev, u32 type)
1660 {
1661         struct openpic *opp;
1662         int ret;
1663
1664         /* We only support one MPIC at a time for now */
1665         if (dev->kvm->arch.mpic)
1666                 return -EINVAL;
1667
1668         opp = kzalloc(sizeof(struct openpic), GFP_KERNEL);
1669         if (!opp)
1670                 return -ENOMEM;
1671
1672         dev->private = opp;
1673         opp->kvm = dev->kvm;
1674         opp->dev = dev;
1675         opp->model = type;
1676         spin_lock_init(&opp->lock);
1677
1678         add_mmio_region(opp, &openpic_gbl_mmio);
1679         add_mmio_region(opp, &openpic_tmr_mmio);
1680         add_mmio_region(opp, &openpic_src_mmio);
1681         add_mmio_region(opp, &openpic_cpu_mmio);
1682
1683         switch (opp->model) {
1684         case KVM_DEV_TYPE_FSL_MPIC_20:
1685                 opp->fsl = &fsl_mpic_20;
1686                 opp->brr1 = 0x00400200;
1687                 opp->flags |= OPENPIC_FLAG_IDR_CRIT;
1688                 opp->nb_irqs = 80;
1689                 opp->mpic_mode_mask = GCR_MODE_MIXED;
1690
1691                 fsl_common_init(opp);
1692
1693                 break;
1694
1695         case KVM_DEV_TYPE_FSL_MPIC_42:
1696                 opp->fsl = &fsl_mpic_42;
1697                 opp->brr1 = 0x00400402;
1698                 opp->flags |= OPENPIC_FLAG_ILR;
1699                 opp->nb_irqs = 196;
1700                 opp->mpic_mode_mask = GCR_MODE_PROXY;
1701
1702                 fsl_common_init(opp);
1703
1704                 break;
1705
1706         default:
1707                 ret = -ENODEV;
1708                 goto err;
1709         }
1710
1711         ret = mpic_set_default_irq_routing(opp);
1712         if (ret)
1713                 goto err;
1714
1715         openpic_reset(opp);
1716
1717         smp_wmb();
1718         dev->kvm->arch.mpic = opp;
1719
1720         return 0;
1721
1722 err:
1723         kfree(opp);
1724         return ret;
1725 }
1726
1727 struct kvm_device_ops kvm_mpic_ops = {
1728         .name = "kvm-mpic",
1729         .create = mpic_create,
1730         .destroy = mpic_destroy,
1731         .set_attr = mpic_set_attr,
1732         .get_attr = mpic_get_attr,
1733         .has_attr = mpic_has_attr,
1734 };
1735
1736 int kvmppc_mpic_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu,
1737                              u32 cpu)
1738 {
1739         struct openpic *opp = dev->private;
1740         int ret = 0;
1741
1742         if (dev->ops != &kvm_mpic_ops)
1743                 return -EPERM;
1744         if (opp->kvm != vcpu->kvm)
1745                 return -EPERM;
1746         if (cpu < 0 || cpu >= MAX_CPU)
1747                 return -EPERM;
1748
1749         spin_lock_irq(&opp->lock);
1750
1751         if (opp->dst[cpu].vcpu) {
1752                 ret = -EEXIST;
1753                 goto out;
1754         }
1755         if (vcpu->arch.irq_type) {
1756                 ret = -EBUSY;
1757                 goto out;
1758         }
1759
1760         opp->dst[cpu].vcpu = vcpu;
1761         opp->nb_cpus = max(opp->nb_cpus, cpu + 1);
1762
1763         vcpu->arch.mpic = opp;
1764         vcpu->arch.irq_cpu_id = cpu;
1765         vcpu->arch.irq_type = KVMPPC_IRQ_MPIC;
1766
1767         /* This might need to be changed if GCR gets extended */
1768         if (opp->mpic_mode_mask == GCR_MODE_PROXY)
1769                 vcpu->arch.epr_flags |= KVMPPC_EPR_KERNEL;
1770
1771 out:
1772         spin_unlock_irq(&opp->lock);
1773         return ret;
1774 }
1775
1776 /*
1777  * This should only happen immediately before the mpic is destroyed,
1778  * so we shouldn't need to worry about anything still trying to
1779  * access the vcpu pointer.
1780  */
1781 void kvmppc_mpic_disconnect_vcpu(struct openpic *opp, struct kvm_vcpu *vcpu)
1782 {
1783         BUG_ON(!opp->dst[vcpu->arch.irq_cpu_id].vcpu);
1784
1785         opp->dst[vcpu->arch.irq_cpu_id].vcpu = NULL;
1786 }
1787
1788 /*
1789  * Return value:
1790  *  < 0   Interrupt was ignored (masked or not delivered for other reasons)
1791  *  = 0   Interrupt was coalesced (previous irq is still pending)
1792  *  > 0   Number of CPUs interrupt was delivered to
1793  */
1794 static int mpic_set_irq(struct kvm_kernel_irq_routing_entry *e,
1795                         struct kvm *kvm, int irq_source_id, int level,
1796                         bool line_status)
1797 {
1798         u32 irq = e->irqchip.pin;
1799         struct openpic *opp = kvm->arch.mpic;
1800         unsigned long flags;
1801
1802         spin_lock_irqsave(&opp->lock, flags);
1803         openpic_set_irq(opp, irq, level);
1804         spin_unlock_irqrestore(&opp->lock, flags);
1805
1806         /* All code paths we care about don't check for the return value */
1807         return 0;
1808 }
1809
1810 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
1811                 struct kvm *kvm, int irq_source_id, int level, bool line_status)
1812 {
1813         struct openpic *opp = kvm->arch.mpic;
1814         unsigned long flags;
1815
1816         spin_lock_irqsave(&opp->lock, flags);
1817
1818         /*
1819          * XXX We ignore the target address for now, as we only support
1820          *     a single MSI bank.
1821          */
1822         openpic_msi_write(kvm->arch.mpic, MSIIR_OFFSET, e->msi.data);
1823         spin_unlock_irqrestore(&opp->lock, flags);
1824
1825         /* All code paths we care about don't check for the return value */
1826         return 0;
1827 }
1828
1829 int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry *e,
1830                           const struct kvm_irq_routing_entry *ue)
1831 {
1832         int r = -EINVAL;
1833
1834         switch (ue->type) {
1835         case KVM_IRQ_ROUTING_IRQCHIP:
1836                 e->set = mpic_set_irq;
1837                 e->irqchip.irqchip = ue->u.irqchip.irqchip;
1838                 e->irqchip.pin = ue->u.irqchip.pin;
1839                 if (e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS)
1840                         goto out;
1841                 break;
1842         case KVM_IRQ_ROUTING_MSI:
1843                 e->set = kvm_set_msi;
1844                 e->msi.address_lo = ue->u.msi.address_lo;
1845                 e->msi.address_hi = ue->u.msi.address_hi;
1846                 e->msi.data = ue->u.msi.data;
1847                 break;
1848         default:
1849                 goto out;
1850         }
1851
1852         r = 0;
1853 out:
1854         return r;
1855 }