Merge branches 'arm/rockchip', 'arm/exynos', 'arm/smmu', 'x86/vt-d', 'x86/amd', ...
[sfrench/cifs-2.6.git] / drivers / irqchip / irq-gic-v3-its.c
1 /*
2  * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
3  * Author: Marc Zyngier <marc.zyngier@arm.com>
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License version 2 as
7  * published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16  */
17
18 #include <linux/bitmap.h>
19 #include <linux/cpu.h>
20 #include <linux/delay.h>
21 #include <linux/interrupt.h>
22 #include <linux/log2.h>
23 #include <linux/mm.h>
24 #include <linux/msi.h>
25 #include <linux/of.h>
26 #include <linux/of_address.h>
27 #include <linux/of_irq.h>
28 #include <linux/of_pci.h>
29 #include <linux/of_platform.h>
30 #include <linux/percpu.h>
31 #include <linux/slab.h>
32
33 #include <linux/irqchip/arm-gic-v3.h>
34
35 #include <asm/cacheflush.h>
36 #include <asm/cputype.h>
37 #include <asm/exception.h>
38
39 #include "irqchip.h"
40
41 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING           (1 << 0)
42
43 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING     (1 << 0)
44
45 /*
46  * Collection structure - just an ID, and a redistributor address to
47  * ping. We use one per CPU as a bag of interrupts assigned to this
48  * CPU.
49  */
50 struct its_collection {
51         u64                     target_address;
52         u16                     col_id;
53 };
54
55 /*
56  * The ITS structure - contains most of the infrastructure, with the
57  * msi_controller, the command queue, the collections, and the list of
58  * devices writing to it.
59  */
60 struct its_node {
61         raw_spinlock_t          lock;
62         struct list_head        entry;
63         struct msi_controller   msi_chip;
64         struct irq_domain       *domain;
65         void __iomem            *base;
66         unsigned long           phys_base;
67         struct its_cmd_block    *cmd_base;
68         struct its_cmd_block    *cmd_write;
69         void                    *tables[GITS_BASER_NR_REGS];
70         struct its_collection   *collections;
71         struct list_head        its_device_list;
72         u64                     flags;
73         u32                     ite_size;
74 };
75
76 #define ITS_ITT_ALIGN           SZ_256
77
78 /*
79  * The ITS view of a device - belongs to an ITS, a collection, owns an
80  * interrupt translation table, and a list of interrupts.
81  */
82 struct its_device {
83         struct list_head        entry;
84         struct its_node         *its;
85         struct its_collection   *collection;
86         void                    *itt;
87         unsigned long           *lpi_map;
88         irq_hw_number_t         lpi_base;
89         int                     nr_lpis;
90         u32                     nr_ites;
91         u32                     device_id;
92 };
93
94 static LIST_HEAD(its_nodes);
95 static DEFINE_SPINLOCK(its_lock);
96 static struct device_node *gic_root_node;
97 static struct rdists *gic_rdists;
98
99 #define gic_data_rdist()                (raw_cpu_ptr(gic_rdists->rdist))
100 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
101
102 /*
103  * ITS command descriptors - parameters to be encoded in a command
104  * block.
105  */
106 struct its_cmd_desc {
107         union {
108                 struct {
109                         struct its_device *dev;
110                         u32 event_id;
111                 } its_inv_cmd;
112
113                 struct {
114                         struct its_device *dev;
115                         u32 event_id;
116                 } its_int_cmd;
117
118                 struct {
119                         struct its_device *dev;
120                         int valid;
121                 } its_mapd_cmd;
122
123                 struct {
124                         struct its_collection *col;
125                         int valid;
126                 } its_mapc_cmd;
127
128                 struct {
129                         struct its_device *dev;
130                         u32 phys_id;
131                         u32 event_id;
132                 } its_mapvi_cmd;
133
134                 struct {
135                         struct its_device *dev;
136                         struct its_collection *col;
137                         u32 id;
138                 } its_movi_cmd;
139
140                 struct {
141                         struct its_device *dev;
142                         u32 event_id;
143                 } its_discard_cmd;
144
145                 struct {
146                         struct its_collection *col;
147                 } its_invall_cmd;
148         };
149 };
150
151 /*
152  * The ITS command block, which is what the ITS actually parses.
153  */
154 struct its_cmd_block {
155         u64     raw_cmd[4];
156 };
157
158 #define ITS_CMD_QUEUE_SZ                SZ_64K
159 #define ITS_CMD_QUEUE_NR_ENTRIES        (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
160
161 typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *,
162                                                     struct its_cmd_desc *);
163
164 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
165 {
166         cmd->raw_cmd[0] &= ~0xffUL;
167         cmd->raw_cmd[0] |= cmd_nr;
168 }
169
170 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
171 {
172         cmd->raw_cmd[0] &= BIT_ULL(32) - 1;
173         cmd->raw_cmd[0] |= ((u64)devid) << 32;
174 }
175
176 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
177 {
178         cmd->raw_cmd[1] &= ~0xffffffffUL;
179         cmd->raw_cmd[1] |= id;
180 }
181
182 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
183 {
184         cmd->raw_cmd[1] &= 0xffffffffUL;
185         cmd->raw_cmd[1] |= ((u64)phys_id) << 32;
186 }
187
188 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
189 {
190         cmd->raw_cmd[1] &= ~0x1fUL;
191         cmd->raw_cmd[1] |= size & 0x1f;
192 }
193
194 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
195 {
196         cmd->raw_cmd[2] &= ~0xffffffffffffUL;
197         cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL;
198 }
199
200 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
201 {
202         cmd->raw_cmd[2] &= ~(1UL << 63);
203         cmd->raw_cmd[2] |= ((u64)!!valid) << 63;
204 }
205
206 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
207 {
208         cmd->raw_cmd[2] &= ~(0xffffffffUL << 16);
209         cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16));
210 }
211
212 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
213 {
214         cmd->raw_cmd[2] &= ~0xffffUL;
215         cmd->raw_cmd[2] |= col;
216 }
217
218 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
219 {
220         /* Let's fixup BE commands */
221         cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]);
222         cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]);
223         cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]);
224         cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);
225 }
226
227 static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd,
228                                                  struct its_cmd_desc *desc)
229 {
230         unsigned long itt_addr;
231         u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
232
233         itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
234         itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
235
236         its_encode_cmd(cmd, GITS_CMD_MAPD);
237         its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
238         its_encode_size(cmd, size - 1);
239         its_encode_itt(cmd, itt_addr);
240         its_encode_valid(cmd, desc->its_mapd_cmd.valid);
241
242         its_fixup_cmd(cmd);
243
244         return desc->its_mapd_cmd.dev->collection;
245 }
246
247 static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd,
248                                                  struct its_cmd_desc *desc)
249 {
250         its_encode_cmd(cmd, GITS_CMD_MAPC);
251         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
252         its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
253         its_encode_valid(cmd, desc->its_mapc_cmd.valid);
254
255         its_fixup_cmd(cmd);
256
257         return desc->its_mapc_cmd.col;
258 }
259
260 static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd,
261                                                   struct its_cmd_desc *desc)
262 {
263         its_encode_cmd(cmd, GITS_CMD_MAPVI);
264         its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id);
265         its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id);
266         its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id);
267         its_encode_collection(cmd, desc->its_mapvi_cmd.dev->collection->col_id);
268
269         its_fixup_cmd(cmd);
270
271         return desc->its_mapvi_cmd.dev->collection;
272 }
273
274 static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd,
275                                                  struct its_cmd_desc *desc)
276 {
277         its_encode_cmd(cmd, GITS_CMD_MOVI);
278         its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
279         its_encode_event_id(cmd, desc->its_movi_cmd.id);
280         its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
281
282         its_fixup_cmd(cmd);
283
284         return desc->its_movi_cmd.dev->collection;
285 }
286
287 static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd,
288                                                     struct its_cmd_desc *desc)
289 {
290         its_encode_cmd(cmd, GITS_CMD_DISCARD);
291         its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
292         its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
293
294         its_fixup_cmd(cmd);
295
296         return desc->its_discard_cmd.dev->collection;
297 }
298
299 static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd,
300                                                 struct its_cmd_desc *desc)
301 {
302         its_encode_cmd(cmd, GITS_CMD_INV);
303         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
304         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
305
306         its_fixup_cmd(cmd);
307
308         return desc->its_inv_cmd.dev->collection;
309 }
310
311 static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd,
312                                                    struct its_cmd_desc *desc)
313 {
314         its_encode_cmd(cmd, GITS_CMD_INVALL);
315         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
316
317         its_fixup_cmd(cmd);
318
319         return NULL;
320 }
321
322 static u64 its_cmd_ptr_to_offset(struct its_node *its,
323                                  struct its_cmd_block *ptr)
324 {
325         return (ptr - its->cmd_base) * sizeof(*ptr);
326 }
327
328 static int its_queue_full(struct its_node *its)
329 {
330         int widx;
331         int ridx;
332
333         widx = its->cmd_write - its->cmd_base;
334         ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
335
336         /* This is incredibly unlikely to happen, unless the ITS locks up. */
337         if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
338                 return 1;
339
340         return 0;
341 }
342
343 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
344 {
345         struct its_cmd_block *cmd;
346         u32 count = 1000000;    /* 1s! */
347
348         while (its_queue_full(its)) {
349                 count--;
350                 if (!count) {
351                         pr_err_ratelimited("ITS queue not draining\n");
352                         return NULL;
353                 }
354                 cpu_relax();
355                 udelay(1);
356         }
357
358         cmd = its->cmd_write++;
359
360         /* Handle queue wrapping */
361         if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
362                 its->cmd_write = its->cmd_base;
363
364         return cmd;
365 }
366
367 static struct its_cmd_block *its_post_commands(struct its_node *its)
368 {
369         u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
370
371         writel_relaxed(wr, its->base + GITS_CWRITER);
372
373         return its->cmd_write;
374 }
375
376 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
377 {
378         /*
379          * Make sure the commands written to memory are observable by
380          * the ITS.
381          */
382         if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
383                 __flush_dcache_area(cmd, sizeof(*cmd));
384         else
385                 dsb(ishst);
386 }
387
388 static void its_wait_for_range_completion(struct its_node *its,
389                                           struct its_cmd_block *from,
390                                           struct its_cmd_block *to)
391 {
392         u64 rd_idx, from_idx, to_idx;
393         u32 count = 1000000;    /* 1s! */
394
395         from_idx = its_cmd_ptr_to_offset(its, from);
396         to_idx = its_cmd_ptr_to_offset(its, to);
397
398         while (1) {
399                 rd_idx = readl_relaxed(its->base + GITS_CREADR);
400                 if (rd_idx >= to_idx || rd_idx < from_idx)
401                         break;
402
403                 count--;
404                 if (!count) {
405                         pr_err_ratelimited("ITS queue timeout\n");
406                         return;
407                 }
408                 cpu_relax();
409                 udelay(1);
410         }
411 }
412
413 static void its_send_single_command(struct its_node *its,
414                                     its_cmd_builder_t builder,
415                                     struct its_cmd_desc *desc)
416 {
417         struct its_cmd_block *cmd, *sync_cmd, *next_cmd;
418         struct its_collection *sync_col;
419         unsigned long flags;
420
421         raw_spin_lock_irqsave(&its->lock, flags);
422
423         cmd = its_allocate_entry(its);
424         if (!cmd) {             /* We're soooooo screewed... */
425                 pr_err_ratelimited("ITS can't allocate, dropping command\n");
426                 raw_spin_unlock_irqrestore(&its->lock, flags);
427                 return;
428         }
429         sync_col = builder(cmd, desc);
430         its_flush_cmd(its, cmd);
431
432         if (sync_col) {
433                 sync_cmd = its_allocate_entry(its);
434                 if (!sync_cmd) {
435                         pr_err_ratelimited("ITS can't SYNC, skipping\n");
436                         goto post;
437                 }
438                 its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
439                 its_encode_target(sync_cmd, sync_col->target_address);
440                 its_fixup_cmd(sync_cmd);
441                 its_flush_cmd(its, sync_cmd);
442         }
443
444 post:
445         next_cmd = its_post_commands(its);
446         raw_spin_unlock_irqrestore(&its->lock, flags);
447
448         its_wait_for_range_completion(its, cmd, next_cmd);
449 }
450
451 static void its_send_inv(struct its_device *dev, u32 event_id)
452 {
453         struct its_cmd_desc desc;
454
455         desc.its_inv_cmd.dev = dev;
456         desc.its_inv_cmd.event_id = event_id;
457
458         its_send_single_command(dev->its, its_build_inv_cmd, &desc);
459 }
460
461 static void its_send_mapd(struct its_device *dev, int valid)
462 {
463         struct its_cmd_desc desc;
464
465         desc.its_mapd_cmd.dev = dev;
466         desc.its_mapd_cmd.valid = !!valid;
467
468         its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
469 }
470
471 static void its_send_mapc(struct its_node *its, struct its_collection *col,
472                           int valid)
473 {
474         struct its_cmd_desc desc;
475
476         desc.its_mapc_cmd.col = col;
477         desc.its_mapc_cmd.valid = !!valid;
478
479         its_send_single_command(its, its_build_mapc_cmd, &desc);
480 }
481
482 static void its_send_mapvi(struct its_device *dev, u32 irq_id, u32 id)
483 {
484         struct its_cmd_desc desc;
485
486         desc.its_mapvi_cmd.dev = dev;
487         desc.its_mapvi_cmd.phys_id = irq_id;
488         desc.its_mapvi_cmd.event_id = id;
489
490         its_send_single_command(dev->its, its_build_mapvi_cmd, &desc);
491 }
492
493 static void its_send_movi(struct its_device *dev,
494                           struct its_collection *col, u32 id)
495 {
496         struct its_cmd_desc desc;
497
498         desc.its_movi_cmd.dev = dev;
499         desc.its_movi_cmd.col = col;
500         desc.its_movi_cmd.id = id;
501
502         its_send_single_command(dev->its, its_build_movi_cmd, &desc);
503 }
504
505 static void its_send_discard(struct its_device *dev, u32 id)
506 {
507         struct its_cmd_desc desc;
508
509         desc.its_discard_cmd.dev = dev;
510         desc.its_discard_cmd.event_id = id;
511
512         its_send_single_command(dev->its, its_build_discard_cmd, &desc);
513 }
514
515 static void its_send_invall(struct its_node *its, struct its_collection *col)
516 {
517         struct its_cmd_desc desc;
518
519         desc.its_invall_cmd.col = col;
520
521         its_send_single_command(its, its_build_invall_cmd, &desc);
522 }
523
524 /*
525  * irqchip functions - assumes MSI, mostly.
526  */
527
528 static inline u32 its_get_event_id(struct irq_data *d)
529 {
530         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
531         return d->hwirq - its_dev->lpi_base;
532 }
533
534 static void lpi_set_config(struct irq_data *d, bool enable)
535 {
536         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
537         irq_hw_number_t hwirq = d->hwirq;
538         u32 id = its_get_event_id(d);
539         u8 *cfg = page_address(gic_rdists->prop_page) + hwirq - 8192;
540
541         if (enable)
542                 *cfg |= LPI_PROP_ENABLED;
543         else
544                 *cfg &= ~LPI_PROP_ENABLED;
545
546         /*
547          * Make the above write visible to the redistributors.
548          * And yes, we're flushing exactly: One. Single. Byte.
549          * Humpf...
550          */
551         if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
552                 __flush_dcache_area(cfg, sizeof(*cfg));
553         else
554                 dsb(ishst);
555         its_send_inv(its_dev, id);
556 }
557
558 static void its_mask_irq(struct irq_data *d)
559 {
560         lpi_set_config(d, false);
561 }
562
563 static void its_unmask_irq(struct irq_data *d)
564 {
565         lpi_set_config(d, true);
566 }
567
568 static void its_eoi_irq(struct irq_data *d)
569 {
570         gic_write_eoir(d->hwirq);
571 }
572
573 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
574                             bool force)
575 {
576         unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask);
577         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
578         struct its_collection *target_col;
579         u32 id = its_get_event_id(d);
580
581         if (cpu >= nr_cpu_ids)
582                 return -EINVAL;
583
584         target_col = &its_dev->its->collections[cpu];
585         its_send_movi(its_dev, target_col, id);
586         its_dev->collection = target_col;
587
588         return IRQ_SET_MASK_OK_DONE;
589 }
590
591 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
592 {
593         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
594         struct its_node *its;
595         u64 addr;
596
597         its = its_dev->its;
598         addr = its->phys_base + GITS_TRANSLATER;
599
600         msg->address_lo         = addr & ((1UL << 32) - 1);
601         msg->address_hi         = addr >> 32;
602         msg->data               = its_get_event_id(d);
603 }
604
605 static struct irq_chip its_irq_chip = {
606         .name                   = "ITS",
607         .irq_mask               = its_mask_irq,
608         .irq_unmask             = its_unmask_irq,
609         .irq_eoi                = its_eoi_irq,
610         .irq_set_affinity       = its_set_affinity,
611         .irq_compose_msi_msg    = its_irq_compose_msi_msg,
612 };
613
614 static void its_mask_msi_irq(struct irq_data *d)
615 {
616         pci_msi_mask_irq(d);
617         irq_chip_mask_parent(d);
618 }
619
620 static void its_unmask_msi_irq(struct irq_data *d)
621 {
622         pci_msi_unmask_irq(d);
623         irq_chip_unmask_parent(d);
624 }
625
626 static struct irq_chip its_msi_irq_chip = {
627         .name                   = "ITS-MSI",
628         .irq_unmask             = its_unmask_msi_irq,
629         .irq_mask               = its_mask_msi_irq,
630         .irq_eoi                = irq_chip_eoi_parent,
631         .irq_write_msi_msg      = pci_msi_domain_write_msg,
632 };
633
634 /*
635  * How we allocate LPIs:
636  *
637  * The GIC has id_bits bits for interrupt identifiers. From there, we
638  * must subtract 8192 which are reserved for SGIs/PPIs/SPIs. Then, as
639  * we allocate LPIs by chunks of 32, we can shift the whole thing by 5
640  * bits to the right.
641  *
642  * This gives us (((1UL << id_bits) - 8192) >> 5) possible allocations.
643  */
644 #define IRQS_PER_CHUNK_SHIFT    5
645 #define IRQS_PER_CHUNK          (1 << IRQS_PER_CHUNK_SHIFT)
646
647 static unsigned long *lpi_bitmap;
648 static u32 lpi_chunks;
649 static DEFINE_SPINLOCK(lpi_lock);
650
651 static int its_lpi_to_chunk(int lpi)
652 {
653         return (lpi - 8192) >> IRQS_PER_CHUNK_SHIFT;
654 }
655
656 static int its_chunk_to_lpi(int chunk)
657 {
658         return (chunk << IRQS_PER_CHUNK_SHIFT) + 8192;
659 }
660
661 static int its_lpi_init(u32 id_bits)
662 {
663         lpi_chunks = its_lpi_to_chunk(1UL << id_bits);
664
665         lpi_bitmap = kzalloc(BITS_TO_LONGS(lpi_chunks) * sizeof(long),
666                              GFP_KERNEL);
667         if (!lpi_bitmap) {
668                 lpi_chunks = 0;
669                 return -ENOMEM;
670         }
671
672         pr_info("ITS: Allocated %d chunks for LPIs\n", (int)lpi_chunks);
673         return 0;
674 }
675
676 static unsigned long *its_lpi_alloc_chunks(int nr_irqs, int *base, int *nr_ids)
677 {
678         unsigned long *bitmap = NULL;
679         int chunk_id;
680         int nr_chunks;
681         int i;
682
683         nr_chunks = DIV_ROUND_UP(nr_irqs, IRQS_PER_CHUNK);
684
685         spin_lock(&lpi_lock);
686
687         do {
688                 chunk_id = bitmap_find_next_zero_area(lpi_bitmap, lpi_chunks,
689                                                       0, nr_chunks, 0);
690                 if (chunk_id < lpi_chunks)
691                         break;
692
693                 nr_chunks--;
694         } while (nr_chunks > 0);
695
696         if (!nr_chunks)
697                 goto out;
698
699         bitmap = kzalloc(BITS_TO_LONGS(nr_chunks * IRQS_PER_CHUNK) * sizeof (long),
700                          GFP_ATOMIC);
701         if (!bitmap)
702                 goto out;
703
704         for (i = 0; i < nr_chunks; i++)
705                 set_bit(chunk_id + i, lpi_bitmap);
706
707         *base = its_chunk_to_lpi(chunk_id);
708         *nr_ids = nr_chunks * IRQS_PER_CHUNK;
709
710 out:
711         spin_unlock(&lpi_lock);
712
713         return bitmap;
714 }
715
716 static void its_lpi_free(unsigned long *bitmap, int base, int nr_ids)
717 {
718         int lpi;
719
720         spin_lock(&lpi_lock);
721
722         for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) {
723                 int chunk = its_lpi_to_chunk(lpi);
724                 BUG_ON(chunk > lpi_chunks);
725                 if (test_bit(chunk, lpi_bitmap)) {
726                         clear_bit(chunk, lpi_bitmap);
727                 } else {
728                         pr_err("Bad LPI chunk %d\n", chunk);
729                 }
730         }
731
732         spin_unlock(&lpi_lock);
733
734         kfree(bitmap);
735 }
736
737 /*
738  * We allocate 64kB for PROPBASE. That gives us at most 64K LPIs to
739  * deal with (one configuration byte per interrupt). PENDBASE has to
740  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
741  */
742 #define LPI_PROPBASE_SZ         SZ_64K
743 #define LPI_PENDBASE_SZ         (LPI_PROPBASE_SZ / 8 + SZ_1K)
744
745 /*
746  * This is how many bits of ID we need, including the useless ones.
747  */
748 #define LPI_NRBITS              ilog2(LPI_PROPBASE_SZ + SZ_8K)
749
750 #define LPI_PROP_DEFAULT_PRIO   0xa0
751
752 static int __init its_alloc_lpi_tables(void)
753 {
754         phys_addr_t paddr;
755
756         gic_rdists->prop_page = alloc_pages(GFP_NOWAIT,
757                                            get_order(LPI_PROPBASE_SZ));
758         if (!gic_rdists->prop_page) {
759                 pr_err("Failed to allocate PROPBASE\n");
760                 return -ENOMEM;
761         }
762
763         paddr = page_to_phys(gic_rdists->prop_page);
764         pr_info("GIC: using LPI property table @%pa\n", &paddr);
765
766         /* Priority 0xa0, Group-1, disabled */
767         memset(page_address(gic_rdists->prop_page),
768                LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1,
769                LPI_PROPBASE_SZ);
770
771         /* Make sure the GIC will observe the written configuration */
772         __flush_dcache_area(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ);
773
774         return 0;
775 }
776
777 static const char *its_base_type_string[] = {
778         [GITS_BASER_TYPE_DEVICE]        = "Devices",
779         [GITS_BASER_TYPE_VCPU]          = "Virtual CPUs",
780         [GITS_BASER_TYPE_CPU]           = "Physical CPUs",
781         [GITS_BASER_TYPE_COLLECTION]    = "Interrupt Collections",
782         [GITS_BASER_TYPE_RESERVED5]     = "Reserved (5)",
783         [GITS_BASER_TYPE_RESERVED6]     = "Reserved (6)",
784         [GITS_BASER_TYPE_RESERVED7]     = "Reserved (7)",
785 };
786
787 static void its_free_tables(struct its_node *its)
788 {
789         int i;
790
791         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
792                 if (its->tables[i]) {
793                         free_page((unsigned long)its->tables[i]);
794                         its->tables[i] = NULL;
795                 }
796         }
797 }
798
799 static int its_alloc_tables(struct its_node *its)
800 {
801         int err;
802         int i;
803         int psz = SZ_64K;
804         u64 shr = GITS_BASER_InnerShareable;
805         u64 cache = GITS_BASER_WaWb;
806
807         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
808                 u64 val = readq_relaxed(its->base + GITS_BASER + i * 8);
809                 u64 type = GITS_BASER_TYPE(val);
810                 u64 entry_size = GITS_BASER_ENTRY_SIZE(val);
811                 int order = get_order(psz);
812                 int alloc_size;
813                 u64 tmp;
814                 void *base;
815
816                 if (type == GITS_BASER_TYPE_NONE)
817                         continue;
818
819                 /*
820                  * Allocate as many entries as required to fit the
821                  * range of device IDs that the ITS can grok... The ID
822                  * space being incredibly sparse, this results in a
823                  * massive waste of memory.
824                  *
825                  * For other tables, only allocate a single page.
826                  */
827                 if (type == GITS_BASER_TYPE_DEVICE) {
828                         u64 typer = readq_relaxed(its->base + GITS_TYPER);
829                         u32 ids = GITS_TYPER_DEVBITS(typer);
830
831                         /*
832                          * 'order' was initialized earlier to the default page
833                          * granule of the the ITS.  We can't have an allocation
834                          * smaller than that.  If the requested allocation
835                          * is smaller, round up to the default page granule.
836                          */
837                         order = max(get_order((1UL << ids) * entry_size),
838                                     order);
839                         if (order >= MAX_ORDER) {
840                                 order = MAX_ORDER - 1;
841                                 pr_warn("%s: Device Table too large, reduce its page order to %u\n",
842                                         its->msi_chip.of_node->full_name, order);
843                         }
844                 }
845
846                 alloc_size = (1 << order) * PAGE_SIZE;
847                 base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
848                 if (!base) {
849                         err = -ENOMEM;
850                         goto out_free;
851                 }
852
853                 its->tables[i] = base;
854
855 retry_baser:
856                 val = (virt_to_phys(base)                                |
857                        (type << GITS_BASER_TYPE_SHIFT)                   |
858                        ((entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) |
859                        cache                                             |
860                        shr                                               |
861                        GITS_BASER_VALID);
862
863                 switch (psz) {
864                 case SZ_4K:
865                         val |= GITS_BASER_PAGE_SIZE_4K;
866                         break;
867                 case SZ_16K:
868                         val |= GITS_BASER_PAGE_SIZE_16K;
869                         break;
870                 case SZ_64K:
871                         val |= GITS_BASER_PAGE_SIZE_64K;
872                         break;
873                 }
874
875                 val |= (alloc_size / psz) - 1;
876
877                 writeq_relaxed(val, its->base + GITS_BASER + i * 8);
878                 tmp = readq_relaxed(its->base + GITS_BASER + i * 8);
879
880                 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
881                         /*
882                          * Shareability didn't stick. Just use
883                          * whatever the read reported, which is likely
884                          * to be the only thing this redistributor
885                          * supports. If that's zero, make it
886                          * non-cacheable as well.
887                          */
888                         shr = tmp & GITS_BASER_SHAREABILITY_MASK;
889                         if (!shr)
890                                 cache = GITS_BASER_nC;
891                         goto retry_baser;
892                 }
893
894                 if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) {
895                         /*
896                          * Page size didn't stick. Let's try a smaller
897                          * size and retry. If we reach 4K, then
898                          * something is horribly wrong...
899                          */
900                         switch (psz) {
901                         case SZ_16K:
902                                 psz = SZ_4K;
903                                 goto retry_baser;
904                         case SZ_64K:
905                                 psz = SZ_16K;
906                                 goto retry_baser;
907                         }
908                 }
909
910                 if (val != tmp) {
911                         pr_err("ITS: %s: GITS_BASER%d doesn't stick: %lx %lx\n",
912                                its->msi_chip.of_node->full_name, i,
913                                (unsigned long) val, (unsigned long) tmp);
914                         err = -ENXIO;
915                         goto out_free;
916                 }
917
918                 pr_info("ITS: allocated %d %s @%lx (psz %dK, shr %d)\n",
919                         (int)(alloc_size / entry_size),
920                         its_base_type_string[type],
921                         (unsigned long)virt_to_phys(base),
922                         psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
923         }
924
925         return 0;
926
927 out_free:
928         its_free_tables(its);
929
930         return err;
931 }
932
933 static int its_alloc_collections(struct its_node *its)
934 {
935         its->collections = kzalloc(nr_cpu_ids * sizeof(*its->collections),
936                                    GFP_KERNEL);
937         if (!its->collections)
938                 return -ENOMEM;
939
940         return 0;
941 }
942
943 static void its_cpu_init_lpis(void)
944 {
945         void __iomem *rbase = gic_data_rdist_rd_base();
946         struct page *pend_page;
947         u64 val, tmp;
948
949         /* If we didn't allocate the pending table yet, do it now */
950         pend_page = gic_data_rdist()->pend_page;
951         if (!pend_page) {
952                 phys_addr_t paddr;
953                 /*
954                  * The pending pages have to be at least 64kB aligned,
955                  * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
956                  */
957                 pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO,
958                                         get_order(max(LPI_PENDBASE_SZ, SZ_64K)));
959                 if (!pend_page) {
960                         pr_err("Failed to allocate PENDBASE for CPU%d\n",
961                                smp_processor_id());
962                         return;
963                 }
964
965                 /* Make sure the GIC will observe the zero-ed page */
966                 __flush_dcache_area(page_address(pend_page), LPI_PENDBASE_SZ);
967
968                 paddr = page_to_phys(pend_page);
969                 pr_info("CPU%d: using LPI pending table @%pa\n",
970                         smp_processor_id(), &paddr);
971                 gic_data_rdist()->pend_page = pend_page;
972         }
973
974         /* Disable LPIs */
975         val = readl_relaxed(rbase + GICR_CTLR);
976         val &= ~GICR_CTLR_ENABLE_LPIS;
977         writel_relaxed(val, rbase + GICR_CTLR);
978
979         /*
980          * Make sure any change to the table is observable by the GIC.
981          */
982         dsb(sy);
983
984         /* set PROPBASE */
985         val = (page_to_phys(gic_rdists->prop_page) |
986                GICR_PROPBASER_InnerShareable |
987                GICR_PROPBASER_WaWb |
988                ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
989
990         writeq_relaxed(val, rbase + GICR_PROPBASER);
991         tmp = readq_relaxed(rbase + GICR_PROPBASER);
992
993         if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
994                 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
995                         /*
996                          * The HW reports non-shareable, we must
997                          * remove the cacheability attributes as
998                          * well.
999                          */
1000                         val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
1001                                  GICR_PROPBASER_CACHEABILITY_MASK);
1002                         val |= GICR_PROPBASER_nC;
1003                         writeq_relaxed(val, rbase + GICR_PROPBASER);
1004                 }
1005                 pr_info_once("GIC: using cache flushing for LPI property table\n");
1006                 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
1007         }
1008
1009         /* set PENDBASE */
1010         val = (page_to_phys(pend_page) |
1011                GICR_PENDBASER_InnerShareable |
1012                GICR_PENDBASER_WaWb);
1013
1014         writeq_relaxed(val, rbase + GICR_PENDBASER);
1015         tmp = readq_relaxed(rbase + GICR_PENDBASER);
1016
1017         if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
1018                 /*
1019                  * The HW reports non-shareable, we must remove the
1020                  * cacheability attributes as well.
1021                  */
1022                 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
1023                          GICR_PENDBASER_CACHEABILITY_MASK);
1024                 val |= GICR_PENDBASER_nC;
1025                 writeq_relaxed(val, rbase + GICR_PENDBASER);
1026         }
1027
1028         /* Enable LPIs */
1029         val = readl_relaxed(rbase + GICR_CTLR);
1030         val |= GICR_CTLR_ENABLE_LPIS;
1031         writel_relaxed(val, rbase + GICR_CTLR);
1032
1033         /* Make sure the GIC has seen the above */
1034         dsb(sy);
1035 }
1036
1037 static void its_cpu_init_collection(void)
1038 {
1039         struct its_node *its;
1040         int cpu;
1041
1042         spin_lock(&its_lock);
1043         cpu = smp_processor_id();
1044
1045         list_for_each_entry(its, &its_nodes, entry) {
1046                 u64 target;
1047
1048                 /*
1049                  * We now have to bind each collection to its target
1050                  * redistributor.
1051                  */
1052                 if (readq_relaxed(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
1053                         /*
1054                          * This ITS wants the physical address of the
1055                          * redistributor.
1056                          */
1057                         target = gic_data_rdist()->phys_base;
1058                 } else {
1059                         /*
1060                          * This ITS wants a linear CPU number.
1061                          */
1062                         target = readq_relaxed(gic_data_rdist_rd_base() + GICR_TYPER);
1063                         target = GICR_TYPER_CPU_NUMBER(target) << 16;
1064                 }
1065
1066                 /* Perform collection mapping */
1067                 its->collections[cpu].target_address = target;
1068                 its->collections[cpu].col_id = cpu;
1069
1070                 its_send_mapc(its, &its->collections[cpu], 1);
1071                 its_send_invall(its, &its->collections[cpu]);
1072         }
1073
1074         spin_unlock(&its_lock);
1075 }
1076
1077 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
1078 {
1079         struct its_device *its_dev = NULL, *tmp;
1080         unsigned long flags;
1081
1082         raw_spin_lock_irqsave(&its->lock, flags);
1083
1084         list_for_each_entry(tmp, &its->its_device_list, entry) {
1085                 if (tmp->device_id == dev_id) {
1086                         its_dev = tmp;
1087                         break;
1088                 }
1089         }
1090
1091         raw_spin_unlock_irqrestore(&its->lock, flags);
1092
1093         return its_dev;
1094 }
1095
1096 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
1097                                             int nvecs)
1098 {
1099         struct its_device *dev;
1100         unsigned long *lpi_map;
1101         unsigned long flags;
1102         void *itt;
1103         int lpi_base;
1104         int nr_lpis;
1105         int nr_ites;
1106         int cpu;
1107         int sz;
1108
1109         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1110         /*
1111          * At least one bit of EventID is being used, hence a minimum
1112          * of two entries. No, the architecture doesn't let you
1113          * express an ITT with a single entry.
1114          */
1115         nr_ites = max(2UL, roundup_pow_of_two(nvecs));
1116         sz = nr_ites * its->ite_size;
1117         sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
1118         itt = kzalloc(sz, GFP_KERNEL);
1119         lpi_map = its_lpi_alloc_chunks(nvecs, &lpi_base, &nr_lpis);
1120
1121         if (!dev || !itt || !lpi_map) {
1122                 kfree(dev);
1123                 kfree(itt);
1124                 kfree(lpi_map);
1125                 return NULL;
1126         }
1127
1128         dev->its = its;
1129         dev->itt = itt;
1130         dev->nr_ites = nr_ites;
1131         dev->lpi_map = lpi_map;
1132         dev->lpi_base = lpi_base;
1133         dev->nr_lpis = nr_lpis;
1134         dev->device_id = dev_id;
1135         INIT_LIST_HEAD(&dev->entry);
1136
1137         raw_spin_lock_irqsave(&its->lock, flags);
1138         list_add(&dev->entry, &its->its_device_list);
1139         raw_spin_unlock_irqrestore(&its->lock, flags);
1140
1141         /* Bind the device to the first possible CPU */
1142         cpu = cpumask_first(cpu_online_mask);
1143         dev->collection = &its->collections[cpu];
1144
1145         /* Map device to its ITT */
1146         its_send_mapd(dev, 1);
1147
1148         return dev;
1149 }
1150
1151 static void its_free_device(struct its_device *its_dev)
1152 {
1153         unsigned long flags;
1154
1155         raw_spin_lock_irqsave(&its_dev->its->lock, flags);
1156         list_del(&its_dev->entry);
1157         raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
1158         kfree(its_dev->itt);
1159         kfree(its_dev);
1160 }
1161
1162 static int its_alloc_device_irq(struct its_device *dev, irq_hw_number_t *hwirq)
1163 {
1164         int idx;
1165
1166         idx = find_first_zero_bit(dev->lpi_map, dev->nr_lpis);
1167         if (idx == dev->nr_lpis)
1168                 return -ENOSPC;
1169
1170         *hwirq = dev->lpi_base + idx;
1171         set_bit(idx, dev->lpi_map);
1172
1173         return 0;
1174 }
1175
1176 struct its_pci_alias {
1177         struct pci_dev  *pdev;
1178         u32             dev_id;
1179         u32             count;
1180 };
1181
1182 static int its_pci_msi_vec_count(struct pci_dev *pdev)
1183 {
1184         int msi, msix;
1185
1186         msi = max(pci_msi_vec_count(pdev), 0);
1187         msix = max(pci_msix_vec_count(pdev), 0);
1188
1189         return max(msi, msix);
1190 }
1191
1192 static int its_get_pci_alias(struct pci_dev *pdev, u16 alias, void *data)
1193 {
1194         struct its_pci_alias *dev_alias = data;
1195
1196         dev_alias->dev_id = alias;
1197         if (pdev != dev_alias->pdev)
1198                 dev_alias->count += its_pci_msi_vec_count(dev_alias->pdev);
1199
1200         return 0;
1201 }
1202
1203 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
1204                            int nvec, msi_alloc_info_t *info)
1205 {
1206         struct pci_dev *pdev;
1207         struct its_node *its;
1208         struct its_device *its_dev;
1209         struct its_pci_alias dev_alias;
1210
1211         if (!dev_is_pci(dev))
1212                 return -EINVAL;
1213
1214         pdev = to_pci_dev(dev);
1215         dev_alias.pdev = pdev;
1216         dev_alias.count = nvec;
1217
1218         pci_for_each_dma_alias(pdev, its_get_pci_alias, &dev_alias);
1219         its = domain->parent->host_data;
1220
1221         its_dev = its_find_device(its, dev_alias.dev_id);
1222         if (its_dev) {
1223                 /*
1224                  * We already have seen this ID, probably through
1225                  * another alias (PCI bridge of some sort). No need to
1226                  * create the device.
1227                  */
1228                 dev_dbg(dev, "Reusing ITT for devID %x\n", dev_alias.dev_id);
1229                 goto out;
1230         }
1231
1232         its_dev = its_create_device(its, dev_alias.dev_id, dev_alias.count);
1233         if (!its_dev)
1234                 return -ENOMEM;
1235
1236         dev_dbg(&pdev->dev, "ITT %d entries, %d bits\n",
1237                 dev_alias.count, ilog2(dev_alias.count));
1238 out:
1239         info->scratchpad[0].ptr = its_dev;
1240         info->scratchpad[1].ptr = dev;
1241         return 0;
1242 }
1243
1244 static struct msi_domain_ops its_pci_msi_ops = {
1245         .msi_prepare    = its_msi_prepare,
1246 };
1247
1248 static struct msi_domain_info its_pci_msi_domain_info = {
1249         .flags  = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
1250                    MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX),
1251         .ops    = &its_pci_msi_ops,
1252         .chip   = &its_msi_irq_chip,
1253 };
1254
1255 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
1256                                     unsigned int virq,
1257                                     irq_hw_number_t hwirq)
1258 {
1259         struct of_phandle_args args;
1260
1261         args.np = domain->parent->of_node;
1262         args.args_count = 3;
1263         args.args[0] = GIC_IRQ_TYPE_LPI;
1264         args.args[1] = hwirq;
1265         args.args[2] = IRQ_TYPE_EDGE_RISING;
1266
1267         return irq_domain_alloc_irqs_parent(domain, virq, 1, &args);
1268 }
1269
1270 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1271                                 unsigned int nr_irqs, void *args)
1272 {
1273         msi_alloc_info_t *info = args;
1274         struct its_device *its_dev = info->scratchpad[0].ptr;
1275         irq_hw_number_t hwirq;
1276         int err;
1277         int i;
1278
1279         for (i = 0; i < nr_irqs; i++) {
1280                 err = its_alloc_device_irq(its_dev, &hwirq);
1281                 if (err)
1282                         return err;
1283
1284                 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq);
1285                 if (err)
1286                         return err;
1287
1288                 irq_domain_set_hwirq_and_chip(domain, virq + i,
1289                                               hwirq, &its_irq_chip, its_dev);
1290                 dev_dbg(info->scratchpad[1].ptr, "ID:%d pID:%d vID:%d\n",
1291                         (int)(hwirq - its_dev->lpi_base), (int)hwirq, virq + i);
1292         }
1293
1294         return 0;
1295 }
1296
1297 static void its_irq_domain_activate(struct irq_domain *domain,
1298                                     struct irq_data *d)
1299 {
1300         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1301         u32 event = its_get_event_id(d);
1302
1303         /* Map the GIC IRQ and event to the device */
1304         its_send_mapvi(its_dev, d->hwirq, event);
1305 }
1306
1307 static void its_irq_domain_deactivate(struct irq_domain *domain,
1308                                       struct irq_data *d)
1309 {
1310         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1311         u32 event = its_get_event_id(d);
1312
1313         /* Stop the delivery of interrupts */
1314         its_send_discard(its_dev, event);
1315 }
1316
1317 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1318                                 unsigned int nr_irqs)
1319 {
1320         struct irq_data *d = irq_domain_get_irq_data(domain, virq);
1321         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1322         int i;
1323
1324         for (i = 0; i < nr_irqs; i++) {
1325                 struct irq_data *data = irq_domain_get_irq_data(domain,
1326                                                                 virq + i);
1327                 u32 event = its_get_event_id(data);
1328
1329                 /* Mark interrupt index as unused */
1330                 clear_bit(event, its_dev->lpi_map);
1331
1332                 /* Nuke the entry in the domain */
1333                 irq_domain_reset_irq_data(data);
1334         }
1335
1336         /* If all interrupts have been freed, start mopping the floor */
1337         if (bitmap_empty(its_dev->lpi_map, its_dev->nr_lpis)) {
1338                 its_lpi_free(its_dev->lpi_map,
1339                              its_dev->lpi_base,
1340                              its_dev->nr_lpis);
1341
1342                 /* Unmap device/itt */
1343                 its_send_mapd(its_dev, 0);
1344                 its_free_device(its_dev);
1345         }
1346
1347         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
1348 }
1349
1350 static const struct irq_domain_ops its_domain_ops = {
1351         .alloc                  = its_irq_domain_alloc,
1352         .free                   = its_irq_domain_free,
1353         .activate               = its_irq_domain_activate,
1354         .deactivate             = its_irq_domain_deactivate,
1355 };
1356
1357 static int its_force_quiescent(void __iomem *base)
1358 {
1359         u32 count = 1000000;    /* 1s */
1360         u32 val;
1361
1362         val = readl_relaxed(base + GITS_CTLR);
1363         if (val & GITS_CTLR_QUIESCENT)
1364                 return 0;
1365
1366         /* Disable the generation of all interrupts to this ITS */
1367         val &= ~GITS_CTLR_ENABLE;
1368         writel_relaxed(val, base + GITS_CTLR);
1369
1370         /* Poll GITS_CTLR and wait until ITS becomes quiescent */
1371         while (1) {
1372                 val = readl_relaxed(base + GITS_CTLR);
1373                 if (val & GITS_CTLR_QUIESCENT)
1374                         return 0;
1375
1376                 count--;
1377                 if (!count)
1378                         return -EBUSY;
1379
1380                 cpu_relax();
1381                 udelay(1);
1382         }
1383 }
1384
1385 static int its_probe(struct device_node *node, struct irq_domain *parent)
1386 {
1387         struct resource res;
1388         struct its_node *its;
1389         void __iomem *its_base;
1390         u32 val;
1391         u64 baser, tmp;
1392         int err;
1393
1394         err = of_address_to_resource(node, 0, &res);
1395         if (err) {
1396                 pr_warn("%s: no regs?\n", node->full_name);
1397                 return -ENXIO;
1398         }
1399
1400         its_base = ioremap(res.start, resource_size(&res));
1401         if (!its_base) {
1402                 pr_warn("%s: unable to map registers\n", node->full_name);
1403                 return -ENOMEM;
1404         }
1405
1406         val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
1407         if (val != 0x30 && val != 0x40) {
1408                 pr_warn("%s: no ITS detected, giving up\n", node->full_name);
1409                 err = -ENODEV;
1410                 goto out_unmap;
1411         }
1412
1413         err = its_force_quiescent(its_base);
1414         if (err) {
1415                 pr_warn("%s: failed to quiesce, giving up\n",
1416                         node->full_name);
1417                 goto out_unmap;
1418         }
1419
1420         pr_info("ITS: %s\n", node->full_name);
1421
1422         its = kzalloc(sizeof(*its), GFP_KERNEL);
1423         if (!its) {
1424                 err = -ENOMEM;
1425                 goto out_unmap;
1426         }
1427
1428         raw_spin_lock_init(&its->lock);
1429         INIT_LIST_HEAD(&its->entry);
1430         INIT_LIST_HEAD(&its->its_device_list);
1431         its->base = its_base;
1432         its->phys_base = res.start;
1433         its->msi_chip.of_node = node;
1434         its->ite_size = ((readl_relaxed(its_base + GITS_TYPER) >> 4) & 0xf) + 1;
1435
1436         its->cmd_base = kzalloc(ITS_CMD_QUEUE_SZ, GFP_KERNEL);
1437         if (!its->cmd_base) {
1438                 err = -ENOMEM;
1439                 goto out_free_its;
1440         }
1441         its->cmd_write = its->cmd_base;
1442
1443         err = its_alloc_tables(its);
1444         if (err)
1445                 goto out_free_cmd;
1446
1447         err = its_alloc_collections(its);
1448         if (err)
1449                 goto out_free_tables;
1450
1451         baser = (virt_to_phys(its->cmd_base)    |
1452                  GITS_CBASER_WaWb               |
1453                  GITS_CBASER_InnerShareable     |
1454                  (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
1455                  GITS_CBASER_VALID);
1456
1457         writeq_relaxed(baser, its->base + GITS_CBASER);
1458         tmp = readq_relaxed(its->base + GITS_CBASER);
1459
1460         if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
1461                 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
1462                         /*
1463                          * The HW reports non-shareable, we must
1464                          * remove the cacheability attributes as
1465                          * well.
1466                          */
1467                         baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
1468                                    GITS_CBASER_CACHEABILITY_MASK);
1469                         baser |= GITS_CBASER_nC;
1470                         writeq_relaxed(baser, its->base + GITS_CBASER);
1471                 }
1472                 pr_info("ITS: using cache flushing for cmd queue\n");
1473                 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
1474         }
1475
1476         writeq_relaxed(0, its->base + GITS_CWRITER);
1477         writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR);
1478
1479         if (of_property_read_bool(its->msi_chip.of_node, "msi-controller")) {
1480                 its->domain = irq_domain_add_tree(NULL, &its_domain_ops, its);
1481                 if (!its->domain) {
1482                         err = -ENOMEM;
1483                         goto out_free_tables;
1484                 }
1485
1486                 its->domain->parent = parent;
1487
1488                 its->msi_chip.domain = pci_msi_create_irq_domain(node,
1489                                                                  &its_pci_msi_domain_info,
1490                                                                  its->domain);
1491                 if (!its->msi_chip.domain) {
1492                         err = -ENOMEM;
1493                         goto out_free_domains;
1494                 }
1495
1496                 err = of_pci_msi_chip_add(&its->msi_chip);
1497                 if (err)
1498                         goto out_free_domains;
1499         }
1500
1501         spin_lock(&its_lock);
1502         list_add(&its->entry, &its_nodes);
1503         spin_unlock(&its_lock);
1504
1505         return 0;
1506
1507 out_free_domains:
1508         if (its->msi_chip.domain)
1509                 irq_domain_remove(its->msi_chip.domain);
1510         if (its->domain)
1511                 irq_domain_remove(its->domain);
1512 out_free_tables:
1513         its_free_tables(its);
1514 out_free_cmd:
1515         kfree(its->cmd_base);
1516 out_free_its:
1517         kfree(its);
1518 out_unmap:
1519         iounmap(its_base);
1520         pr_err("ITS: failed probing %s (%d)\n", node->full_name, err);
1521         return err;
1522 }
1523
1524 static bool gic_rdists_supports_plpis(void)
1525 {
1526         return !!(readl_relaxed(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
1527 }
1528
1529 int its_cpu_init(void)
1530 {
1531         if (!list_empty(&its_nodes)) {
1532                 if (!gic_rdists_supports_plpis()) {
1533                         pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
1534                         return -ENXIO;
1535                 }
1536                 its_cpu_init_lpis();
1537                 its_cpu_init_collection();
1538         }
1539
1540         return 0;
1541 }
1542
1543 static struct of_device_id its_device_id[] = {
1544         {       .compatible     = "arm,gic-v3-its",     },
1545         {},
1546 };
1547
1548 int its_init(struct device_node *node, struct rdists *rdists,
1549              struct irq_domain *parent_domain)
1550 {
1551         struct device_node *np;
1552
1553         for (np = of_find_matching_node(node, its_device_id); np;
1554              np = of_find_matching_node(np, its_device_id)) {
1555                 its_probe(np, parent_domain);
1556         }
1557
1558         if (list_empty(&its_nodes)) {
1559                 pr_warn("ITS: No ITS available, not enabling LPIs\n");
1560                 return -ENXIO;
1561         }
1562
1563         gic_rdists = rdists;
1564         gic_root_node = node;
1565
1566         its_alloc_lpi_tables();
1567         its_lpi_init(rdists->id_bits);
1568
1569         return 0;
1570 }