clocksource/arm_arch_timer: Fix arch_timer_mem_find_best_frame()
[sfrench/cifs-2.6.git] / drivers / clocksource / arm_arch_timer.c
1 /*
2  *  linux/drivers/clocksource/arm_arch_timer.c
3  *
4  *  Copyright (C) 2011 ARM Ltd.
5  *  All Rights Reserved
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11
12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
13
14 #include <linux/init.h>
15 #include <linux/kernel.h>
16 #include <linux/device.h>
17 #include <linux/smp.h>
18 #include <linux/cpu.h>
19 #include <linux/cpu_pm.h>
20 #include <linux/clockchips.h>
21 #include <linux/clocksource.h>
22 #include <linux/interrupt.h>
23 #include <linux/of_irq.h>
24 #include <linux/of_address.h>
25 #include <linux/io.h>
26 #include <linux/slab.h>
27 #include <linux/sched/clock.h>
28 #include <linux/sched_clock.h>
29 #include <linux/acpi.h>
30
31 #include <asm/arch_timer.h>
32 #include <asm/virt.h>
33
34 #include <clocksource/arm_arch_timer.h>
35
36 #undef pr_fmt
37 #define pr_fmt(fmt) "arch_timer: " fmt
38
39 #define CNTTIDR         0x08
40 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
41
42 #define CNTACR(n)       (0x40 + ((n) * 4))
43 #define CNTACR_RPCT     BIT(0)
44 #define CNTACR_RVCT     BIT(1)
45 #define CNTACR_RFRQ     BIT(2)
46 #define CNTACR_RVOFF    BIT(3)
47 #define CNTACR_RWVT     BIT(4)
48 #define CNTACR_RWPT     BIT(5)
49
50 #define CNTVCT_LO       0x08
51 #define CNTVCT_HI       0x0c
52 #define CNTFRQ          0x10
53 #define CNTP_TVAL       0x28
54 #define CNTP_CTL        0x2c
55 #define CNTV_TVAL       0x38
56 #define CNTV_CTL        0x3c
57
58 static unsigned arch_timers_present __initdata;
59
60 static void __iomem *arch_counter_base;
61
62 struct arch_timer {
63         void __iomem *base;
64         struct clock_event_device evt;
65 };
66
67 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
68
69 static u32 arch_timer_rate;
70 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI];
71
72 static struct clock_event_device __percpu *arch_timer_evt;
73
74 static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI;
75 static bool arch_timer_c3stop;
76 static bool arch_timer_mem_use_virtual;
77 static bool arch_counter_suspend_stop;
78 static bool vdso_default = true;
79
80 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
81
82 static int __init early_evtstrm_cfg(char *buf)
83 {
84         return strtobool(buf, &evtstrm_enable);
85 }
86 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
87
88 /*
89  * Architected system timer support.
90  */
91
92 static __always_inline
93 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
94                           struct clock_event_device *clk)
95 {
96         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
97                 struct arch_timer *timer = to_arch_timer(clk);
98                 switch (reg) {
99                 case ARCH_TIMER_REG_CTRL:
100                         writel_relaxed(val, timer->base + CNTP_CTL);
101                         break;
102                 case ARCH_TIMER_REG_TVAL:
103                         writel_relaxed(val, timer->base + CNTP_TVAL);
104                         break;
105                 }
106         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
107                 struct arch_timer *timer = to_arch_timer(clk);
108                 switch (reg) {
109                 case ARCH_TIMER_REG_CTRL:
110                         writel_relaxed(val, timer->base + CNTV_CTL);
111                         break;
112                 case ARCH_TIMER_REG_TVAL:
113                         writel_relaxed(val, timer->base + CNTV_TVAL);
114                         break;
115                 }
116         } else {
117                 arch_timer_reg_write_cp15(access, reg, val);
118         }
119 }
120
121 static __always_inline
122 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
123                         struct clock_event_device *clk)
124 {
125         u32 val;
126
127         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
128                 struct arch_timer *timer = to_arch_timer(clk);
129                 switch (reg) {
130                 case ARCH_TIMER_REG_CTRL:
131                         val = readl_relaxed(timer->base + CNTP_CTL);
132                         break;
133                 case ARCH_TIMER_REG_TVAL:
134                         val = readl_relaxed(timer->base + CNTP_TVAL);
135                         break;
136                 }
137         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
138                 struct arch_timer *timer = to_arch_timer(clk);
139                 switch (reg) {
140                 case ARCH_TIMER_REG_CTRL:
141                         val = readl_relaxed(timer->base + CNTV_CTL);
142                         break;
143                 case ARCH_TIMER_REG_TVAL:
144                         val = readl_relaxed(timer->base + CNTV_TVAL);
145                         break;
146                 }
147         } else {
148                 val = arch_timer_reg_read_cp15(access, reg);
149         }
150
151         return val;
152 }
153
154 /*
155  * Default to cp15 based access because arm64 uses this function for
156  * sched_clock() before DT is probed and the cp15 method is guaranteed
157  * to exist on arm64. arm doesn't use this before DT is probed so even
158  * if we don't have the cp15 accessors we won't have a problem.
159  */
160 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
161
162 static u64 arch_counter_read(struct clocksource *cs)
163 {
164         return arch_timer_read_counter();
165 }
166
167 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
168 {
169         return arch_timer_read_counter();
170 }
171
172 static struct clocksource clocksource_counter = {
173         .name   = "arch_sys_counter",
174         .rating = 400,
175         .read   = arch_counter_read,
176         .mask   = CLOCKSOURCE_MASK(56),
177         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
178 };
179
180 static struct cyclecounter cyclecounter __ro_after_init = {
181         .read   = arch_counter_read_cc,
182         .mask   = CLOCKSOURCE_MASK(56),
183 };
184
185 struct ate_acpi_oem_info {
186         char oem_id[ACPI_OEM_ID_SIZE + 1];
187         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
188         u32 oem_revision;
189 };
190
191 #ifdef CONFIG_FSL_ERRATUM_A008585
192 /*
193  * The number of retries is an arbitrary value well beyond the highest number
194  * of iterations the loop has been observed to take.
195  */
196 #define __fsl_a008585_read_reg(reg) ({                  \
197         u64 _old, _new;                                 \
198         int _retries = 200;                             \
199                                                         \
200         do {                                            \
201                 _old = read_sysreg(reg);                \
202                 _new = read_sysreg(reg);                \
203                 _retries--;                             \
204         } while (unlikely(_old != _new) && _retries);   \
205                                                         \
206         WARN_ON_ONCE(!_retries);                        \
207         _new;                                           \
208 })
209
210 static u32 notrace fsl_a008585_read_cntp_tval_el0(void)
211 {
212         return __fsl_a008585_read_reg(cntp_tval_el0);
213 }
214
215 static u32 notrace fsl_a008585_read_cntv_tval_el0(void)
216 {
217         return __fsl_a008585_read_reg(cntv_tval_el0);
218 }
219
220 static u64 notrace fsl_a008585_read_cntvct_el0(void)
221 {
222         return __fsl_a008585_read_reg(cntvct_el0);
223 }
224 #endif
225
226 #ifdef CONFIG_HISILICON_ERRATUM_161010101
227 /*
228  * Verify whether the value of the second read is larger than the first by
229  * less than 32 is the only way to confirm the value is correct, so clear the
230  * lower 5 bits to check whether the difference is greater than 32 or not.
231  * Theoretically the erratum should not occur more than twice in succession
232  * when reading the system counter, but it is possible that some interrupts
233  * may lead to more than twice read errors, triggering the warning, so setting
234  * the number of retries far beyond the number of iterations the loop has been
235  * observed to take.
236  */
237 #define __hisi_161010101_read_reg(reg) ({                               \
238         u64 _old, _new;                                         \
239         int _retries = 50;                                      \
240                                                                 \
241         do {                                                    \
242                 _old = read_sysreg(reg);                        \
243                 _new = read_sysreg(reg);                        \
244                 _retries--;                                     \
245         } while (unlikely((_new - _old) >> 5) && _retries);     \
246                                                                 \
247         WARN_ON_ONCE(!_retries);                                \
248         _new;                                                   \
249 })
250
251 static u32 notrace hisi_161010101_read_cntp_tval_el0(void)
252 {
253         return __hisi_161010101_read_reg(cntp_tval_el0);
254 }
255
256 static u32 notrace hisi_161010101_read_cntv_tval_el0(void)
257 {
258         return __hisi_161010101_read_reg(cntv_tval_el0);
259 }
260
261 static u64 notrace hisi_161010101_read_cntvct_el0(void)
262 {
263         return __hisi_161010101_read_reg(cntvct_el0);
264 }
265
266 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
267         /*
268          * Note that trailing spaces are required to properly match
269          * the OEM table information.
270          */
271         {
272                 .oem_id         = "HISI  ",
273                 .oem_table_id   = "HIP05   ",
274                 .oem_revision   = 0,
275         },
276         {
277                 .oem_id         = "HISI  ",
278                 .oem_table_id   = "HIP06   ",
279                 .oem_revision   = 0,
280         },
281         {
282                 .oem_id         = "HISI  ",
283                 .oem_table_id   = "HIP07   ",
284                 .oem_revision   = 0,
285         },
286         { /* Sentinel indicating the end of the OEM array */ },
287 };
288 #endif
289
290 #ifdef CONFIG_ARM64_ERRATUM_858921
291 static u64 notrace arm64_858921_read_cntvct_el0(void)
292 {
293         u64 old, new;
294
295         old = read_sysreg(cntvct_el0);
296         new = read_sysreg(cntvct_el0);
297         return (((old ^ new) >> 32) & 1) ? old : new;
298 }
299 #endif
300
301 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
302 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *,
303                timer_unstable_counter_workaround);
304 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
305
306 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
307 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
308
309 static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,
310                                                 struct clock_event_device *clk)
311 {
312         unsigned long ctrl;
313         u64 cval = evt + arch_counter_get_cntvct();
314
315         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
316         ctrl |= ARCH_TIMER_CTRL_ENABLE;
317         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
318
319         if (access == ARCH_TIMER_PHYS_ACCESS)
320                 write_sysreg(cval, cntp_cval_el0);
321         else
322                 write_sysreg(cval, cntv_cval_el0);
323
324         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
325 }
326
327 static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt,
328                                             struct clock_event_device *clk)
329 {
330         erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
331         return 0;
332 }
333
334 static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt,
335                                             struct clock_event_device *clk)
336 {
337         erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
338         return 0;
339 }
340
341 static const struct arch_timer_erratum_workaround ool_workarounds[] = {
342 #ifdef CONFIG_FSL_ERRATUM_A008585
343         {
344                 .match_type = ate_match_dt,
345                 .id = "fsl,erratum-a008585",
346                 .desc = "Freescale erratum a005858",
347                 .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,
348                 .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,
349                 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
350                 .set_next_event_phys = erratum_set_next_event_tval_phys,
351                 .set_next_event_virt = erratum_set_next_event_tval_virt,
352         },
353 #endif
354 #ifdef CONFIG_HISILICON_ERRATUM_161010101
355         {
356                 .match_type = ate_match_dt,
357                 .id = "hisilicon,erratum-161010101",
358                 .desc = "HiSilicon erratum 161010101",
359                 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
360                 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
361                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
362                 .set_next_event_phys = erratum_set_next_event_tval_phys,
363                 .set_next_event_virt = erratum_set_next_event_tval_virt,
364         },
365         {
366                 .match_type = ate_match_acpi_oem_info,
367                 .id = hisi_161010101_oem_info,
368                 .desc = "HiSilicon erratum 161010101",
369                 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
370                 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
371                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
372                 .set_next_event_phys = erratum_set_next_event_tval_phys,
373                 .set_next_event_virt = erratum_set_next_event_tval_virt,
374         },
375 #endif
376 #ifdef CONFIG_ARM64_ERRATUM_858921
377         {
378                 .match_type = ate_match_local_cap_id,
379                 .id = (void *)ARM64_WORKAROUND_858921,
380                 .desc = "ARM erratum 858921",
381                 .read_cntvct_el0 = arm64_858921_read_cntvct_el0,
382         },
383 #endif
384 };
385
386 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
387                                const void *);
388
389 static
390 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
391                                  const void *arg)
392 {
393         const struct device_node *np = arg;
394
395         return of_property_read_bool(np, wa->id);
396 }
397
398 static
399 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
400                                         const void *arg)
401 {
402         return this_cpu_has_cap((uintptr_t)wa->id);
403 }
404
405
406 static
407 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
408                                        const void *arg)
409 {
410         static const struct ate_acpi_oem_info empty_oem_info = {};
411         const struct ate_acpi_oem_info *info = wa->id;
412         const struct acpi_table_header *table = arg;
413
414         /* Iterate over the ACPI OEM info array, looking for a match */
415         while (memcmp(info, &empty_oem_info, sizeof(*info))) {
416                 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
417                     !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
418                     info->oem_revision == table->oem_revision)
419                         return true;
420
421                 info++;
422         }
423
424         return false;
425 }
426
427 static const struct arch_timer_erratum_workaround *
428 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
429                           ate_match_fn_t match_fn,
430                           void *arg)
431 {
432         int i;
433
434         for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
435                 if (ool_workarounds[i].match_type != type)
436                         continue;
437
438                 if (match_fn(&ool_workarounds[i], arg))
439                         return &ool_workarounds[i];
440         }
441
442         return NULL;
443 }
444
445 static
446 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
447                                   bool local)
448 {
449         int i;
450
451         if (local) {
452                 __this_cpu_write(timer_unstable_counter_workaround, wa);
453         } else {
454                 for_each_possible_cpu(i)
455                         per_cpu(timer_unstable_counter_workaround, i) = wa;
456         }
457
458         static_branch_enable(&arch_timer_read_ool_enabled);
459
460         /*
461          * Don't use the vdso fastpath if errata require using the
462          * out-of-line counter accessor. We may change our mind pretty
463          * late in the game (with a per-CPU erratum, for example), so
464          * change both the default value and the vdso itself.
465          */
466         if (wa->read_cntvct_el0) {
467                 clocksource_counter.archdata.vdso_direct = false;
468                 vdso_default = false;
469         }
470 }
471
472 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
473                                             void *arg)
474 {
475         const struct arch_timer_erratum_workaround *wa;
476         ate_match_fn_t match_fn = NULL;
477         bool local = false;
478
479         switch (type) {
480         case ate_match_dt:
481                 match_fn = arch_timer_check_dt_erratum;
482                 break;
483         case ate_match_local_cap_id:
484                 match_fn = arch_timer_check_local_cap_erratum;
485                 local = true;
486                 break;
487         case ate_match_acpi_oem_info:
488                 match_fn = arch_timer_check_acpi_oem_erratum;
489                 break;
490         default:
491                 WARN_ON(1);
492                 return;
493         }
494
495         wa = arch_timer_iterate_errata(type, match_fn, arg);
496         if (!wa)
497                 return;
498
499         if (needs_unstable_timer_counter_workaround()) {
500                 const struct arch_timer_erratum_workaround *__wa;
501                 __wa = __this_cpu_read(timer_unstable_counter_workaround);
502                 if (__wa && wa != __wa)
503                         pr_warn("Can't enable workaround for %s (clashes with %s\n)",
504                                 wa->desc, __wa->desc);
505
506                 if (__wa)
507                         return;
508         }
509
510         arch_timer_enable_workaround(wa, local);
511         pr_info("Enabling %s workaround for %s\n",
512                 local ? "local" : "global", wa->desc);
513 }
514
515 #define erratum_handler(fn, r, ...)                                     \
516 ({                                                                      \
517         bool __val;                                                     \
518         if (needs_unstable_timer_counter_workaround()) {                \
519                 const struct arch_timer_erratum_workaround *__wa;       \
520                 __wa = __this_cpu_read(timer_unstable_counter_workaround); \
521                 if (__wa && __wa->fn) {                                 \
522                         r = __wa->fn(__VA_ARGS__);                      \
523                         __val = true;                                   \
524                 } else {                                                \
525                         __val = false;                                  \
526                 }                                                       \
527         } else {                                                        \
528                 __val = false;                                          \
529         }                                                               \
530         __val;                                                          \
531 })
532
533 static bool arch_timer_this_cpu_has_cntvct_wa(void)
534 {
535         const struct arch_timer_erratum_workaround *wa;
536
537         wa = __this_cpu_read(timer_unstable_counter_workaround);
538         return wa && wa->read_cntvct_el0;
539 }
540 #else
541 #define arch_timer_check_ool_workaround(t,a)            do { } while(0)
542 #define erratum_set_next_event_tval_virt(...)           ({BUG(); 0;})
543 #define erratum_set_next_event_tval_phys(...)           ({BUG(); 0;})
544 #define erratum_handler(fn, r, ...)                     ({false;})
545 #define arch_timer_this_cpu_has_cntvct_wa()             ({false;})
546 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
547
548 static __always_inline irqreturn_t timer_handler(const int access,
549                                         struct clock_event_device *evt)
550 {
551         unsigned long ctrl;
552
553         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
554         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
555                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
556                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
557                 evt->event_handler(evt);
558                 return IRQ_HANDLED;
559         }
560
561         return IRQ_NONE;
562 }
563
564 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
565 {
566         struct clock_event_device *evt = dev_id;
567
568         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
569 }
570
571 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
572 {
573         struct clock_event_device *evt = dev_id;
574
575         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
576 }
577
578 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
579 {
580         struct clock_event_device *evt = dev_id;
581
582         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
583 }
584
585 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
586 {
587         struct clock_event_device *evt = dev_id;
588
589         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
590 }
591
592 static __always_inline int timer_shutdown(const int access,
593                                           struct clock_event_device *clk)
594 {
595         unsigned long ctrl;
596
597         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
598         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
599         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
600
601         return 0;
602 }
603
604 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
605 {
606         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
607 }
608
609 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
610 {
611         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
612 }
613
614 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
615 {
616         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
617 }
618
619 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
620 {
621         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
622 }
623
624 static __always_inline void set_next_event(const int access, unsigned long evt,
625                                            struct clock_event_device *clk)
626 {
627         unsigned long ctrl;
628         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
629         ctrl |= ARCH_TIMER_CTRL_ENABLE;
630         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
631         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
632         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
633 }
634
635 static int arch_timer_set_next_event_virt(unsigned long evt,
636                                           struct clock_event_device *clk)
637 {
638         int ret;
639
640         if (erratum_handler(set_next_event_virt, ret, evt, clk))
641                 return ret;
642
643         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
644         return 0;
645 }
646
647 static int arch_timer_set_next_event_phys(unsigned long evt,
648                                           struct clock_event_device *clk)
649 {
650         int ret;
651
652         if (erratum_handler(set_next_event_phys, ret, evt, clk))
653                 return ret;
654
655         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
656         return 0;
657 }
658
659 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
660                                               struct clock_event_device *clk)
661 {
662         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
663         return 0;
664 }
665
666 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
667                                               struct clock_event_device *clk)
668 {
669         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
670         return 0;
671 }
672
673 static void __arch_timer_setup(unsigned type,
674                                struct clock_event_device *clk)
675 {
676         clk->features = CLOCK_EVT_FEAT_ONESHOT;
677
678         if (type == ARCH_TIMER_TYPE_CP15) {
679                 if (arch_timer_c3stop)
680                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
681                 clk->name = "arch_sys_timer";
682                 clk->rating = 450;
683                 clk->cpumask = cpumask_of(smp_processor_id());
684                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
685                 switch (arch_timer_uses_ppi) {
686                 case ARCH_TIMER_VIRT_PPI:
687                         clk->set_state_shutdown = arch_timer_shutdown_virt;
688                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
689                         clk->set_next_event = arch_timer_set_next_event_virt;
690                         break;
691                 case ARCH_TIMER_PHYS_SECURE_PPI:
692                 case ARCH_TIMER_PHYS_NONSECURE_PPI:
693                 case ARCH_TIMER_HYP_PPI:
694                         clk->set_state_shutdown = arch_timer_shutdown_phys;
695                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
696                         clk->set_next_event = arch_timer_set_next_event_phys;
697                         break;
698                 default:
699                         BUG();
700                 }
701
702                 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
703         } else {
704                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
705                 clk->name = "arch_mem_timer";
706                 clk->rating = 400;
707                 clk->cpumask = cpu_all_mask;
708                 if (arch_timer_mem_use_virtual) {
709                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
710                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
711                         clk->set_next_event =
712                                 arch_timer_set_next_event_virt_mem;
713                 } else {
714                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
715                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
716                         clk->set_next_event =
717                                 arch_timer_set_next_event_phys_mem;
718                 }
719         }
720
721         clk->set_state_shutdown(clk);
722
723         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
724 }
725
726 static void arch_timer_evtstrm_enable(int divider)
727 {
728         u32 cntkctl = arch_timer_get_cntkctl();
729
730         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
731         /* Set the divider and enable virtual event stream */
732         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
733                         | ARCH_TIMER_VIRT_EVT_EN;
734         arch_timer_set_cntkctl(cntkctl);
735         elf_hwcap |= HWCAP_EVTSTRM;
736 #ifdef CONFIG_COMPAT
737         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
738 #endif
739 }
740
741 static void arch_timer_configure_evtstream(void)
742 {
743         int evt_stream_div, pos;
744
745         /* Find the closest power of two to the divisor */
746         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
747         pos = fls(evt_stream_div);
748         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
749                 pos--;
750         /* enable event stream */
751         arch_timer_evtstrm_enable(min(pos, 15));
752 }
753
754 static void arch_counter_set_user_access(void)
755 {
756         u32 cntkctl = arch_timer_get_cntkctl();
757
758         /* Disable user access to the timers and both counters */
759         /* Also disable virtual event stream */
760         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
761                         | ARCH_TIMER_USR_VT_ACCESS_EN
762                         | ARCH_TIMER_USR_VCT_ACCESS_EN
763                         | ARCH_TIMER_VIRT_EVT_EN
764                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
765
766         /*
767          * Enable user access to the virtual counter if it doesn't
768          * need to be workaround. The vdso may have been already
769          * disabled though.
770          */
771         if (arch_timer_this_cpu_has_cntvct_wa())
772                 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
773         else
774                 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
775
776         arch_timer_set_cntkctl(cntkctl);
777 }
778
779 static bool arch_timer_has_nonsecure_ppi(void)
780 {
781         return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
782                 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
783 }
784
785 static u32 check_ppi_trigger(int irq)
786 {
787         u32 flags = irq_get_trigger_type(irq);
788
789         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
790                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
791                 pr_warn("WARNING: Please fix your firmware\n");
792                 flags = IRQF_TRIGGER_LOW;
793         }
794
795         return flags;
796 }
797
798 static int arch_timer_starting_cpu(unsigned int cpu)
799 {
800         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
801         u32 flags;
802
803         __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
804
805         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
806         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
807
808         if (arch_timer_has_nonsecure_ppi()) {
809                 flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
810                 enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
811                                   flags);
812         }
813
814         arch_counter_set_user_access();
815         if (evtstrm_enable)
816                 arch_timer_configure_evtstream();
817
818         return 0;
819 }
820
821 /*
822  * For historical reasons, when probing with DT we use whichever (non-zero)
823  * rate was probed first, and don't verify that others match. If the first node
824  * probed has a clock-frequency property, this overrides the HW register.
825  */
826 static void arch_timer_of_configure_rate(u32 rate, struct device_node *np)
827 {
828         /* Who has more than one independent system counter? */
829         if (arch_timer_rate)
830                 return;
831
832         if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
833                 arch_timer_rate = rate;
834
835         /* Check the timer frequency. */
836         if (arch_timer_rate == 0)
837                 pr_warn("frequency not available\n");
838 }
839
840 static void arch_timer_banner(unsigned type)
841 {
842         pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
843                 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
844                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
845                         " and " : "",
846                 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
847                 (unsigned long)arch_timer_rate / 1000000,
848                 (unsigned long)(arch_timer_rate / 10000) % 100,
849                 type & ARCH_TIMER_TYPE_CP15 ?
850                         (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
851                         "",
852                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
853                 type & ARCH_TIMER_TYPE_MEM ?
854                         arch_timer_mem_use_virtual ? "virt" : "phys" :
855                         "");
856 }
857
858 u32 arch_timer_get_rate(void)
859 {
860         return arch_timer_rate;
861 }
862
863 static u64 arch_counter_get_cntvct_mem(void)
864 {
865         u32 vct_lo, vct_hi, tmp_hi;
866
867         do {
868                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
869                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
870                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
871         } while (vct_hi != tmp_hi);
872
873         return ((u64) vct_hi << 32) | vct_lo;
874 }
875
876 static struct arch_timer_kvm_info arch_timer_kvm_info;
877
878 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
879 {
880         return &arch_timer_kvm_info;
881 }
882
883 static void __init arch_counter_register(unsigned type)
884 {
885         u64 start_count;
886
887         /* Register the CP15 based counter if we have one */
888         if (type & ARCH_TIMER_TYPE_CP15) {
889                 if (IS_ENABLED(CONFIG_ARM64) ||
890                     arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI)
891                         arch_timer_read_counter = arch_counter_get_cntvct;
892                 else
893                         arch_timer_read_counter = arch_counter_get_cntpct;
894
895                 clocksource_counter.archdata.vdso_direct = vdso_default;
896         } else {
897                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
898         }
899
900         if (!arch_counter_suspend_stop)
901                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
902         start_count = arch_timer_read_counter();
903         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
904         cyclecounter.mult = clocksource_counter.mult;
905         cyclecounter.shift = clocksource_counter.shift;
906         timecounter_init(&arch_timer_kvm_info.timecounter,
907                          &cyclecounter, start_count);
908
909         /* 56 bits minimum, so we assume worst case rollover */
910         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
911 }
912
913 static void arch_timer_stop(struct clock_event_device *clk)
914 {
915         pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
916
917         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
918         if (arch_timer_has_nonsecure_ppi())
919                 disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
920
921         clk->set_state_shutdown(clk);
922 }
923
924 static int arch_timer_dying_cpu(unsigned int cpu)
925 {
926         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
927
928         arch_timer_stop(clk);
929         return 0;
930 }
931
932 #ifdef CONFIG_CPU_PM
933 static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
934 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
935                                     unsigned long action, void *hcpu)
936 {
937         if (action == CPU_PM_ENTER)
938                 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
939         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
940                 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
941         return NOTIFY_OK;
942 }
943
944 static struct notifier_block arch_timer_cpu_pm_notifier = {
945         .notifier_call = arch_timer_cpu_pm_notify,
946 };
947
948 static int __init arch_timer_cpu_pm_init(void)
949 {
950         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
951 }
952
953 static void __init arch_timer_cpu_pm_deinit(void)
954 {
955         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
956 }
957
958 #else
959 static int __init arch_timer_cpu_pm_init(void)
960 {
961         return 0;
962 }
963
964 static void __init arch_timer_cpu_pm_deinit(void)
965 {
966 }
967 #endif
968
969 static int __init arch_timer_register(void)
970 {
971         int err;
972         int ppi;
973
974         arch_timer_evt = alloc_percpu(struct clock_event_device);
975         if (!arch_timer_evt) {
976                 err = -ENOMEM;
977                 goto out;
978         }
979
980         ppi = arch_timer_ppi[arch_timer_uses_ppi];
981         switch (arch_timer_uses_ppi) {
982         case ARCH_TIMER_VIRT_PPI:
983                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
984                                          "arch_timer", arch_timer_evt);
985                 break;
986         case ARCH_TIMER_PHYS_SECURE_PPI:
987         case ARCH_TIMER_PHYS_NONSECURE_PPI:
988                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
989                                          "arch_timer", arch_timer_evt);
990                 if (!err && arch_timer_has_nonsecure_ppi()) {
991                         ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
992                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
993                                                  "arch_timer", arch_timer_evt);
994                         if (err)
995                                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
996                                                 arch_timer_evt);
997                 }
998                 break;
999         case ARCH_TIMER_HYP_PPI:
1000                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1001                                          "arch_timer", arch_timer_evt);
1002                 break;
1003         default:
1004                 BUG();
1005         }
1006
1007         if (err) {
1008                 pr_err("can't register interrupt %d (%d)\n", ppi, err);
1009                 goto out_free;
1010         }
1011
1012         err = arch_timer_cpu_pm_init();
1013         if (err)
1014                 goto out_unreg_notify;
1015
1016
1017         /* Register and immediately configure the timer on the boot CPU */
1018         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1019                                 "clockevents/arm/arch_timer:starting",
1020                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
1021         if (err)
1022                 goto out_unreg_cpupm;
1023         return 0;
1024
1025 out_unreg_cpupm:
1026         arch_timer_cpu_pm_deinit();
1027
1028 out_unreg_notify:
1029         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1030         if (arch_timer_has_nonsecure_ppi())
1031                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1032                                 arch_timer_evt);
1033
1034 out_free:
1035         free_percpu(arch_timer_evt);
1036 out:
1037         return err;
1038 }
1039
1040 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1041 {
1042         int ret;
1043         irq_handler_t func;
1044         struct arch_timer *t;
1045
1046         t = kzalloc(sizeof(*t), GFP_KERNEL);
1047         if (!t)
1048                 return -ENOMEM;
1049
1050         t->base = base;
1051         t->evt.irq = irq;
1052         __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &t->evt);
1053
1054         if (arch_timer_mem_use_virtual)
1055                 func = arch_timer_handler_virt_mem;
1056         else
1057                 func = arch_timer_handler_phys_mem;
1058
1059         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
1060         if (ret) {
1061                 pr_err("Failed to request mem timer irq\n");
1062                 kfree(t);
1063         }
1064
1065         return ret;
1066 }
1067
1068 static const struct of_device_id arch_timer_of_match[] __initconst = {
1069         { .compatible   = "arm,armv7-timer",    },
1070         { .compatible   = "arm,armv8-timer",    },
1071         {},
1072 };
1073
1074 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1075         { .compatible   = "arm,armv7-timer-mem", },
1076         {},
1077 };
1078
1079 static bool __init arch_timer_needs_of_probing(void)
1080 {
1081         struct device_node *dn;
1082         bool needs_probing = false;
1083         unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1084
1085         /* We have two timers, and both device-tree nodes are probed. */
1086         if ((arch_timers_present & mask) == mask)
1087                 return false;
1088
1089         /*
1090          * Only one type of timer is probed,
1091          * check if we have another type of timer node in device-tree.
1092          */
1093         if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1094                 dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1095         else
1096                 dn = of_find_matching_node(NULL, arch_timer_of_match);
1097
1098         if (dn && of_device_is_available(dn))
1099                 needs_probing = true;
1100
1101         of_node_put(dn);
1102
1103         return needs_probing;
1104 }
1105
1106 static int __init arch_timer_common_init(void)
1107 {
1108         arch_timer_banner(arch_timers_present);
1109         arch_counter_register(arch_timers_present);
1110         return arch_timer_arch_init();
1111 }
1112
1113 /**
1114  * arch_timer_select_ppi() - Select suitable PPI for the current system.
1115  *
1116  * If HYP mode is available, we know that the physical timer
1117  * has been configured to be accessible from PL1. Use it, so
1118  * that a guest can use the virtual timer instead.
1119  *
1120  * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1121  * accesses to CNTP_*_EL1 registers are silently redirected to
1122  * their CNTHP_*_EL2 counterparts, and use a different PPI
1123  * number.
1124  *
1125  * If no interrupt provided for virtual timer, we'll have to
1126  * stick to the physical timer. It'd better be accessible...
1127  * For arm64 we never use the secure interrupt.
1128  *
1129  * Return: a suitable PPI type for the current system.
1130  */
1131 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1132 {
1133         if (is_kernel_in_hyp_mode())
1134                 return ARCH_TIMER_HYP_PPI;
1135
1136         if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1137                 return ARCH_TIMER_VIRT_PPI;
1138
1139         if (IS_ENABLED(CONFIG_ARM64))
1140                 return ARCH_TIMER_PHYS_NONSECURE_PPI;
1141
1142         return ARCH_TIMER_PHYS_SECURE_PPI;
1143 }
1144
1145 static int __init arch_timer_of_init(struct device_node *np)
1146 {
1147         int i, ret;
1148         u32 rate;
1149
1150         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1151                 pr_warn("multiple nodes in dt, skipping\n");
1152                 return 0;
1153         }
1154
1155         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1156         for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++)
1157                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
1158
1159         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1160
1161         rate = arch_timer_get_cntfrq();
1162         arch_timer_of_configure_rate(rate, np);
1163
1164         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1165
1166         /* Check for globally applicable workarounds */
1167         arch_timer_check_ool_workaround(ate_match_dt, np);
1168
1169         /*
1170          * If we cannot rely on firmware initializing the timer registers then
1171          * we should use the physical timers instead.
1172          */
1173         if (IS_ENABLED(CONFIG_ARM) &&
1174             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1175                 arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1176         else
1177                 arch_timer_uses_ppi = arch_timer_select_ppi();
1178
1179         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1180                 pr_err("No interrupt available, giving up\n");
1181                 return -EINVAL;
1182         }
1183
1184         /* On some systems, the counter stops ticking when in suspend. */
1185         arch_counter_suspend_stop = of_property_read_bool(np,
1186                                                          "arm,no-tick-in-suspend");
1187
1188         ret = arch_timer_register();
1189         if (ret)
1190                 return ret;
1191
1192         if (arch_timer_needs_of_probing())
1193                 return 0;
1194
1195         return arch_timer_common_init();
1196 }
1197 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1198 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1199
1200 static u32 __init
1201 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1202 {
1203         void __iomem *base;
1204         u32 rate;
1205
1206         base = ioremap(frame->cntbase, frame->size);
1207         if (!base) {
1208                 pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1209                 return 0;
1210         }
1211
1212         rate = readl_relaxed(frame + CNTFRQ);
1213
1214         iounmap(frame);
1215
1216         return rate;
1217 }
1218
1219 static struct arch_timer_mem_frame * __init
1220 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1221 {
1222         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1223         void __iomem *cntctlbase;
1224         u32 cnttidr;
1225         int i;
1226
1227         cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1228         if (!cntctlbase) {
1229                 pr_err("Can't map CNTCTLBase @ %pa\n",
1230                         &timer_mem->cntctlbase);
1231                 return NULL;
1232         }
1233
1234         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1235
1236         /*
1237          * Try to find a virtual capable frame. Otherwise fall back to a
1238          * physical capable frame.
1239          */
1240         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1241                 u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1242                              CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1243
1244                 frame = &timer_mem->frame[i];
1245                 if (!frame->valid)
1246                         continue;
1247
1248                 /* Try enabling everything, and see what sticks */
1249                 writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1250                 cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1251
1252                 if ((cnttidr & CNTTIDR_VIRT(i)) &&
1253                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1254                         best_frame = frame;
1255                         arch_timer_mem_use_virtual = true;
1256                         break;
1257                 }
1258
1259                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1260                         continue;
1261
1262                 best_frame = frame;
1263         }
1264
1265         iounmap(cntctlbase);
1266
1267         if (!best_frame)
1268                 pr_err("Unable to find a suitable frame in timer @ %pa\n",
1269                         &timer_mem->cntctlbase);
1270
1271         return best_frame;
1272 }
1273
1274 static int __init
1275 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1276 {
1277         void __iomem *base;
1278         int ret, irq = 0;
1279
1280         if (arch_timer_mem_use_virtual)
1281                 irq = frame->virt_irq;
1282         else
1283                 irq = frame->phys_irq;
1284
1285         if (!irq) {
1286                 pr_err("Frame missing %s irq.\n",
1287                        arch_timer_mem_use_virtual ? "virt" : "phys");
1288                 return -EINVAL;
1289         }
1290
1291         if (!request_mem_region(frame->cntbase, frame->size,
1292                                 "arch_mem_timer"))
1293                 return -EBUSY;
1294
1295         base = ioremap(frame->cntbase, frame->size);
1296         if (!base) {
1297                 pr_err("Can't map frame's registers\n");
1298                 return -ENXIO;
1299         }
1300
1301         ret = arch_timer_mem_register(base, irq);
1302         if (ret) {
1303                 iounmap(base);
1304                 return ret;
1305         }
1306
1307         arch_counter_base = base;
1308         arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1309
1310         return 0;
1311 }
1312
1313 static int __init arch_timer_mem_of_init(struct device_node *np)
1314 {
1315         struct arch_timer_mem *timer_mem;
1316         struct arch_timer_mem_frame *frame;
1317         struct device_node *frame_node;
1318         struct resource res;
1319         int ret = -EINVAL;
1320         u32 rate;
1321
1322         timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1323         if (!timer_mem)
1324                 return -ENOMEM;
1325
1326         if (of_address_to_resource(np, 0, &res))
1327                 goto out;
1328         timer_mem->cntctlbase = res.start;
1329         timer_mem->size = resource_size(&res);
1330
1331         for_each_available_child_of_node(np, frame_node) {
1332                 u32 n;
1333                 struct arch_timer_mem_frame *frame;
1334
1335                 if (of_property_read_u32(frame_node, "frame-number", &n)) {
1336                         pr_err(FW_BUG "Missing frame-number.\n");
1337                         of_node_put(frame_node);
1338                         goto out;
1339                 }
1340                 if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1341                         pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1342                                ARCH_TIMER_MEM_MAX_FRAMES - 1);
1343                         of_node_put(frame_node);
1344                         goto out;
1345                 }
1346                 frame = &timer_mem->frame[n];
1347
1348                 if (frame->valid) {
1349                         pr_err(FW_BUG "Duplicated frame-number.\n");
1350                         of_node_put(frame_node);
1351                         goto out;
1352                 }
1353
1354                 if (of_address_to_resource(frame_node, 0, &res)) {
1355                         of_node_put(frame_node);
1356                         goto out;
1357                 }
1358                 frame->cntbase = res.start;
1359                 frame->size = resource_size(&res);
1360
1361                 frame->virt_irq = irq_of_parse_and_map(frame_node,
1362                                                        ARCH_TIMER_VIRT_SPI);
1363                 frame->phys_irq = irq_of_parse_and_map(frame_node,
1364                                                        ARCH_TIMER_PHYS_SPI);
1365
1366                 frame->valid = true;
1367         }
1368
1369         frame = arch_timer_mem_find_best_frame(timer_mem);
1370         if (!frame) {
1371                 ret = -EINVAL;
1372                 goto out;
1373         }
1374
1375         rate = arch_timer_mem_frame_get_cntfrq(frame);
1376         arch_timer_of_configure_rate(rate, np);
1377
1378         ret = arch_timer_mem_frame_register(frame);
1379         if (!ret && !arch_timer_needs_of_probing())
1380                 ret = arch_timer_common_init();
1381 out:
1382         kfree(timer_mem);
1383         return ret;
1384 }
1385 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1386                        arch_timer_mem_of_init);
1387
1388 #ifdef CONFIG_ACPI_GTDT
1389 static int __init
1390 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1391 {
1392         struct arch_timer_mem_frame *frame;
1393         u32 rate;
1394         int i;
1395
1396         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1397                 frame = &timer_mem->frame[i];
1398
1399                 if (!frame->valid)
1400                         continue;
1401
1402                 rate = arch_timer_mem_frame_get_cntfrq(frame);
1403                 if (rate == arch_timer_rate)
1404                         continue;
1405
1406                 pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1407                         &frame->cntbase,
1408                         (unsigned long)rate, (unsigned long)arch_timer_rate);
1409
1410                 return -EINVAL;
1411         }
1412
1413         return 0;
1414 }
1415
1416 static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1417 {
1418         struct arch_timer_mem *timers, *timer;
1419         struct arch_timer_mem_frame *frame;
1420         int timer_count, i, ret = 0;
1421
1422         timers = kcalloc(platform_timer_count, sizeof(*timers),
1423                             GFP_KERNEL);
1424         if (!timers)
1425                 return -ENOMEM;
1426
1427         ret = acpi_arch_timer_mem_init(timers, &timer_count);
1428         if (ret || !timer_count)
1429                 goto out;
1430
1431         for (i = 0; i < timer_count; i++) {
1432                 ret = arch_timer_mem_verify_cntfrq(&timers[i]);
1433                 if (ret) {
1434                         pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1435                         goto out;
1436                 }
1437         }
1438
1439         /*
1440          * While unlikely, it's theoretically possible that none of the frames
1441          * in a timer expose the combination of feature we want.
1442          */
1443         for (i = i; i < timer_count; i++) {
1444                 timer = &timers[i];
1445
1446                 frame = arch_timer_mem_find_best_frame(timer);
1447                 if (frame)
1448                         break;
1449         }
1450
1451         if (frame)
1452                 ret = arch_timer_mem_frame_register(frame);
1453 out:
1454         kfree(timers);
1455         return ret;
1456 }
1457
1458 /* Initialize per-processor generic timer and memory-mapped timer(if present) */
1459 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1460 {
1461         int ret, platform_timer_count;
1462
1463         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1464                 pr_warn("already initialized, skipping\n");
1465                 return -EINVAL;
1466         }
1467
1468         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1469
1470         ret = acpi_gtdt_init(table, &platform_timer_count);
1471         if (ret) {
1472                 pr_err("Failed to init GTDT table.\n");
1473                 return ret;
1474         }
1475
1476         arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1477                 acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1478
1479         arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1480                 acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1481
1482         arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1483                 acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1484
1485         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1486
1487         /*
1488          * When probing via ACPI, we have no mechanism to override the sysreg
1489          * CNTFRQ value. This *must* be correct.
1490          */
1491         arch_timer_rate = arch_timer_get_cntfrq();
1492         if (!arch_timer_rate) {
1493                 pr_err(FW_BUG "frequency not available.\n");
1494                 return -EINVAL;
1495         }
1496
1497         arch_timer_uses_ppi = arch_timer_select_ppi();
1498         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1499                 pr_err("No interrupt available, giving up\n");
1500                 return -EINVAL;
1501         }
1502
1503         /* Always-on capability */
1504         arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1505
1506         /* Check for globally applicable workarounds */
1507         arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1508
1509         ret = arch_timer_register();
1510         if (ret)
1511                 return ret;
1512
1513         if (platform_timer_count &&
1514             arch_timer_mem_acpi_init(platform_timer_count))
1515                 pr_err("Failed to initialize memory-mapped timer.\n");
1516
1517         return arch_timer_common_init();
1518 }
1519 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1520 #endif