drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11
  12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
  13
  14 #include <linux/init.h>
  15 #include <linux/kernel.h>
  16 #include <linux/device.h>
  17 #include <linux/smp.h>
  18 #include <linux/cpu.h>
  19 #include <linux/cpu_pm.h>
  20 #include <linux/clockchips.h>
  21 #include <linux/clocksource.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/of_irq.h>
  24 #include <linux/of_address.h>
  25 #include <linux/io.h>
  26 #include <linux/slab.h>
  27 #include <linux/sched/clock.h>
  28 #include <linux/sched_clock.h>
  29 #include <linux/acpi.h>
  30
  31 #include <asm/arch_timer.h>
  32 #include <asm/virt.h>
  33
  34 #include <clocksource/arm_arch_timer.h>
  35
  36 #undef pr_fmt
  37 #define pr_fmt(fmt) "arch_timer: " fmt
  38
  39 #define CNTTIDR         0x08
  40 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  41
  42 #define CNTACR(n)       (0x40 + ((n) * 4))
  43 #define CNTACR_RPCT     BIT(0)
  44 #define CNTACR_RVCT     BIT(1)
  45 #define CNTACR_RFRQ     BIT(2)
  46 #define CNTACR_RVOFF    BIT(3)
  47 #define CNTACR_RWVT     BIT(4)
  48 #define CNTACR_RWPT     BIT(5)
  49
  50 #define CNTVCT_LO       0x08
  51 #define CNTVCT_HI       0x0c
  52 #define CNTFRQ          0x10
  53 #define CNTP_TVAL       0x28
  54 #define CNTP_CTL        0x2c
  55 #define CNTV_TVAL       0x38
  56 #define CNTV_CTL        0x3c
  57
  58 static unsigned arch_timers_present __initdata;
  59
  60 static void __iomem *arch_counter_base;
  61
  62 struct arch_timer {
  63         void __iomem *base;
  64         struct clock_event_device evt;
  65 };
  66
  67 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  68
  69 static u32 arch_timer_rate;
  70 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI];
  71
  72 static struct clock_event_device __percpu *arch_timer_evt;
  73
  74 static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI;
  75 static bool arch_timer_c3stop;
  76 static bool arch_timer_mem_use_virtual;
  77 static bool arch_counter_suspend_stop;
  78 static bool vdso_default = true;
  79
  80 static cpumask_t evtstrm_available = CPU_MASK_NONE;
  81 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  82
  83 static int __init early_evtstrm_cfg(char *buf)
  84 {
  85         return strtobool(buf, &evtstrm_enable);
  86 }
  87 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
  88
  89 /*
  90  * Architected system timer support.
  91  */
  92
  93 static __always_inline
  94 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
  95                           struct clock_event_device *clk)
  96 {
  97         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
  98                 struct arch_timer *timer = to_arch_timer(clk);
  99                 switch (reg) {
 100                 case ARCH_TIMER_REG_CTRL:
 101                         writel_relaxed(val, timer->base + CNTP_CTL);
 102                         break;
 103                 case ARCH_TIMER_REG_TVAL:
 104                         writel_relaxed(val, timer->base + CNTP_TVAL);
 105                         break;
 106                 }
 107         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 108                 struct arch_timer *timer = to_arch_timer(clk);
 109                 switch (reg) {
 110                 case ARCH_TIMER_REG_CTRL:
 111                         writel_relaxed(val, timer->base + CNTV_CTL);
 112                         break;
 113                 case ARCH_TIMER_REG_TVAL:
 114                         writel_relaxed(val, timer->base + CNTV_TVAL);
 115                         break;
 116                 }
 117         } else {
 118                 arch_timer_reg_write_cp15(access, reg, val);
 119         }
 120 }
 121
 122 static __always_inline
 123 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 124                         struct clock_event_device *clk)
 125 {
 126         u32 val;
 127
 128         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 129                 struct arch_timer *timer = to_arch_timer(clk);
 130                 switch (reg) {
 131                 case ARCH_TIMER_REG_CTRL:
 132                         val = readl_relaxed(timer->base + CNTP_CTL);
 133                         break;
 134                 case ARCH_TIMER_REG_TVAL:
 135                         val = readl_relaxed(timer->base + CNTP_TVAL);
 136                         break;
 137                 }
 138         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 139                 struct arch_timer *timer = to_arch_timer(clk);
 140                 switch (reg) {
 141                 case ARCH_TIMER_REG_CTRL:
 142                         val = readl_relaxed(timer->base + CNTV_CTL);
 143                         break;
 144                 case ARCH_TIMER_REG_TVAL:
 145                         val = readl_relaxed(timer->base + CNTV_TVAL);
 146                         break;
 147                 }
 148         } else {
 149                 val = arch_timer_reg_read_cp15(access, reg);
 150         }
 151
 152         return val;
 153 }
 154
 155 /*
 156  * Default to cp15 based access because arm64 uses this function for
 157  * sched_clock() before DT is probed and the cp15 method is guaranteed
 158  * to exist on arm64. arm doesn't use this before DT is probed so even
 159  * if we don't have the cp15 accessors we won't have a problem.
 160  */
 161 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 162 EXPORT_SYMBOL_GPL(arch_timer_read_counter);
 163
 164 static u64 arch_counter_read(struct clocksource *cs)
 165 {
 166         return arch_timer_read_counter();
 167 }
 168
 169 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
 170 {
 171         return arch_timer_read_counter();
 172 }
 173
 174 static struct clocksource clocksource_counter = {
 175         .name   = "arch_sys_counter",
 176         .rating = 400,
 177         .read   = arch_counter_read,
 178         .mask   = CLOCKSOURCE_MASK(56),
 179         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
 180 };
 181
 182 static struct cyclecounter cyclecounter __ro_after_init = {
 183         .read   = arch_counter_read_cc,
 184         .mask   = CLOCKSOURCE_MASK(56),
 185 };
 186
 187 struct ate_acpi_oem_info {
 188         char oem_id[ACPI_OEM_ID_SIZE + 1];
 189         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
 190         u32 oem_revision;
 191 };
 192
 193 #ifdef CONFIG_FSL_ERRATUM_A008585
 194 /*
 195  * The number of retries is an arbitrary value well beyond the highest number
 196  * of iterations the loop has been observed to take.
 197  */
 198 #define __fsl_a008585_read_reg(reg) ({                  \
 199         u64 _old, _new;                                 \
 200         int _retries = 200;                             \
 201                                                         \
 202         do {                                            \
 203                 _old = read_sysreg(reg);                \
 204                 _new = read_sysreg(reg);                \
 205                 _retries--;                             \
 206         } while (unlikely(_old != _new) && _retries);   \
 207                                                         \
 208         WARN_ON_ONCE(!_retries);                        \
 209         _new;                                           \
 210 })
 211
 212 static u32 notrace fsl_a008585_read_cntp_tval_el0(void)
 213 {
 214         return __fsl_a008585_read_reg(cntp_tval_el0);
 215 }
 216
 217 static u32 notrace fsl_a008585_read_cntv_tval_el0(void)
 218 {
 219         return __fsl_a008585_read_reg(cntv_tval_el0);
 220 }
 221
 222 static u64 notrace fsl_a008585_read_cntpct_el0(void)
 223 {
 224         return __fsl_a008585_read_reg(cntpct_el0);
 225 }
 226
 227 static u64 notrace fsl_a008585_read_cntvct_el0(void)
 228 {
 229         return __fsl_a008585_read_reg(cntvct_el0);
 230 }
 231 #endif
 232
 233 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 234 /*
 235  * Verify whether the value of the second read is larger than the first by
 236  * less than 32 is the only way to confirm the value is correct, so clear the
 237  * lower 5 bits to check whether the difference is greater than 32 or not.
 238  * Theoretically the erratum should not occur more than twice in succession
 239  * when reading the system counter, but it is possible that some interrupts
 240  * may lead to more than twice read errors, triggering the warning, so setting
 241  * the number of retries far beyond the number of iterations the loop has been
 242  * observed to take.
 243  */
 244 #define __hisi_161010101_read_reg(reg) ({                               \
 245         u64 _old, _new;                                         \
 246         int _retries = 50;                                      \
 247                                                                 \
 248         do {                                                    \
 249                 _old = read_sysreg(reg);                        \
 250                 _new = read_sysreg(reg);                        \
 251                 _retries--;                                     \
 252         } while (unlikely((_new - _old) >> 5) && _retries);     \
 253                                                                 \
 254         WARN_ON_ONCE(!_retries);                                \
 255         _new;                                                   \
 256 })
 257
 258 static u32 notrace hisi_161010101_read_cntp_tval_el0(void)
 259 {
 260         return __hisi_161010101_read_reg(cntp_tval_el0);
 261 }
 262
 263 static u32 notrace hisi_161010101_read_cntv_tval_el0(void)
 264 {
 265         return __hisi_161010101_read_reg(cntv_tval_el0);
 266 }
 267
 268 static u64 notrace hisi_161010101_read_cntpct_el0(void)
 269 {
 270         return __hisi_161010101_read_reg(cntpct_el0);
 271 }
 272
 273 static u64 notrace hisi_161010101_read_cntvct_el0(void)
 274 {
 275         return __hisi_161010101_read_reg(cntvct_el0);
 276 }
 277
 278 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
 279         /*
 280          * Note that trailing spaces are required to properly match
 281          * the OEM table information.
 282          */
 283         {
 284                 .oem_id         = "HISI  ",
 285                 .oem_table_id   = "HIP05   ",
 286                 .oem_revision   = 0,
 287         },
 288         {
 289                 .oem_id         = "HISI  ",
 290                 .oem_table_id   = "HIP06   ",
 291                 .oem_revision   = 0,
 292         },
 293         {
 294                 .oem_id         = "HISI  ",
 295                 .oem_table_id   = "HIP07   ",
 296                 .oem_revision   = 0,
 297         },
 298         { /* Sentinel indicating the end of the OEM array */ },
 299 };
 300 #endif
 301
 302 #ifdef CONFIG_ARM64_ERRATUM_858921
 303 static u64 notrace arm64_858921_read_cntpct_el0(void)
 304 {
 305         u64 old, new;
 306
 307         old = read_sysreg(cntpct_el0);
 308         new = read_sysreg(cntpct_el0);
 309         return (((old ^ new) >> 32) & 1) ? old : new;
 310 }
 311
 312 static u64 notrace arm64_858921_read_cntvct_el0(void)
 313 {
 314         u64 old, new;
 315
 316         old = read_sysreg(cntvct_el0);
 317         new = read_sysreg(cntvct_el0);
 318         return (((old ^ new) >> 32) & 1) ? old : new;
 319 }
 320 #endif
 321
 322 #ifdef CONFIG_ARM64_ERRATUM_1188873
 323 static u64 notrace arm64_1188873_read_cntvct_el0(void)
 324 {
 325         return read_sysreg(cntvct_el0);
 326 }
 327 #endif
 328
 329 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 330 /*
 331  * The low bits of the counter registers are indeterminate while bit 10 or
 332  * greater is rolling over. Since the counter value can jump both backward
 333  * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values
 334  * with all ones or all zeros in the low bits. Bound the loop by the maximum
 335  * number of CPU cycles in 3 consecutive 24 MHz counter periods.
 336  */
 337 #define __sun50i_a64_read_reg(reg) ({                                   \
 338         u64 _val;                                                       \
 339         int _retries = 150;                                             \
 340                                                                         \
 341         do {                                                            \
 342                 _val = read_sysreg(reg);                                \
 343                 _retries--;                                             \
 344         } while (((_val + 1) & GENMASK(9, 0)) <= 1 && _retries);        \
 345                                                                         \
 346         WARN_ON_ONCE(!_retries);                                        \
 347         _val;                                                           \
 348 })
 349
 350 static u64 notrace sun50i_a64_read_cntpct_el0(void)
 351 {
 352         return __sun50i_a64_read_reg(cntpct_el0);
 353 }
 354
 355 static u64 notrace sun50i_a64_read_cntvct_el0(void)
 356 {
 357         return __sun50i_a64_read_reg(cntvct_el0);
 358 }
 359
 360 static u32 notrace sun50i_a64_read_cntp_tval_el0(void)
 361 {
 362         return read_sysreg(cntp_cval_el0) - sun50i_a64_read_cntpct_el0();
 363 }
 364
 365 static u32 notrace sun50i_a64_read_cntv_tval_el0(void)
 366 {
 367         return read_sysreg(cntv_cval_el0) - sun50i_a64_read_cntvct_el0();
 368 }
 369 #endif
 370
 371 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
 372 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
 373 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
 374
 375 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
 376 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
 377
 378 static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,
 379                                                 struct clock_event_device *clk)
 380 {
 381         unsigned long ctrl;
 382         u64 cval;
 383
 384         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 385         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 386         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 387
 388         if (access == ARCH_TIMER_PHYS_ACCESS) {
 389                 cval = evt + arch_counter_get_cntpct();
 390                 write_sysreg(cval, cntp_cval_el0);
 391         } else {
 392                 cval = evt + arch_counter_get_cntvct();
 393                 write_sysreg(cval, cntv_cval_el0);
 394         }
 395
 396         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 397 }
 398
 399 static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt,
 400                                             struct clock_event_device *clk)
 401 {
 402         erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 403         return 0;
 404 }
 405
 406 static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt,
 407                                             struct clock_event_device *clk)
 408 {
 409         erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 410         return 0;
 411 }
 412
 413 static const struct arch_timer_erratum_workaround ool_workarounds[] = {
 414 #ifdef CONFIG_FSL_ERRATUM_A008585
 415         {
 416                 .match_type = ate_match_dt,
 417                 .id = "fsl,erratum-a008585",
 418                 .desc = "Freescale erratum a005858",
 419                 .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,
 420                 .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,
 421                 .read_cntpct_el0 = fsl_a008585_read_cntpct_el0,
 422                 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
 423                 .set_next_event_phys = erratum_set_next_event_tval_phys,
 424                 .set_next_event_virt = erratum_set_next_event_tval_virt,
 425         },
 426 #endif
 427 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 428         {
 429                 .match_type = ate_match_dt,
 430                 .id = "hisilicon,erratum-161010101",
 431                 .desc = "HiSilicon erratum 161010101",
 432                 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
 433                 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
 434                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 435                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 436                 .set_next_event_phys = erratum_set_next_event_tval_phys,
 437                 .set_next_event_virt = erratum_set_next_event_tval_virt,
 438         },
 439         {
 440                 .match_type = ate_match_acpi_oem_info,
 441                 .id = hisi_161010101_oem_info,
 442                 .desc = "HiSilicon erratum 161010101",
 443                 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
 444                 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
 445                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 446                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 447                 .set_next_event_phys = erratum_set_next_event_tval_phys,
 448                 .set_next_event_virt = erratum_set_next_event_tval_virt,
 449         },
 450 #endif
 451 #ifdef CONFIG_ARM64_ERRATUM_858921
 452         {
 453                 .match_type = ate_match_local_cap_id,
 454                 .id = (void *)ARM64_WORKAROUND_858921,
 455                 .desc = "ARM erratum 858921",
 456                 .read_cntpct_el0 = arm64_858921_read_cntpct_el0,
 457                 .read_cntvct_el0 = arm64_858921_read_cntvct_el0,
 458         },
 459 #endif
 460 #ifdef CONFIG_ARM64_ERRATUM_1188873
 461         {
 462                 .match_type = ate_match_local_cap_id,
 463                 .id = (void *)ARM64_WORKAROUND_1188873,
 464                 .desc = "ARM erratum 1188873",
 465                 .read_cntvct_el0 = arm64_1188873_read_cntvct_el0,
 466         },
 467 #endif
 468 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 469         {
 470                 .match_type = ate_match_dt,
 471                 .id = "allwinner,erratum-unknown1",
 472                 .desc = "Allwinner erratum UNKNOWN1",
 473                 .read_cntp_tval_el0 = sun50i_a64_read_cntp_tval_el0,
 474                 .read_cntv_tval_el0 = sun50i_a64_read_cntv_tval_el0,
 475                 .read_cntpct_el0 = sun50i_a64_read_cntpct_el0,
 476                 .read_cntvct_el0 = sun50i_a64_read_cntvct_el0,
 477                 .set_next_event_phys = erratum_set_next_event_tval_phys,
 478                 .set_next_event_virt = erratum_set_next_event_tval_virt,
 479         },
 480 #endif
 481 };
 482
 483 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
 484                                const void *);
 485
 486 static
 487 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
 488                                  const void *arg)
 489 {
 490         const struct device_node *np = arg;
 491
 492         return of_property_read_bool(np, wa->id);
 493 }
 494
 495 static
 496 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
 497                                         const void *arg)
 498 {
 499         return this_cpu_has_cap((uintptr_t)wa->id);
 500 }
 501
 502
 503 static
 504 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
 505                                        const void *arg)
 506 {
 507         static const struct ate_acpi_oem_info empty_oem_info = {};
 508         const struct ate_acpi_oem_info *info = wa->id;
 509         const struct acpi_table_header *table = arg;
 510
 511         /* Iterate over the ACPI OEM info array, looking for a match */
 512         while (memcmp(info, &empty_oem_info, sizeof(*info))) {
 513                 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
 514                     !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
 515                     info->oem_revision == table->oem_revision)
 516                         return true;
 517
 518                 info++;
 519         }
 520
 521         return false;
 522 }
 523
 524 static const struct arch_timer_erratum_workaround *
 525 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
 526                           ate_match_fn_t match_fn,
 527                           void *arg)
 528 {
 529         int i;
 530
 531         for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
 532                 if (ool_workarounds[i].match_type != type)
 533                         continue;
 534
 535                 if (match_fn(&ool_workarounds[i], arg))
 536                         return &ool_workarounds[i];
 537         }
 538
 539         return NULL;
 540 }
 541
 542 static
 543 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
 544                                   bool local)
 545 {
 546         int i;
 547
 548         if (local) {
 549                 __this_cpu_write(timer_unstable_counter_workaround, wa);
 550         } else {
 551                 for_each_possible_cpu(i)
 552                         per_cpu(timer_unstable_counter_workaround, i) = wa;
 553         }
 554
 555         /*
 556          * Use the locked version, as we're called from the CPU
 557          * hotplug framework. Otherwise, we end-up in deadlock-land.
 558          */
 559         static_branch_enable_cpuslocked(&arch_timer_read_ool_enabled);
 560
 561         /*
 562          * Don't use the vdso fastpath if errata require using the
 563          * out-of-line counter accessor. We may change our mind pretty
 564          * late in the game (with a per-CPU erratum, for example), so
 565          * change both the default value and the vdso itself.
 566          */
 567         if (wa->read_cntvct_el0) {
 568                 clocksource_counter.archdata.vdso_direct = false;
 569                 vdso_default = false;
 570         }
 571 }
 572
 573 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
 574                                             void *arg)
 575 {
 576         const struct arch_timer_erratum_workaround *wa;
 577         ate_match_fn_t match_fn = NULL;
 578         bool local = false;
 579
 580         switch (type) {
 581         case ate_match_dt:
 582                 match_fn = arch_timer_check_dt_erratum;
 583                 break;
 584         case ate_match_local_cap_id:
 585                 match_fn = arch_timer_check_local_cap_erratum;
 586                 local = true;
 587                 break;
 588         case ate_match_acpi_oem_info:
 589                 match_fn = arch_timer_check_acpi_oem_erratum;
 590                 break;
 591         default:
 592                 WARN_ON(1);
 593                 return;
 594         }
 595
 596         wa = arch_timer_iterate_errata(type, match_fn, arg);
 597         if (!wa)
 598                 return;
 599
 600         if (needs_unstable_timer_counter_workaround()) {
 601                 const struct arch_timer_erratum_workaround *__wa;
 602                 __wa = __this_cpu_read(timer_unstable_counter_workaround);
 603                 if (__wa && wa != __wa)
 604                         pr_warn("Can't enable workaround for %s (clashes with %s\n)",
 605                                 wa->desc, __wa->desc);
 606
 607                 if (__wa)
 608                         return;
 609         }
 610
 611         arch_timer_enable_workaround(wa, local);
 612         pr_info("Enabling %s workaround for %s\n",
 613                 local ? "local" : "global", wa->desc);
 614 }
 615
 616 #define erratum_handler(fn, r, ...)                                     \
 617 ({                                                                      \
 618         bool __val;                                                     \
 619         if (needs_unstable_timer_counter_workaround()) {                \
 620                 const struct arch_timer_erratum_workaround *__wa;       \
 621                 __wa = __this_cpu_read(timer_unstable_counter_workaround); \
 622                 if (__wa && __wa->fn) {                                 \
 623                         r = __wa->fn(__VA_ARGS__);                      \
 624                         __val = true;                                   \
 625                 } else {                                                \
 626                         __val = false;                                  \
 627                 }                                                       \
 628         } else {                                                        \
 629                 __val = false;                                          \
 630         }                                                               \
 631         __val;                                                          \
 632 })
 633
 634 static bool arch_timer_this_cpu_has_cntvct_wa(void)
 635 {
 636         const struct arch_timer_erratum_workaround *wa;
 637
 638         wa = __this_cpu_read(timer_unstable_counter_workaround);
 639         return wa && wa->read_cntvct_el0;
 640 }
 641 #else
 642 #define arch_timer_check_ool_workaround(t,a)            do { } while(0)
 643 #define erratum_set_next_event_tval_virt(...)           ({BUG(); 0;})
 644 #define erratum_set_next_event_tval_phys(...)           ({BUG(); 0;})
 645 #define erratum_handler(fn, r, ...)                     ({false;})
 646 #define arch_timer_this_cpu_has_cntvct_wa()             ({false;})
 647 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
 648
 649 static __always_inline irqreturn_t timer_handler(const int access,
 650                                         struct clock_event_device *evt)
 651 {
 652         unsigned long ctrl;
 653
 654         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 655         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 656                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 657                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 658                 evt->event_handler(evt);
 659                 return IRQ_HANDLED;
 660         }
 661
 662         return IRQ_NONE;
 663 }
 664
 665 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 666 {
 667         struct clock_event_device *evt = dev_id;
 668
 669         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 670 }
 671
 672 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 673 {
 674         struct clock_event_device *evt = dev_id;
 675
 676         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 677 }
 678
 679 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 680 {
 681         struct clock_event_device *evt = dev_id;
 682
 683         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 684 }
 685
 686 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 687 {
 688         struct clock_event_device *evt = dev_id;
 689
 690         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 691 }
 692
 693 static __always_inline int timer_shutdown(const int access,
 694                                           struct clock_event_device *clk)
 695 {
 696         unsigned long ctrl;
 697
 698         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 699         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 700         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 701
 702         return 0;
 703 }
 704
 705 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 706 {
 707         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 708 }
 709
 710 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 711 {
 712         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 713 }
 714
 715 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 716 {
 717         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 718 }
 719
 720 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 721 {
 722         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 723 }
 724
 725 static __always_inline void set_next_event(const int access, unsigned long evt,
 726                                            struct clock_event_device *clk)
 727 {
 728         unsigned long ctrl;
 729         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 730         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 731         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 732         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 733         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 734 }
 735
 736 static int arch_timer_set_next_event_virt(unsigned long evt,
 737                                           struct clock_event_device *clk)
 738 {
 739         int ret;
 740
 741         if (erratum_handler(set_next_event_virt, ret, evt, clk))
 742                 return ret;
 743
 744         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 745         return 0;
 746 }
 747
 748 static int arch_timer_set_next_event_phys(unsigned long evt,
 749                                           struct clock_event_device *clk)
 750 {
 751         int ret;
 752
 753         if (erratum_handler(set_next_event_phys, ret, evt, clk))
 754                 return ret;
 755
 756         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 757         return 0;
 758 }
 759
 760 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 761                                               struct clock_event_device *clk)
 762 {
 763         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 764         return 0;
 765 }
 766
 767 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 768                                               struct clock_event_device *clk)
 769 {
 770         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 771         return 0;
 772 }
 773
 774 static void __arch_timer_setup(unsigned type,
 775                                struct clock_event_device *clk)
 776 {
 777         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 778
 779         if (type == ARCH_TIMER_TYPE_CP15) {
 780                 if (arch_timer_c3stop)
 781                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 782                 clk->name = "arch_sys_timer";
 783                 clk->rating = 450;
 784                 clk->cpumask = cpumask_of(smp_processor_id());
 785                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 786                 switch (arch_timer_uses_ppi) {
 787                 case ARCH_TIMER_VIRT_PPI:
 788                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 789                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 790                         clk->set_next_event = arch_timer_set_next_event_virt;
 791                         break;
 792                 case ARCH_TIMER_PHYS_SECURE_PPI:
 793                 case ARCH_TIMER_PHYS_NONSECURE_PPI:
 794                 case ARCH_TIMER_HYP_PPI:
 795                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 796                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 797                         clk->set_next_event = arch_timer_set_next_event_phys;
 798                         break;
 799                 default:
 800                         BUG();
 801                 }
 802
 803                 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
 804         } else {
 805                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 806                 clk->name = "arch_mem_timer";
 807                 clk->rating = 400;
 808                 clk->cpumask = cpu_possible_mask;
 809                 if (arch_timer_mem_use_virtual) {
 810                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 811                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 812                         clk->set_next_event =
 813                                 arch_timer_set_next_event_virt_mem;
 814                 } else {
 815                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 816                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 817                         clk->set_next_event =
 818                                 arch_timer_set_next_event_phys_mem;
 819                 }
 820         }
 821
 822         clk->set_state_shutdown(clk);
 823
 824         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 825 }
 826
 827 static void arch_timer_evtstrm_enable(int divider)
 828 {
 829         u32 cntkctl = arch_timer_get_cntkctl();
 830
 831         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 832         /* Set the divider and enable virtual event stream */
 833         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 834                         | ARCH_TIMER_VIRT_EVT_EN;
 835         arch_timer_set_cntkctl(cntkctl);
 836         elf_hwcap |= HWCAP_EVTSTRM;
 837 #ifdef CONFIG_COMPAT
 838         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 839 #endif
 840         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
 841 }
 842
 843 static void arch_timer_configure_evtstream(void)
 844 {
 845         int evt_stream_div, pos;
 846
 847         /* Find the closest power of two to the divisor */
 848         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 849         pos = fls(evt_stream_div);
 850         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 851                 pos--;
 852         /* enable event stream */
 853         arch_timer_evtstrm_enable(min(pos, 15));
 854 }
 855
 856 static void arch_counter_set_user_access(void)
 857 {
 858         u32 cntkctl = arch_timer_get_cntkctl();
 859
 860         /* Disable user access to the timers and both counters */
 861         /* Also disable virtual event stream */
 862         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 863                         | ARCH_TIMER_USR_VT_ACCESS_EN
 864                         | ARCH_TIMER_USR_VCT_ACCESS_EN
 865                         | ARCH_TIMER_VIRT_EVT_EN
 866                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 867
 868         /*
 869          * Enable user access to the virtual counter if it doesn't
 870          * need to be workaround. The vdso may have been already
 871          * disabled though.
 872          */
 873         if (arch_timer_this_cpu_has_cntvct_wa())
 874                 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
 875         else
 876                 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 877
 878         arch_timer_set_cntkctl(cntkctl);
 879 }
 880
 881 static bool arch_timer_has_nonsecure_ppi(void)
 882 {
 883         return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
 884                 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 885 }
 886
 887 static u32 check_ppi_trigger(int irq)
 888 {
 889         u32 flags = irq_get_trigger_type(irq);
 890
 891         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 892                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 893                 pr_warn("WARNING: Please fix your firmware\n");
 894                 flags = IRQF_TRIGGER_LOW;
 895         }
 896
 897         return flags;
 898 }
 899
 900 static int arch_timer_starting_cpu(unsigned int cpu)
 901 {
 902         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 903         u32 flags;
 904
 905         __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
 906
 907         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
 908         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 909
 910         if (arch_timer_has_nonsecure_ppi()) {
 911                 flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 912                 enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
 913                                   flags);
 914         }
 915
 916         arch_counter_set_user_access();
 917         if (evtstrm_enable)
 918                 arch_timer_configure_evtstream();
 919
 920         return 0;
 921 }
 922
 923 /*
 924  * For historical reasons, when probing with DT we use whichever (non-zero)
 925  * rate was probed first, and don't verify that others match. If the first node
 926  * probed has a clock-frequency property, this overrides the HW register.
 927  */
 928 static void arch_timer_of_configure_rate(u32 rate, struct device_node *np)
 929 {
 930         /* Who has more than one independent system counter? */
 931         if (arch_timer_rate)
 932                 return;
 933
 934         if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
 935                 arch_timer_rate = rate;
 936
 937         /* Check the timer frequency. */
 938         if (arch_timer_rate == 0)
 939                 pr_warn("frequency not available\n");
 940 }
 941
 942 static void arch_timer_banner(unsigned type)
 943 {
 944         pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 945                 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
 946                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
 947                         " and " : "",
 948                 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
 949                 (unsigned long)arch_timer_rate / 1000000,
 950                 (unsigned long)(arch_timer_rate / 10000) % 100,
 951                 type & ARCH_TIMER_TYPE_CP15 ?
 952                         (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
 953                         "",
 954                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
 955                 type & ARCH_TIMER_TYPE_MEM ?
 956                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 957                         "");
 958 }
 959
 960 u32 arch_timer_get_rate(void)
 961 {
 962         return arch_timer_rate;
 963 }
 964
 965 bool arch_timer_evtstrm_available(void)
 966 {
 967         /*
 968          * We might get called from a preemptible context. This is fine
 969          * because availability of the event stream should be always the same
 970          * for a preemptible context and context where we might resume a task.
 971          */
 972         return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
 973 }
 974
 975 static u64 arch_counter_get_cntvct_mem(void)
 976 {
 977         u32 vct_lo, vct_hi, tmp_hi;
 978
 979         do {
 980                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 981                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 982                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 983         } while (vct_hi != tmp_hi);
 984
 985         return ((u64) vct_hi << 32) | vct_lo;
 986 }
 987
 988 static struct arch_timer_kvm_info arch_timer_kvm_info;
 989
 990 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
 991 {
 992         return &arch_timer_kvm_info;
 993 }
 994
 995 static void __init arch_counter_register(unsigned type)
 996 {
 997         u64 start_count;
 998
 999         /* Register the CP15 based counter if we have one */
1000         if (type & ARCH_TIMER_TYPE_CP15) {
1001                 if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
1002                     arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI)
1003                         arch_timer_read_counter = arch_counter_get_cntvct;
1004                 else
1005                         arch_timer_read_counter = arch_counter_get_cntpct;
1006
1007                 clocksource_counter.archdata.vdso_direct = vdso_default;
1008         } else {
1009                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
1010         }
1011
1012         if (!arch_counter_suspend_stop)
1013                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
1014         start_count = arch_timer_read_counter();
1015         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
1016         cyclecounter.mult = clocksource_counter.mult;
1017         cyclecounter.shift = clocksource_counter.shift;
1018         timecounter_init(&arch_timer_kvm_info.timecounter,
1019                          &cyclecounter, start_count);
1020
1021         /* 56 bits minimum, so we assume worst case rollover */
1022         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
1023 }
1024
1025 static void arch_timer_stop(struct clock_event_device *clk)
1026 {
1027         pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
1028
1029         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
1030         if (arch_timer_has_nonsecure_ppi())
1031                 disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1032
1033         clk->set_state_shutdown(clk);
1034 }
1035
1036 static int arch_timer_dying_cpu(unsigned int cpu)
1037 {
1038         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1039
1040         cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1041
1042         arch_timer_stop(clk);
1043         return 0;
1044 }
1045
1046 #ifdef CONFIG_CPU_PM
1047 static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
1048 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
1049                                     unsigned long action, void *hcpu)
1050 {
1051         if (action == CPU_PM_ENTER) {
1052                 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
1053
1054                 cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1055         } else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
1056                 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
1057
1058                 if (elf_hwcap & HWCAP_EVTSTRM)
1059                         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
1060         }
1061         return NOTIFY_OK;
1062 }
1063
1064 static struct notifier_block arch_timer_cpu_pm_notifier = {
1065         .notifier_call = arch_timer_cpu_pm_notify,
1066 };
1067
1068 static int __init arch_timer_cpu_pm_init(void)
1069 {
1070         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
1071 }
1072
1073 static void __init arch_timer_cpu_pm_deinit(void)
1074 {
1075         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
1076 }
1077
1078 #else
1079 static int __init arch_timer_cpu_pm_init(void)
1080 {
1081         return 0;
1082 }
1083
1084 static void __init arch_timer_cpu_pm_deinit(void)
1085 {
1086 }
1087 #endif
1088
1089 static int __init arch_timer_register(void)
1090 {
1091         int err;
1092         int ppi;
1093
1094         arch_timer_evt = alloc_percpu(struct clock_event_device);
1095         if (!arch_timer_evt) {
1096                 err = -ENOMEM;
1097                 goto out;
1098         }
1099
1100         ppi = arch_timer_ppi[arch_timer_uses_ppi];
1101         switch (arch_timer_uses_ppi) {
1102         case ARCH_TIMER_VIRT_PPI:
1103                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
1104                                          "arch_timer", arch_timer_evt);
1105                 break;
1106         case ARCH_TIMER_PHYS_SECURE_PPI:
1107         case ARCH_TIMER_PHYS_NONSECURE_PPI:
1108                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1109                                          "arch_timer", arch_timer_evt);
1110                 if (!err && arch_timer_has_nonsecure_ppi()) {
1111                         ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1112                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
1113                                                  "arch_timer", arch_timer_evt);
1114                         if (err)
1115                                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1116                                                 arch_timer_evt);
1117                 }
1118                 break;
1119         case ARCH_TIMER_HYP_PPI:
1120                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1121                                          "arch_timer", arch_timer_evt);
1122                 break;
1123         default:
1124                 BUG();
1125         }
1126
1127         if (err) {
1128                 pr_err("can't register interrupt %d (%d)\n", ppi, err);
1129                 goto out_free;
1130         }
1131
1132         err = arch_timer_cpu_pm_init();
1133         if (err)
1134                 goto out_unreg_notify;
1135
1136         /* Register and immediately configure the timer on the boot CPU */
1137         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1138                                 "clockevents/arm/arch_timer:starting",
1139                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
1140         if (err)
1141                 goto out_unreg_cpupm;
1142         return 0;
1143
1144 out_unreg_cpupm:
1145         arch_timer_cpu_pm_deinit();
1146
1147 out_unreg_notify:
1148         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1149         if (arch_timer_has_nonsecure_ppi())
1150                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1151                                 arch_timer_evt);
1152
1153 out_free:
1154         free_percpu(arch_timer_evt);
1155 out:
1156         return err;
1157 }
1158
1159 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1160 {
1161         int ret;
1162         irq_handler_t func;
1163         struct arch_timer *t;
1164
1165         t = kzalloc(sizeof(*t), GFP_KERNEL);
1166         if (!t)
1167                 return -ENOMEM;
1168
1169         t->base = base;
1170         t->evt.irq = irq;
1171         __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &t->evt);
1172
1173         if (arch_timer_mem_use_virtual)
1174                 func = arch_timer_handler_virt_mem;
1175         else
1176                 func = arch_timer_handler_phys_mem;
1177
1178         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
1179         if (ret) {
1180                 pr_err("Failed to request mem timer irq\n");
1181                 kfree(t);
1182         }
1183
1184         return ret;
1185 }
1186
1187 static const struct of_device_id arch_timer_of_match[] __initconst = {
1188         { .compatible   = "arm,armv7-timer",    },
1189         { .compatible   = "arm,armv8-timer",    },
1190         {},
1191 };
1192
1193 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1194         { .compatible   = "arm,armv7-timer-mem", },
1195         {},
1196 };
1197
1198 static bool __init arch_timer_needs_of_probing(void)
1199 {
1200         struct device_node *dn;
1201         bool needs_probing = false;
1202         unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1203
1204         /* We have two timers, and both device-tree nodes are probed. */
1205         if ((arch_timers_present & mask) == mask)
1206                 return false;
1207
1208         /*
1209          * Only one type of timer is probed,
1210          * check if we have another type of timer node in device-tree.
1211          */
1212         if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1213                 dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1214         else
1215                 dn = of_find_matching_node(NULL, arch_timer_of_match);
1216
1217         if (dn && of_device_is_available(dn))
1218                 needs_probing = true;
1219
1220         of_node_put(dn);
1221
1222         return needs_probing;
1223 }
1224
1225 static int __init arch_timer_common_init(void)
1226 {
1227         arch_timer_banner(arch_timers_present);
1228         arch_counter_register(arch_timers_present);
1229         return arch_timer_arch_init();
1230 }
1231
1232 /**
1233  * arch_timer_select_ppi() - Select suitable PPI for the current system.
1234  *
1235  * If HYP mode is available, we know that the physical timer
1236  * has been configured to be accessible from PL1. Use it, so
1237  * that a guest can use the virtual timer instead.
1238  *
1239  * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1240  * accesses to CNTP_*_EL1 registers are silently redirected to
1241  * their CNTHP_*_EL2 counterparts, and use a different PPI
1242  * number.
1243  *
1244  * If no interrupt provided for virtual timer, we'll have to
1245  * stick to the physical timer. It'd better be accessible...
1246  * For arm64 we never use the secure interrupt.
1247  *
1248  * Return: a suitable PPI type for the current system.
1249  */
1250 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1251 {
1252         if (is_kernel_in_hyp_mode())
1253                 return ARCH_TIMER_HYP_PPI;
1254
1255         if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1256                 return ARCH_TIMER_VIRT_PPI;
1257
1258         if (IS_ENABLED(CONFIG_ARM64))
1259                 return ARCH_TIMER_PHYS_NONSECURE_PPI;
1260
1261         return ARCH_TIMER_PHYS_SECURE_PPI;
1262 }
1263
1264 static int __init arch_timer_of_init(struct device_node *np)
1265 {
1266         int i, ret;
1267         u32 rate;
1268
1269         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1270                 pr_warn("multiple nodes in dt, skipping\n");
1271                 return 0;
1272         }
1273
1274         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1275         for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++)
1276                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
1277
1278         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1279
1280         rate = arch_timer_get_cntfrq();
1281         arch_timer_of_configure_rate(rate, np);
1282
1283         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1284
1285         /* Check for globally applicable workarounds */
1286         arch_timer_check_ool_workaround(ate_match_dt, np);
1287
1288         /*
1289          * If we cannot rely on firmware initializing the timer registers then
1290          * we should use the physical timers instead.
1291          */
1292         if (IS_ENABLED(CONFIG_ARM) &&
1293             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1294                 arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1295         else
1296                 arch_timer_uses_ppi = arch_timer_select_ppi();
1297
1298         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1299                 pr_err("No interrupt available, giving up\n");
1300                 return -EINVAL;
1301         }
1302
1303         /* On some systems, the counter stops ticking when in suspend. */
1304         arch_counter_suspend_stop = of_property_read_bool(np,
1305                                                          "arm,no-tick-in-suspend");
1306
1307         ret = arch_timer_register();
1308         if (ret)
1309                 return ret;
1310
1311         if (arch_timer_needs_of_probing())
1312                 return 0;
1313
1314         return arch_timer_common_init();
1315 }
1316 TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1317 TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1318
1319 static u32 __init
1320 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1321 {
1322         void __iomem *base;
1323         u32 rate;
1324
1325         base = ioremap(frame->cntbase, frame->size);
1326         if (!base) {
1327                 pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1328                 return 0;
1329         }
1330
1331         rate = readl_relaxed(base + CNTFRQ);
1332
1333         iounmap(base);
1334
1335         return rate;
1336 }
1337
1338 static struct arch_timer_mem_frame * __init
1339 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1340 {
1341         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1342         void __iomem *cntctlbase;
1343         u32 cnttidr;
1344         int i;
1345
1346         cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1347         if (!cntctlbase) {
1348                 pr_err("Can't map CNTCTLBase @ %pa\n",
1349                         &timer_mem->cntctlbase);
1350                 return NULL;
1351         }
1352
1353         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1354
1355         /*
1356          * Try to find a virtual capable frame. Otherwise fall back to a
1357          * physical capable frame.
1358          */
1359         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1360                 u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1361                              CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1362
1363                 frame = &timer_mem->frame[i];
1364                 if (!frame->valid)
1365                         continue;
1366
1367                 /* Try enabling everything, and see what sticks */
1368                 writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1369                 cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1370
1371                 if ((cnttidr & CNTTIDR_VIRT(i)) &&
1372                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1373                         best_frame = frame;
1374                         arch_timer_mem_use_virtual = true;
1375                         break;
1376                 }
1377
1378                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1379                         continue;
1380
1381                 best_frame = frame;
1382         }
1383
1384         iounmap(cntctlbase);
1385
1386         return best_frame;
1387 }
1388
1389 static int __init
1390 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1391 {
1392         void __iomem *base;
1393         int ret, irq = 0;
1394
1395         if (arch_timer_mem_use_virtual)
1396                 irq = frame->virt_irq;
1397         else
1398                 irq = frame->phys_irq;
1399
1400         if (!irq) {
1401                 pr_err("Frame missing %s irq.\n",
1402                        arch_timer_mem_use_virtual ? "virt" : "phys");
1403                 return -EINVAL;
1404         }
1405
1406         if (!request_mem_region(frame->cntbase, frame->size,
1407                                 "arch_mem_timer"))
1408                 return -EBUSY;
1409
1410         base = ioremap(frame->cntbase, frame->size);
1411         if (!base) {
1412                 pr_err("Can't map frame's registers\n");
1413                 return -ENXIO;
1414         }
1415
1416         ret = arch_timer_mem_register(base, irq);
1417         if (ret) {
1418                 iounmap(base);
1419                 return ret;
1420         }
1421
1422         arch_counter_base = base;
1423         arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1424
1425         return 0;
1426 }
1427
1428 static int __init arch_timer_mem_of_init(struct device_node *np)
1429 {
1430         struct arch_timer_mem *timer_mem;
1431         struct arch_timer_mem_frame *frame;
1432         struct device_node *frame_node;
1433         struct resource res;
1434         int ret = -EINVAL;
1435         u32 rate;
1436
1437         timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1438         if (!timer_mem)
1439                 return -ENOMEM;
1440
1441         if (of_address_to_resource(np, 0, &res))
1442                 goto out;
1443         timer_mem->cntctlbase = res.start;
1444         timer_mem->size = resource_size(&res);
1445
1446         for_each_available_child_of_node(np, frame_node) {
1447                 u32 n;
1448                 struct arch_timer_mem_frame *frame;
1449
1450                 if (of_property_read_u32(frame_node, "frame-number", &n)) {
1451                         pr_err(FW_BUG "Missing frame-number.\n");
1452                         of_node_put(frame_node);
1453                         goto out;
1454                 }
1455                 if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1456                         pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1457                                ARCH_TIMER_MEM_MAX_FRAMES - 1);
1458                         of_node_put(frame_node);
1459                         goto out;
1460                 }
1461                 frame = &timer_mem->frame[n];
1462
1463                 if (frame->valid) {
1464                         pr_err(FW_BUG "Duplicated frame-number.\n");
1465                         of_node_put(frame_node);
1466                         goto out;
1467                 }
1468
1469                 if (of_address_to_resource(frame_node, 0, &res)) {
1470                         of_node_put(frame_node);
1471                         goto out;
1472                 }
1473                 frame->cntbase = res.start;
1474                 frame->size = resource_size(&res);
1475
1476                 frame->virt_irq = irq_of_parse_and_map(frame_node,
1477                                                        ARCH_TIMER_VIRT_SPI);
1478                 frame->phys_irq = irq_of_parse_and_map(frame_node,
1479                                                        ARCH_TIMER_PHYS_SPI);
1480
1481                 frame->valid = true;
1482         }
1483
1484         frame = arch_timer_mem_find_best_frame(timer_mem);
1485         if (!frame) {
1486                 pr_err("Unable to find a suitable frame in timer @ %pa\n",
1487                         &timer_mem->cntctlbase);
1488                 ret = -EINVAL;
1489                 goto out;
1490         }
1491
1492         rate = arch_timer_mem_frame_get_cntfrq(frame);
1493         arch_timer_of_configure_rate(rate, np);
1494
1495         ret = arch_timer_mem_frame_register(frame);
1496         if (!ret && !arch_timer_needs_of_probing())
1497                 ret = arch_timer_common_init();
1498 out:
1499         kfree(timer_mem);
1500         return ret;
1501 }
1502 TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1503                        arch_timer_mem_of_init);
1504
1505 #ifdef CONFIG_ACPI_GTDT
1506 static int __init
1507 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1508 {
1509         struct arch_timer_mem_frame *frame;
1510         u32 rate;
1511         int i;
1512
1513         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1514                 frame = &timer_mem->frame[i];
1515
1516                 if (!frame->valid)
1517                         continue;
1518
1519                 rate = arch_timer_mem_frame_get_cntfrq(frame);
1520                 if (rate == arch_timer_rate)
1521                         continue;
1522
1523                 pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1524                         &frame->cntbase,
1525                         (unsigned long)rate, (unsigned long)arch_timer_rate);
1526
1527                 return -EINVAL;
1528         }
1529
1530         return 0;
1531 }
1532
1533 static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1534 {
1535         struct arch_timer_mem *timers, *timer;
1536         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1537         int timer_count, i, ret = 0;
1538
1539         timers = kcalloc(platform_timer_count, sizeof(*timers),
1540                             GFP_KERNEL);
1541         if (!timers)
1542                 return -ENOMEM;
1543
1544         ret = acpi_arch_timer_mem_init(timers, &timer_count);
1545         if (ret || !timer_count)
1546                 goto out;
1547
1548         /*
1549          * While unlikely, it's theoretically possible that none of the frames
1550          * in a timer expose the combination of feature we want.
1551          */
1552         for (i = 0; i < timer_count; i++) {
1553                 timer = &timers[i];
1554
1555                 frame = arch_timer_mem_find_best_frame(timer);
1556                 if (!best_frame)
1557                         best_frame = frame;
1558
1559                 ret = arch_timer_mem_verify_cntfrq(timer);
1560                 if (ret) {
1561                         pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1562                         goto out;
1563                 }
1564
1565                 if (!best_frame) /* implies !frame */
1566                         /*
1567                          * Only complain about missing suitable frames if we
1568                          * haven't already found one in a previous iteration.
1569                          */
1570                         pr_err("Unable to find a suitable frame in timer @ %pa\n",
1571                                 &timer->cntctlbase);
1572         }
1573
1574         if (best_frame)
1575                 ret = arch_timer_mem_frame_register(best_frame);
1576 out:
1577         kfree(timers);
1578         return ret;
1579 }
1580
1581 /* Initialize per-processor generic timer and memory-mapped timer(if present) */
1582 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1583 {
1584         int ret, platform_timer_count;
1585
1586         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1587                 pr_warn("already initialized, skipping\n");
1588                 return -EINVAL;
1589         }
1590
1591         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1592
1593         ret = acpi_gtdt_init(table, &platform_timer_count);
1594         if (ret) {
1595                 pr_err("Failed to init GTDT table.\n");
1596                 return ret;
1597         }
1598
1599         arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1600                 acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1601
1602         arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1603                 acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1604
1605         arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1606                 acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1607
1608         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1609
1610         /*
1611          * When probing via ACPI, we have no mechanism to override the sysreg
1612          * CNTFRQ value. This *must* be correct.
1613          */
1614         arch_timer_rate = arch_timer_get_cntfrq();
1615         if (!arch_timer_rate) {
1616                 pr_err(FW_BUG "frequency not available.\n");
1617                 return -EINVAL;
1618         }
1619
1620         arch_timer_uses_ppi = arch_timer_select_ppi();
1621         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1622                 pr_err("No interrupt available, giving up\n");
1623                 return -EINVAL;
1624         }
1625
1626         /* Always-on capability */
1627         arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1628
1629         /* Check for globally applicable workarounds */
1630         arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1631
1632         ret = arch_timer_register();
1633         if (ret)
1634                 return ret;
1635
1636         if (platform_timer_count &&
1637             arch_timer_mem_acpi_init(platform_timer_count))
1638                 pr_err("Failed to initialize memory-mapped timer.\n");
1639
1640         return arch_timer_common_init();
1641 }
1642 TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1643 #endif