drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11
  12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
  13
  14 #include <linux/init.h>
  15 #include <linux/kernel.h>
  16 #include <linux/device.h>
  17 #include <linux/smp.h>
  18 #include <linux/cpu.h>
  19 #include <linux/cpu_pm.h>
  20 #include <linux/clockchips.h>
  21 #include <linux/clocksource.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/of_irq.h>
  24 #include <linux/of_address.h>
  25 #include <linux/io.h>
  26 #include <linux/slab.h>
  27 #include <linux/sched_clock.h>
  28 #include <linux/acpi.h>
  29
  30 #include <asm/arch_timer.h>
  31 #include <asm/virt.h>
  32
  33 #include <clocksource/arm_arch_timer.h>
  34
  35 #define CNTTIDR         0x08
  36 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  37
  38 #define CNTACR(n)       (0x40 + ((n) * 4))
  39 #define CNTACR_RPCT     BIT(0)
  40 #define CNTACR_RVCT     BIT(1)
  41 #define CNTACR_RFRQ     BIT(2)
  42 #define CNTACR_RVOFF    BIT(3)
  43 #define CNTACR_RWVT     BIT(4)
  44 #define CNTACR_RWPT     BIT(5)
  45
  46 #define CNTVCT_LO       0x08
  47 #define CNTVCT_HI       0x0c
  48 #define CNTFRQ          0x10
  49 #define CNTP_TVAL       0x28
  50 #define CNTP_CTL        0x2c
  51 #define CNTV_TVAL       0x38
  52 #define CNTV_CTL        0x3c
  53
  54 #define ARCH_CP15_TIMER BIT(0)
  55 #define ARCH_MEM_TIMER  BIT(1)
  56 static unsigned arch_timers_present __initdata;
  57
  58 static void __iomem *arch_counter_base;
  59
  60 struct arch_timer {
  61         void __iomem *base;
  62         struct clock_event_device evt;
  63 };
  64
  65 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  66
  67 static u32 arch_timer_rate;
  68
  69 enum ppi_nr {
  70         PHYS_SECURE_PPI,
  71         PHYS_NONSECURE_PPI,
  72         VIRT_PPI,
  73         HYP_PPI,
  74         MAX_TIMER_PPI
  75 };
  76
  77 static int arch_timer_ppi[MAX_TIMER_PPI];
  78
  79 static struct clock_event_device __percpu *arch_timer_evt;
  80
  81 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
  82 static bool arch_timer_c3stop;
  83 static bool arch_timer_mem_use_virtual;
  84 static bool arch_counter_suspend_stop;
  85
  86 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  87
  88 static int __init early_evtstrm_cfg(char *buf)
  89 {
  90         return strtobool(buf, &evtstrm_enable);
  91 }
  92 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
  93
  94 /*
  95  * Architected system timer support.
  96  */
  97
  98 #ifdef CONFIG_FSL_ERRATUM_A008585
  99 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
 100 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
 101
 102 static int fsl_a008585_enable = -1;
 103
 104 u32 __fsl_a008585_read_cntp_tval_el0(void)
 105 {
 106         return __fsl_a008585_read_reg(cntp_tval_el0);
 107 }
 108
 109 u32 __fsl_a008585_read_cntv_tval_el0(void)
 110 {
 111         return __fsl_a008585_read_reg(cntv_tval_el0);
 112 }
 113
 114 u64 __fsl_a008585_read_cntvct_el0(void)
 115 {
 116         return __fsl_a008585_read_reg(cntvct_el0);
 117 }
 118 EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0);
 119 #endif /* CONFIG_FSL_ERRATUM_A008585 */
 120
 121 static __always_inline
 122 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
 123                           struct clock_event_device *clk)
 124 {
 125         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 126                 struct arch_timer *timer = to_arch_timer(clk);
 127                 switch (reg) {
 128                 case ARCH_TIMER_REG_CTRL:
 129                         writel_relaxed(val, timer->base + CNTP_CTL);
 130                         break;
 131                 case ARCH_TIMER_REG_TVAL:
 132                         writel_relaxed(val, timer->base + CNTP_TVAL);
 133                         break;
 134                 }
 135         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 136                 struct arch_timer *timer = to_arch_timer(clk);
 137                 switch (reg) {
 138                 case ARCH_TIMER_REG_CTRL:
 139                         writel_relaxed(val, timer->base + CNTV_CTL);
 140                         break;
 141                 case ARCH_TIMER_REG_TVAL:
 142                         writel_relaxed(val, timer->base + CNTV_TVAL);
 143                         break;
 144                 }
 145         } else {
 146                 arch_timer_reg_write_cp15(access, reg, val);
 147         }
 148 }
 149
 150 static __always_inline
 151 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 152                         struct clock_event_device *clk)
 153 {
 154         u32 val;
 155
 156         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 157                 struct arch_timer *timer = to_arch_timer(clk);
 158                 switch (reg) {
 159                 case ARCH_TIMER_REG_CTRL:
 160                         val = readl_relaxed(timer->base + CNTP_CTL);
 161                         break;
 162                 case ARCH_TIMER_REG_TVAL:
 163                         val = readl_relaxed(timer->base + CNTP_TVAL);
 164                         break;
 165                 }
 166         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 167                 struct arch_timer *timer = to_arch_timer(clk);
 168                 switch (reg) {
 169                 case ARCH_TIMER_REG_CTRL:
 170                         val = readl_relaxed(timer->base + CNTV_CTL);
 171                         break;
 172                 case ARCH_TIMER_REG_TVAL:
 173                         val = readl_relaxed(timer->base + CNTV_TVAL);
 174                         break;
 175                 }
 176         } else {
 177                 val = arch_timer_reg_read_cp15(access, reg);
 178         }
 179
 180         return val;
 181 }
 182
 183 static __always_inline irqreturn_t timer_handler(const int access,
 184                                         struct clock_event_device *evt)
 185 {
 186         unsigned long ctrl;
 187
 188         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 189         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 190                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 191                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 192                 evt->event_handler(evt);
 193                 return IRQ_HANDLED;
 194         }
 195
 196         return IRQ_NONE;
 197 }
 198
 199 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 200 {
 201         struct clock_event_device *evt = dev_id;
 202
 203         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 204 }
 205
 206 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 207 {
 208         struct clock_event_device *evt = dev_id;
 209
 210         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 211 }
 212
 213 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 214 {
 215         struct clock_event_device *evt = dev_id;
 216
 217         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 218 }
 219
 220 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 221 {
 222         struct clock_event_device *evt = dev_id;
 223
 224         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 225 }
 226
 227 static __always_inline int timer_shutdown(const int access,
 228                                           struct clock_event_device *clk)
 229 {
 230         unsigned long ctrl;
 231
 232         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 233         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 234         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 235
 236         return 0;
 237 }
 238
 239 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 240 {
 241         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 242 }
 243
 244 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 245 {
 246         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 247 }
 248
 249 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 250 {
 251         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 252 }
 253
 254 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 255 {
 256         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 257 }
 258
 259 static __always_inline void set_next_event(const int access, unsigned long evt,
 260                                            struct clock_event_device *clk)
 261 {
 262         unsigned long ctrl;
 263         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 264         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 265         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 266         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 267         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 268 }
 269
 270 #ifdef CONFIG_FSL_ERRATUM_A008585
 271 static __always_inline void fsl_a008585_set_next_event(const int access,
 272                 unsigned long evt, struct clock_event_device *clk)
 273 {
 274         unsigned long ctrl;
 275         u64 cval = evt + arch_counter_get_cntvct();
 276
 277         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 278         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 279         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 280
 281         if (access == ARCH_TIMER_PHYS_ACCESS)
 282                 write_sysreg(cval, cntp_cval_el0);
 283         else if (access == ARCH_TIMER_VIRT_ACCESS)
 284                 write_sysreg(cval, cntv_cval_el0);
 285
 286         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 287 }
 288
 289 static int fsl_a008585_set_next_event_virt(unsigned long evt,
 290                                            struct clock_event_device *clk)
 291 {
 292         fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 293         return 0;
 294 }
 295
 296 static int fsl_a008585_set_next_event_phys(unsigned long evt,
 297                                            struct clock_event_device *clk)
 298 {
 299         fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 300         return 0;
 301 }
 302 #endif /* CONFIG_FSL_ERRATUM_A008585 */
 303
 304 static int arch_timer_set_next_event_virt(unsigned long evt,
 305                                           struct clock_event_device *clk)
 306 {
 307         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 308         return 0;
 309 }
 310
 311 static int arch_timer_set_next_event_phys(unsigned long evt,
 312                                           struct clock_event_device *clk)
 313 {
 314         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 315         return 0;
 316 }
 317
 318 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 319                                               struct clock_event_device *clk)
 320 {
 321         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 322         return 0;
 323 }
 324
 325 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 326                                               struct clock_event_device *clk)
 327 {
 328         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 329         return 0;
 330 }
 331
 332 static void fsl_a008585_set_sne(struct clock_event_device *clk)
 333 {
 334 #ifdef CONFIG_FSL_ERRATUM_A008585
 335         if (!static_branch_unlikely(&arch_timer_read_ool_enabled))
 336                 return;
 337
 338         if (arch_timer_uses_ppi == VIRT_PPI)
 339                 clk->set_next_event = fsl_a008585_set_next_event_virt;
 340         else
 341                 clk->set_next_event = fsl_a008585_set_next_event_phys;
 342 #endif
 343 }
 344
 345 static void __arch_timer_setup(unsigned type,
 346                                struct clock_event_device *clk)
 347 {
 348         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 349
 350         if (type == ARCH_CP15_TIMER) {
 351                 if (arch_timer_c3stop)
 352                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 353                 clk->name = "arch_sys_timer";
 354                 clk->rating = 450;
 355                 clk->cpumask = cpumask_of(smp_processor_id());
 356                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 357                 switch (arch_timer_uses_ppi) {
 358                 case VIRT_PPI:
 359                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 360                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 361                         clk->set_next_event = arch_timer_set_next_event_virt;
 362                         break;
 363                 case PHYS_SECURE_PPI:
 364                 case PHYS_NONSECURE_PPI:
 365                 case HYP_PPI:
 366                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 367                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 368                         clk->set_next_event = arch_timer_set_next_event_phys;
 369                         break;
 370                 default:
 371                         BUG();
 372                 }
 373
 374                 fsl_a008585_set_sne(clk);
 375         } else {
 376                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 377                 clk->name = "arch_mem_timer";
 378                 clk->rating = 400;
 379                 clk->cpumask = cpu_all_mask;
 380                 if (arch_timer_mem_use_virtual) {
 381                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 382                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 383                         clk->set_next_event =
 384                                 arch_timer_set_next_event_virt_mem;
 385                 } else {
 386                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 387                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 388                         clk->set_next_event =
 389                                 arch_timer_set_next_event_phys_mem;
 390                 }
 391         }
 392
 393         clk->set_state_shutdown(clk);
 394
 395         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 396 }
 397
 398 static void arch_timer_evtstrm_enable(int divider)
 399 {
 400         u32 cntkctl = arch_timer_get_cntkctl();
 401
 402         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 403         /* Set the divider and enable virtual event stream */
 404         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 405                         | ARCH_TIMER_VIRT_EVT_EN;
 406         arch_timer_set_cntkctl(cntkctl);
 407         elf_hwcap |= HWCAP_EVTSTRM;
 408 #ifdef CONFIG_COMPAT
 409         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 410 #endif
 411 }
 412
 413 static void arch_timer_configure_evtstream(void)
 414 {
 415         int evt_stream_div, pos;
 416
 417         /* Find the closest power of two to the divisor */
 418         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 419         pos = fls(evt_stream_div);
 420         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 421                 pos--;
 422         /* enable event stream */
 423         arch_timer_evtstrm_enable(min(pos, 15));
 424 }
 425
 426 static void arch_counter_set_user_access(void)
 427 {
 428         u32 cntkctl = arch_timer_get_cntkctl();
 429
 430         /* Disable user access to the timers and the physical counter */
 431         /* Also disable virtual event stream */
 432         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 433                         | ARCH_TIMER_USR_VT_ACCESS_EN
 434                         | ARCH_TIMER_VIRT_EVT_EN
 435                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 436
 437         /* Enable user access to the virtual counter */
 438         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 439
 440         arch_timer_set_cntkctl(cntkctl);
 441 }
 442
 443 static bool arch_timer_has_nonsecure_ppi(void)
 444 {
 445         return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
 446                 arch_timer_ppi[PHYS_NONSECURE_PPI]);
 447 }
 448
 449 static u32 check_ppi_trigger(int irq)
 450 {
 451         u32 flags = irq_get_trigger_type(irq);
 452
 453         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 454                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 455                 pr_warn("WARNING: Please fix your firmware\n");
 456                 flags = IRQF_TRIGGER_LOW;
 457         }
 458
 459         return flags;
 460 }
 461
 462 static int arch_timer_starting_cpu(unsigned int cpu)
 463 {
 464         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 465         u32 flags;
 466
 467         __arch_timer_setup(ARCH_CP15_TIMER, clk);
 468
 469         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
 470         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 471
 472         if (arch_timer_has_nonsecure_ppi()) {
 473                 flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 474                 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
 475         }
 476
 477         arch_counter_set_user_access();
 478         if (evtstrm_enable)
 479                 arch_timer_configure_evtstream();
 480
 481         return 0;
 482 }
 483
 484 static void
 485 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
 486 {
 487         /* Who has more than one independent system counter? */
 488         if (arch_timer_rate)
 489                 return;
 490
 491         /*
 492          * Try to determine the frequency from the device tree or CNTFRQ,
 493          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
 494          */
 495         if (!acpi_disabled ||
 496             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
 497                 if (cntbase)
 498                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
 499                 else
 500                         arch_timer_rate = arch_timer_get_cntfrq();
 501         }
 502
 503         /* Check the timer frequency. */
 504         if (arch_timer_rate == 0)
 505                 pr_warn("Architected timer frequency not available\n");
 506 }
 507
 508 static void arch_timer_banner(unsigned type)
 509 {
 510         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 511                      type & ARCH_CP15_TIMER ? "cp15" : "",
 512                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
 513                      type & ARCH_MEM_TIMER ? "mmio" : "",
 514                      (unsigned long)arch_timer_rate / 1000000,
 515                      (unsigned long)(arch_timer_rate / 10000) % 100,
 516                      type & ARCH_CP15_TIMER ?
 517                      (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
 518                         "",
 519                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
 520                      type & ARCH_MEM_TIMER ?
 521                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 522                         "");
 523 }
 524
 525 u32 arch_timer_get_rate(void)
 526 {
 527         return arch_timer_rate;
 528 }
 529
 530 static u64 arch_counter_get_cntvct_mem(void)
 531 {
 532         u32 vct_lo, vct_hi, tmp_hi;
 533
 534         do {
 535                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 536                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 537                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 538         } while (vct_hi != tmp_hi);
 539
 540         return ((u64) vct_hi << 32) | vct_lo;
 541 }
 542
 543 /*
 544  * Default to cp15 based access because arm64 uses this function for
 545  * sched_clock() before DT is probed and the cp15 method is guaranteed
 546  * to exist on arm64. arm doesn't use this before DT is probed so even
 547  * if we don't have the cp15 accessors we won't have a problem.
 548  */
 549 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 550
 551 static u64 arch_counter_read(struct clocksource *cs)
 552 {
 553         return arch_timer_read_counter();
 554 }
 555
 556 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
 557 {
 558         return arch_timer_read_counter();
 559 }
 560
 561 static struct clocksource clocksource_counter = {
 562         .name   = "arch_sys_counter",
 563         .rating = 400,
 564         .read   = arch_counter_read,
 565         .mask   = CLOCKSOURCE_MASK(56),
 566         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
 567 };
 568
 569 static struct cyclecounter cyclecounter = {
 570         .read   = arch_counter_read_cc,
 571         .mask   = CLOCKSOURCE_MASK(56),
 572 };
 573
 574 static struct arch_timer_kvm_info arch_timer_kvm_info;
 575
 576 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
 577 {
 578         return &arch_timer_kvm_info;
 579 }
 580
 581 static void __init arch_counter_register(unsigned type)
 582 {
 583         u64 start_count;
 584
 585         /* Register the CP15 based counter if we have one */
 586         if (type & ARCH_CP15_TIMER) {
 587                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
 588                         arch_timer_read_counter = arch_counter_get_cntvct;
 589                 else
 590                         arch_timer_read_counter = arch_counter_get_cntpct;
 591
 592                 clocksource_counter.archdata.vdso_direct = true;
 593
 594 #ifdef CONFIG_FSL_ERRATUM_A008585
 595                 /*
 596                  * Don't use the vdso fastpath if errata require using
 597                  * the out-of-line counter accessor.
 598                  */
 599                 if (static_branch_unlikely(&arch_timer_read_ool_enabled))
 600                         clocksource_counter.archdata.vdso_direct = false;
 601 #endif
 602         } else {
 603                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
 604         }
 605
 606         if (!arch_counter_suspend_stop)
 607                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
 608         start_count = arch_timer_read_counter();
 609         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 610         cyclecounter.mult = clocksource_counter.mult;
 611         cyclecounter.shift = clocksource_counter.shift;
 612         timecounter_init(&arch_timer_kvm_info.timecounter,
 613                          &cyclecounter, start_count);
 614
 615         /* 56 bits minimum, so we assume worst case rollover */
 616         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 617 }
 618
 619 static void arch_timer_stop(struct clock_event_device *clk)
 620 {
 621         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 622                  clk->irq, smp_processor_id());
 623
 624         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
 625         if (arch_timer_has_nonsecure_ppi())
 626                 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 627
 628         clk->set_state_shutdown(clk);
 629 }
 630
 631 static int arch_timer_dying_cpu(unsigned int cpu)
 632 {
 633         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 634
 635         arch_timer_stop(clk);
 636         return 0;
 637 }
 638
 639 #ifdef CONFIG_CPU_PM
 640 static unsigned int saved_cntkctl;
 641 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 642                                     unsigned long action, void *hcpu)
 643 {
 644         if (action == CPU_PM_ENTER)
 645                 saved_cntkctl = arch_timer_get_cntkctl();
 646         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 647                 arch_timer_set_cntkctl(saved_cntkctl);
 648         return NOTIFY_OK;
 649 }
 650
 651 static struct notifier_block arch_timer_cpu_pm_notifier = {
 652         .notifier_call = arch_timer_cpu_pm_notify,
 653 };
 654
 655 static int __init arch_timer_cpu_pm_init(void)
 656 {
 657         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 658 }
 659
 660 static void __init arch_timer_cpu_pm_deinit(void)
 661 {
 662         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
 663 }
 664
 665 #else
 666 static int __init arch_timer_cpu_pm_init(void)
 667 {
 668         return 0;
 669 }
 670
 671 static void __init arch_timer_cpu_pm_deinit(void)
 672 {
 673 }
 674 #endif
 675
 676 static int __init arch_timer_register(void)
 677 {
 678         int err;
 679         int ppi;
 680
 681         arch_timer_evt = alloc_percpu(struct clock_event_device);
 682         if (!arch_timer_evt) {
 683                 err = -ENOMEM;
 684                 goto out;
 685         }
 686
 687         ppi = arch_timer_ppi[arch_timer_uses_ppi];
 688         switch (arch_timer_uses_ppi) {
 689         case VIRT_PPI:
 690                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 691                                          "arch_timer", arch_timer_evt);
 692                 break;
 693         case PHYS_SECURE_PPI:
 694         case PHYS_NONSECURE_PPI:
 695                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 696                                          "arch_timer", arch_timer_evt);
 697                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 698                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 699                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 700                                                  "arch_timer", arch_timer_evt);
 701                         if (err)
 702                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 703                                                 arch_timer_evt);
 704                 }
 705                 break;
 706         case HYP_PPI:
 707                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 708                                          "arch_timer", arch_timer_evt);
 709                 break;
 710         default:
 711                 BUG();
 712         }
 713
 714         if (err) {
 715                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 716                        ppi, err);
 717                 goto out_free;
 718         }
 719
 720         err = arch_timer_cpu_pm_init();
 721         if (err)
 722                 goto out_unreg_notify;
 723
 724
 725         /* Register and immediately configure the timer on the boot CPU */
 726         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
 727                                 "clockevents/arm/arch_timer:starting",
 728                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
 729         if (err)
 730                 goto out_unreg_cpupm;
 731         return 0;
 732
 733 out_unreg_cpupm:
 734         arch_timer_cpu_pm_deinit();
 735
 736 out_unreg_notify:
 737         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
 738         if (arch_timer_has_nonsecure_ppi())
 739                 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 740                                 arch_timer_evt);
 741
 742 out_free:
 743         free_percpu(arch_timer_evt);
 744 out:
 745         return err;
 746 }
 747
 748 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
 749 {
 750         int ret;
 751         irq_handler_t func;
 752         struct arch_timer *t;
 753
 754         t = kzalloc(sizeof(*t), GFP_KERNEL);
 755         if (!t)
 756                 return -ENOMEM;
 757
 758         t->base = base;
 759         t->evt.irq = irq;
 760         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
 761
 762         if (arch_timer_mem_use_virtual)
 763                 func = arch_timer_handler_virt_mem;
 764         else
 765                 func = arch_timer_handler_phys_mem;
 766
 767         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
 768         if (ret) {
 769                 pr_err("arch_timer: Failed to request mem timer irq\n");
 770                 kfree(t);
 771         }
 772
 773         return ret;
 774 }
 775
 776 static const struct of_device_id arch_timer_of_match[] __initconst = {
 777         { .compatible   = "arm,armv7-timer",    },
 778         { .compatible   = "arm,armv8-timer",    },
 779         {},
 780 };
 781
 782 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
 783         { .compatible   = "arm,armv7-timer-mem", },
 784         {},
 785 };
 786
 787 static bool __init
 788 arch_timer_needs_probing(int type, const struct of_device_id *matches)
 789 {
 790         struct device_node *dn;
 791         bool needs_probing = false;
 792
 793         dn = of_find_matching_node(NULL, matches);
 794         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
 795                 needs_probing = true;
 796         of_node_put(dn);
 797
 798         return needs_probing;
 799 }
 800
 801 static int __init arch_timer_common_init(void)
 802 {
 803         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
 804
 805         /* Wait until both nodes are probed if we have two timers */
 806         if ((arch_timers_present & mask) != mask) {
 807                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
 808                         return 0;
 809                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
 810                         return 0;
 811         }
 812
 813         arch_timer_banner(arch_timers_present);
 814         arch_counter_register(arch_timers_present);
 815         return arch_timer_arch_init();
 816 }
 817
 818 static int __init arch_timer_init(void)
 819 {
 820         int ret;
 821         /*
 822          * If HYP mode is available, we know that the physical timer
 823          * has been configured to be accessible from PL1. Use it, so
 824          * that a guest can use the virtual timer instead.
 825          *
 826          * If no interrupt provided for virtual timer, we'll have to
 827          * stick to the physical timer. It'd better be accessible...
 828          *
 829          * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
 830          * accesses to CNTP_*_EL1 registers are silently redirected to
 831          * their CNTHP_*_EL2 counterparts, and use a different PPI
 832          * number.
 833          */
 834         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 835                 bool has_ppi;
 836
 837                 if (is_kernel_in_hyp_mode()) {
 838                         arch_timer_uses_ppi = HYP_PPI;
 839                         has_ppi = !!arch_timer_ppi[HYP_PPI];
 840                 } else {
 841                         arch_timer_uses_ppi = PHYS_SECURE_PPI;
 842                         has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
 843                                    !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
 844                 }
 845
 846                 if (!has_ppi) {
 847                         pr_warn("arch_timer: No interrupt available, giving up\n");
 848                         return -EINVAL;
 849                 }
 850         }
 851
 852         ret = arch_timer_register();
 853         if (ret)
 854                 return ret;
 855
 856         ret = arch_timer_common_init();
 857         if (ret)
 858                 return ret;
 859
 860         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
 861
 862         return 0;
 863 }
 864
 865 static int __init arch_timer_of_init(struct device_node *np)
 866 {
 867         int i;
 868
 869         if (arch_timers_present & ARCH_CP15_TIMER) {
 870                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 871                 return 0;
 872         }
 873
 874         arch_timers_present |= ARCH_CP15_TIMER;
 875         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 876                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 877
 878         arch_timer_detect_rate(NULL, np);
 879
 880         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 881
 882 #ifdef CONFIG_FSL_ERRATUM_A008585
 883         if (fsl_a008585_enable < 0)
 884                 fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585");
 885         if (fsl_a008585_enable) {
 886                 static_branch_enable(&arch_timer_read_ool_enabled);
 887                 pr_info("Enabling workaround for FSL erratum A-008585\n");
 888         }
 889 #endif
 890
 891         /*
 892          * If we cannot rely on firmware initializing the timer registers then
 893          * we should use the physical timers instead.
 894          */
 895         if (IS_ENABLED(CONFIG_ARM) &&
 896             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
 897                 arch_timer_uses_ppi = PHYS_SECURE_PPI;
 898
 899         /* On some systems, the counter stops ticking when in suspend. */
 900         arch_counter_suspend_stop = of_property_read_bool(np,
 901                                                          "arm,no-tick-in-suspend");
 902
 903         return arch_timer_init();
 904 }
 905 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
 906 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
 907
 908 static int __init arch_timer_mem_init(struct device_node *np)
 909 {
 910         struct device_node *frame, *best_frame = NULL;
 911         void __iomem *cntctlbase, *base;
 912         unsigned int irq, ret = -EINVAL;
 913         u32 cnttidr;
 914
 915         arch_timers_present |= ARCH_MEM_TIMER;
 916         cntctlbase = of_iomap(np, 0);
 917         if (!cntctlbase) {
 918                 pr_err("arch_timer: Can't find CNTCTLBase\n");
 919                 return -ENXIO;
 920         }
 921
 922         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
 923
 924         /*
 925          * Try to find a virtual capable frame. Otherwise fall back to a
 926          * physical capable frame.
 927          */
 928         for_each_available_child_of_node(np, frame) {
 929                 int n;
 930                 u32 cntacr;
 931
 932                 if (of_property_read_u32(frame, "frame-number", &n)) {
 933                         pr_err("arch_timer: Missing frame-number\n");
 934                         of_node_put(frame);
 935                         goto out;
 936                 }
 937
 938                 /* Try enabling everything, and see what sticks */
 939                 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
 940                          CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
 941                 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
 942                 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
 943
 944                 if ((cnttidr & CNTTIDR_VIRT(n)) &&
 945                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
 946                         of_node_put(best_frame);
 947                         best_frame = frame;
 948                         arch_timer_mem_use_virtual = true;
 949                         break;
 950                 }
 951
 952                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
 953                         continue;
 954
 955                 of_node_put(best_frame);
 956                 best_frame = of_node_get(frame);
 957         }
 958
 959         ret= -ENXIO;
 960         base = arch_counter_base = of_io_request_and_map(best_frame, 0,
 961                                                          "arch_mem_timer");
 962         if (IS_ERR(base)) {
 963                 pr_err("arch_timer: Can't map frame's registers\n");
 964                 goto out;
 965         }
 966
 967         if (arch_timer_mem_use_virtual)
 968                 irq = irq_of_parse_and_map(best_frame, 1);
 969         else
 970                 irq = irq_of_parse_and_map(best_frame, 0);
 971
 972         ret = -EINVAL;
 973         if (!irq) {
 974                 pr_err("arch_timer: Frame missing %s irq",
 975                        arch_timer_mem_use_virtual ? "virt" : "phys");
 976                 goto out;
 977         }
 978
 979         arch_timer_detect_rate(base, np);
 980         ret = arch_timer_mem_register(base, irq);
 981         if (ret)
 982                 goto out;
 983
 984         return arch_timer_common_init();
 985 out:
 986         iounmap(cntctlbase);
 987         of_node_put(best_frame);
 988         return ret;
 989 }
 990 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
 991                        arch_timer_mem_init);
 992
 993 #ifdef CONFIG_ACPI
 994 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
 995 {
 996         int trigger, polarity;
 997
 998         if (!interrupt)
 999                 return 0;
1000
1001         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
1002                         : ACPI_LEVEL_SENSITIVE;
1003
1004         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
1005                         : ACPI_ACTIVE_HIGH;
1006
1007         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
1008 }
1009
1010 /* Initialize per-processor generic timer */
1011 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1012 {
1013         struct acpi_table_gtdt *gtdt;
1014
1015         if (arch_timers_present & ARCH_CP15_TIMER) {
1016                 pr_warn("arch_timer: already initialized, skipping\n");
1017                 return -EINVAL;
1018         }
1019
1020         gtdt = container_of(table, struct acpi_table_gtdt, header);
1021
1022         arch_timers_present |= ARCH_CP15_TIMER;
1023
1024         arch_timer_ppi[PHYS_SECURE_PPI] =
1025                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
1026                 gtdt->secure_el1_flags);
1027
1028         arch_timer_ppi[PHYS_NONSECURE_PPI] =
1029                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
1030                 gtdt->non_secure_el1_flags);
1031
1032         arch_timer_ppi[VIRT_PPI] =
1033                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
1034                 gtdt->virtual_timer_flags);
1035
1036         arch_timer_ppi[HYP_PPI] =
1037                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
1038                 gtdt->non_secure_el2_flags);
1039
1040         /* Get the frequency from CNTFRQ */
1041         arch_timer_detect_rate(NULL, NULL);
1042
1043         /* Always-on capability */
1044         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
1045
1046         arch_timer_init();
1047         return 0;
1048 }
1049 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1050 #endif