Merge branch 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
authorLinus Torvalds <torvalds@linux-foundation.org>
Tue, 30 Jan 2018 00:50:58 +0000 (16:50 -0800)
committerLinus Torvalds <torvalds@linux-foundation.org>
Tue, 30 Jan 2018 00:50:58 +0000 (16:50 -0800)
Pull timer updates from Thomas Gleixner:
 "The timer departement presents:

   - A rather large rework of the hrtimer infrastructure which
     introduces softirq based hrtimers to replace the spread of
     hrtimer/tasklet combos which force the actual callback execution
     into softirq context. The approach is completely different from the
     initial implementation which you cursed at 10 years ago rightfully.

     The softirq based timers have their own queues and there is no
     nasty indirection and list reshuffling in the hard interrupt
     anymore. This comes with conversion of some of the hrtimer/tasklet
     users, the rest and the final removal of that horrible interface
     will come towards the end of the merge window or go through the
     relevant maintainer trees.

     Note: The top commit merged the last minute bugfix for the 10 years
     old CPU hotplug bug as I wanted to make sure that I fatfinger the
     merge conflict resolution myself.

   - The overhaul of the STM32 clocksource/clockevents driver

   - A new driver for the Spreadtrum SC9860 timer

   - A new driver dor the Actions Semi S700 timer

   - The usual set of fixes and updates all over the place"

* 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (53 commits)
  usb/gadget/NCM: Replace tasklet with softirq hrtimer
  ALSA/dummy: Replace tasklet with softirq hrtimer
  hrtimer: Implement SOFT/HARD clock base selection
  hrtimer: Implement support for softirq based hrtimers
  hrtimer: Prepare handling of hard and softirq based hrtimers
  hrtimer: Add clock bases and hrtimer mode for softirq context
  hrtimer: Use irqsave/irqrestore around __run_hrtimer()
  hrtimer: Factor out __hrtimer_next_event_base()
  hrtimer: Factor out __hrtimer_start_range_ns()
  hrtimer: Remove the 'base' parameter from hrtimer_reprogram()
  hrtimer: Make remote enqueue decision less restrictive
  hrtimer: Unify remote enqueue handling
  hrtimer: Unify hrtimer removal handling
  hrtimer: Make hrtimer_force_reprogramm() unconditionally available
  hrtimer: Make hrtimer_reprogramm() unconditional
  hrtimer: Make hrtimer_cpu_base.next_timer handling unconditional
  hrtimer: Make the remote enqueue check unconditional
  hrtimer: Use accesor functions instead of direct access
  hrtimer: Make the hrtimer_cpu_base::hres_active field unconditional, to simplify the code
  hrtimer: Make room in 'struct hrtimer_cpu_base'
  ...

22 files changed:
Documentation/devicetree/bindings/timer/actions,owl-timer.txt
Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt [new file with mode: 0644]
drivers/clocksource/Kconfig
drivers/clocksource/Makefile
drivers/clocksource/owl-timer.c
drivers/clocksource/tcb_clksrc.c
drivers/clocksource/timer-of.c
drivers/clocksource/timer-of.h
drivers/clocksource/timer-sprd.c [new file with mode: 0644]
drivers/clocksource/timer-stm32.c
drivers/usb/gadget/function/f_ncm.c
include/linux/hrtimer.h
include/linux/posix-timers.h
include/trace/events/timer.h
kernel/time/hrtimer.c
kernel/time/posix-clock.c
kernel/time/posix-cpu-timers.c
kernel/time/tick-internal.h
kernel/time/tick-sched.c
kernel/time/timer.c
sound/drivers/dummy.c
tools/testing/selftests/ptp/testptp.c

index e3c28da80cb259f8e41c94514478200e7caaf770..977054f87563ce8e0faa3a5887ce23f24a2c1954 100644 (file)
@@ -2,6 +2,7 @@ Actions Semi Owl Timer
 
 Required properties:
 - compatible      :  "actions,s500-timer" for S500
+                     "actions,s700-timer" for S700
                      "actions,s900-timer" for S900
 - reg             :  Offset and length of the register set for the device.
 - interrupts      :  Should contain the interrupts.
diff --git a/Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt b/Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt
new file mode 100644 (file)
index 0000000..6d97e7d
--- /dev/null
@@ -0,0 +1,20 @@
+Spreadtrum timers
+
+The Spreadtrum SC9860 platform provides 3 general-purpose timers.
+These timers can support 32bit or 64bit counter, as well as supporting
+period mode or one-shot mode, and they are can be wakeup source
+during deep sleep.
+
+Required properties:
+- compatible: should be "sprd,sc9860-timer" for SC9860 platform.
+- reg: The register address of the timer device.
+- interrupts: Should contain the interrupt for the timer device.
+- clocks: The phandle to the source clock (usually a 32.768 KHz fixed clock).
+
+Example:
+       timer@40050000 {
+               compatible = "sprd,sc9860-timer";
+               reg = <0 0x40050000 0 0x20>;
+               interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
+               clocks = <&ext_32k>;
+       };
index c729a88007d0fa7b0c5132ad36dec959e396607a..b3b4ed9b68742bfbb8219af9c24c9d542d6f02ec 100644 (file)
@@ -269,6 +269,7 @@ config CLKSRC_STM32
        bool "Clocksource for STM32 SoCs" if !ARCH_STM32
        depends on OF && ARM && (ARCH_STM32 || COMPILE_TEST)
        select CLKSRC_MMIO
+       select TIMER_OF
 
 config CLKSRC_MPS2
        bool "Clocksource for MPS2 SoCs" if COMPILE_TEST
@@ -441,6 +442,13 @@ config MTK_TIMER
        help
          Support for Mediatek timer driver.
 
+config SPRD_TIMER
+       bool "Spreadtrum timer driver" if COMPILE_TEST
+       depends on HAS_IOMEM
+       select TIMER_OF
+       help
+         Enables support for the Spreadtrum timer driver.
+
 config SYS_SUPPORTS_SH_MTU2
         bool
 
index 72711f1491e3ee14e67beacc7c4b9e8688e9e07d..d6dec4489d66e4e8996e11726681958dbf9d9bec 100644 (file)
@@ -54,6 +54,7 @@ obj-$(CONFIG_CLKSRC_TI_32K)   += timer-ti-32k.o
 obj-$(CONFIG_CLKSRC_NPS)       += timer-nps.o
 obj-$(CONFIG_OXNAS_RPS_TIMER)  += timer-oxnas-rps.o
 obj-$(CONFIG_OWL_TIMER)                += owl-timer.o
+obj-$(CONFIG_SPRD_TIMER)       += timer-sprd.o
 
 obj-$(CONFIG_ARC_TIMERS)               += arc_timer.o
 obj-$(CONFIG_ARM_ARCH_TIMER)           += arm_arch_timer.o
index c686305650797a8b75b2e7f5be4a2ec568d84c28..ea00a5e8f95d291581d268659ea0c16b22362958 100644 (file)
@@ -168,5 +168,6 @@ static int __init owl_timer_init(struct device_node *node)
 
        return 0;
 }
-CLOCKSOURCE_OF_DECLARE(owl_s500, "actions,s500-timer", owl_timer_init);
-CLOCKSOURCE_OF_DECLARE(owl_s900, "actions,s900-timer", owl_timer_init);
+TIMER_OF_DECLARE(owl_s500, "actions,s500-timer", owl_timer_init);
+TIMER_OF_DECLARE(owl_s700, "actions,s700-timer", owl_timer_init);
+TIMER_OF_DECLARE(owl_s900, "actions,s900-timer", owl_timer_init);
index 9de47d4d2d9ef9338d87bf4c45f4695df7b1702c..43f4d5c4d6fa4fdb8f4581d71010e4b54ecd18da 100644 (file)
@@ -384,7 +384,7 @@ static int __init tcb_clksrc_init(void)
 
        printk(bootinfo, clksrc.name, CONFIG_ATMEL_TCB_CLKSRC_BLOCK,
                        divided_rate / 1000000,
-                       ((divided_rate + 500000) % 1000000) / 1000);
+                       ((divided_rate % 1000000) + 500) / 1000);
 
        if (tc->tcb_config && tc->tcb_config->counter_width == 32) {
                /* use apropriate function to read 32 bit counter */
index a31990408153d13e33211363e2d4e3442f8b9d8e..06ed88a2a8a0d4944937be3884784e773237e732 100644 (file)
 
 #include "timer-of.h"
 
-static __init void timer_irq_exit(struct of_timer_irq *of_irq)
+/**
+ * timer_of_irq_exit - Release the interrupt
+ * @of_irq: an of_timer_irq structure pointer
+ *
+ * Free the irq resource
+ */
+static __init void timer_of_irq_exit(struct of_timer_irq *of_irq)
 {
        struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
 
@@ -34,8 +40,24 @@ static __init void timer_irq_exit(struct of_timer_irq *of_irq)
                free_irq(of_irq->irq, clkevt);
 }
 
-static __init int timer_irq_init(struct device_node *np,
-                                struct of_timer_irq *of_irq)
+/**
+ * timer_of_irq_init - Request the interrupt
+ * @np: a device tree node pointer
+ * @of_irq: an of_timer_irq structure pointer
+ *
+ * Get the interrupt number from the DT from its definition and
+ * request it. The interrupt is gotten by falling back the following way:
+ *
+ * - Get interrupt number by name
+ * - Get interrupt number by index
+ *
+ * When the interrupt is per CPU, 'request_percpu_irq()' is called,
+ * otherwise 'request_irq()' is used.
+ *
+ * Returns 0 on success, < 0 otherwise
+ */
+static __init int timer_of_irq_init(struct device_node *np,
+                                   struct of_timer_irq *of_irq)
 {
        int ret;
        struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
@@ -72,15 +94,30 @@ static __init int timer_irq_init(struct device_node *np,
        return 0;
 }
 
-static __init void timer_clk_exit(struct of_timer_clk *of_clk)
+/**
+ * timer_of_clk_exit - Release the clock resources
+ * @of_clk: a of_timer_clk structure pointer
+ *
+ * Disables and releases the refcount on the clk
+ */
+static __init void timer_of_clk_exit(struct of_timer_clk *of_clk)
 {
        of_clk->rate = 0;
        clk_disable_unprepare(of_clk->clk);
        clk_put(of_clk->clk);
 }
 
-static __init int timer_clk_init(struct device_node *np,
-                                struct of_timer_clk *of_clk)
+/**
+ * timer_of_clk_init - Initialize the clock resources
+ * @np: a device tree node pointer
+ * @of_clk: a of_timer_clk structure pointer
+ *
+ * Get the clock by name or by index, enable it and get the rate
+ *
+ * Returns 0 on success, < 0 otherwise
+ */
+static __init int timer_of_clk_init(struct device_node *np,
+                                   struct of_timer_clk *of_clk)
 {
        int ret;
 
@@ -116,19 +153,19 @@ out_clk_put:
        goto out;
 }
 
-static __init void timer_base_exit(struct of_timer_base *of_base)
+static __init void timer_of_base_exit(struct of_timer_base *of_base)
 {
        iounmap(of_base->base);
 }
 
-static __init int timer_base_init(struct device_node *np,
-                                 struct of_timer_base *of_base)
+static __init int timer_of_base_init(struct device_node *np,
+                                    struct of_timer_base *of_base)
 {
-       const char *name = of_base->name ? of_base->name : np->full_name;
-
-       of_base->base = of_io_request_and_map(np, of_base->index, name);
+       of_base->base = of_base->name ?
+               of_io_request_and_map(np, of_base->index, of_base->name) :
+               of_iomap(np, of_base->index);
        if (IS_ERR(of_base->base)) {
-               pr_err("Failed to iomap (%s)\n", name);
+               pr_err("Failed to iomap (%s)\n", of_base->name);
                return PTR_ERR(of_base->base);
        }
 
@@ -141,21 +178,21 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
        int flags = 0;
 
        if (to->flags & TIMER_OF_BASE) {
-               ret = timer_base_init(np, &to->of_base);
+               ret = timer_of_base_init(np, &to->of_base);
                if (ret)
                        goto out_fail;
                flags |= TIMER_OF_BASE;
        }
 
        if (to->flags & TIMER_OF_CLOCK) {
-               ret = timer_clk_init(np, &to->of_clk);
+               ret = timer_of_clk_init(np, &to->of_clk);
                if (ret)
                        goto out_fail;
                flags |= TIMER_OF_CLOCK;
        }
 
        if (to->flags & TIMER_OF_IRQ) {
-               ret = timer_irq_init(np, &to->of_irq);
+               ret = timer_of_irq_init(np, &to->of_irq);
                if (ret)
                        goto out_fail;
                flags |= TIMER_OF_IRQ;
@@ -163,17 +200,20 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
 
        if (!to->clkevt.name)
                to->clkevt.name = np->name;
+
+       to->np = np;
+
        return ret;
 
 out_fail:
        if (flags & TIMER_OF_IRQ)
-               timer_irq_exit(&to->of_irq);
+               timer_of_irq_exit(&to->of_irq);
 
        if (flags & TIMER_OF_CLOCK)
-               timer_clk_exit(&to->of_clk);
+               timer_of_clk_exit(&to->of_clk);
 
        if (flags & TIMER_OF_BASE)
-               timer_base_exit(&to->of_base);
+               timer_of_base_exit(&to->of_base);
        return ret;
 }
 
@@ -187,11 +227,11 @@ out_fail:
 void __init timer_of_cleanup(struct timer_of *to)
 {
        if (to->flags & TIMER_OF_IRQ)
-               timer_irq_exit(&to->of_irq);
+               timer_of_irq_exit(&to->of_irq);
 
        if (to->flags & TIMER_OF_CLOCK)
-               timer_clk_exit(&to->of_clk);
+               timer_of_clk_exit(&to->of_clk);
 
        if (to->flags & TIMER_OF_BASE)
-               timer_base_exit(&to->of_base);
+               timer_of_base_exit(&to->of_base);
 }
index 3f708f1be43d8671359ab5b9d0bc7934b94aec5b..a5478f3e8589df8f1cdc476ce0cf47a00122aca5 100644 (file)
@@ -33,6 +33,7 @@ struct of_timer_clk {
 
 struct timer_of {
        unsigned int flags;
+       struct device_node *np;
        struct clock_event_device clkevt;
        struct of_timer_base of_base;
        struct of_timer_irq  of_irq;
diff --git a/drivers/clocksource/timer-sprd.c b/drivers/clocksource/timer-sprd.c
new file mode 100644 (file)
index 0000000..ef9ebea
--- /dev/null
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 Spreadtrum Communications Inc.
+ */
+
+#include <linux/init.h>
+#include <linux/interrupt.h>
+
+#include "timer-of.h"
+
+#define TIMER_NAME             "sprd_timer"
+
+#define TIMER_LOAD_LO          0x0
+#define TIMER_LOAD_HI          0x4
+#define TIMER_VALUE_LO         0x8
+#define TIMER_VALUE_HI         0xc
+
+#define TIMER_CTL              0x10
+#define TIMER_CTL_PERIOD_MODE  BIT(0)
+#define TIMER_CTL_ENABLE       BIT(1)
+#define TIMER_CTL_64BIT_WIDTH  BIT(16)
+
+#define TIMER_INT              0x14
+#define TIMER_INT_EN           BIT(0)
+#define TIMER_INT_RAW_STS      BIT(1)
+#define TIMER_INT_MASK_STS     BIT(2)
+#define TIMER_INT_CLR          BIT(3)
+
+#define TIMER_VALUE_SHDW_LO    0x18
+#define TIMER_VALUE_SHDW_HI    0x1c
+
+#define TIMER_VALUE_LO_MASK    GENMASK(31, 0)
+
+static void sprd_timer_enable(void __iomem *base, u32 flag)
+{
+       u32 val = readl_relaxed(base + TIMER_CTL);
+
+       val |= TIMER_CTL_ENABLE;
+       if (flag & TIMER_CTL_64BIT_WIDTH)
+               val |= TIMER_CTL_64BIT_WIDTH;
+       else
+               val &= ~TIMER_CTL_64BIT_WIDTH;
+
+       if (flag & TIMER_CTL_PERIOD_MODE)
+               val |= TIMER_CTL_PERIOD_MODE;
+       else
+               val &= ~TIMER_CTL_PERIOD_MODE;
+
+       writel_relaxed(val, base + TIMER_CTL);
+}
+
+static void sprd_timer_disable(void __iomem *base)
+{
+       u32 val = readl_relaxed(base + TIMER_CTL);
+
+       val &= ~TIMER_CTL_ENABLE;
+       writel_relaxed(val, base + TIMER_CTL);
+}
+
+static void sprd_timer_update_counter(void __iomem *base, unsigned long cycles)
+{
+       writel_relaxed(cycles & TIMER_VALUE_LO_MASK, base + TIMER_LOAD_LO);
+       writel_relaxed(0, base + TIMER_LOAD_HI);
+}
+
+static void sprd_timer_enable_interrupt(void __iomem *base)
+{
+       writel_relaxed(TIMER_INT_EN, base + TIMER_INT);
+}
+
+static void sprd_timer_clear_interrupt(void __iomem *base)
+{
+       u32 val = readl_relaxed(base + TIMER_INT);
+
+       val |= TIMER_INT_CLR;
+       writel_relaxed(val, base + TIMER_INT);
+}
+
+static int sprd_timer_set_next_event(unsigned long cycles,
+                                    struct clock_event_device *ce)
+{
+       struct timer_of *to = to_timer_of(ce);
+
+       sprd_timer_disable(timer_of_base(to));
+       sprd_timer_update_counter(timer_of_base(to), cycles);
+       sprd_timer_enable(timer_of_base(to), 0);
+
+       return 0;
+}
+
+static int sprd_timer_set_periodic(struct clock_event_device *ce)
+{
+       struct timer_of *to = to_timer_of(ce);
+
+       sprd_timer_disable(timer_of_base(to));
+       sprd_timer_update_counter(timer_of_base(to), timer_of_period(to));
+       sprd_timer_enable(timer_of_base(to), TIMER_CTL_PERIOD_MODE);
+
+       return 0;
+}
+
+static int sprd_timer_shutdown(struct clock_event_device *ce)
+{
+       struct timer_of *to = to_timer_of(ce);
+
+       sprd_timer_disable(timer_of_base(to));
+       return 0;
+}
+
+static irqreturn_t sprd_timer_interrupt(int irq, void *dev_id)
+{
+       struct clock_event_device *ce = (struct clock_event_device *)dev_id;
+       struct timer_of *to = to_timer_of(ce);
+
+       sprd_timer_clear_interrupt(timer_of_base(to));
+
+       if (clockevent_state_oneshot(ce))
+               sprd_timer_disable(timer_of_base(to));
+
+       ce->event_handler(ce);
+       return IRQ_HANDLED;
+}
+
+static struct timer_of to = {
+       .flags = TIMER_OF_IRQ | TIMER_OF_BASE | TIMER_OF_CLOCK,
+
+       .clkevt = {
+               .name = TIMER_NAME,
+               .rating = 300,
+               .features = CLOCK_EVT_FEAT_DYNIRQ | CLOCK_EVT_FEAT_PERIODIC |
+                       CLOCK_EVT_FEAT_ONESHOT,
+               .set_state_shutdown = sprd_timer_shutdown,
+               .set_state_periodic = sprd_timer_set_periodic,
+               .set_next_event = sprd_timer_set_next_event,
+               .cpumask = cpu_possible_mask,
+       },
+
+       .of_irq = {
+               .handler = sprd_timer_interrupt,
+               .flags = IRQF_TIMER | IRQF_IRQPOLL,
+       },
+};
+
+static int __init sprd_timer_init(struct device_node *np)
+{
+       int ret;
+
+       ret = timer_of_init(np, &to);
+       if (ret)
+               return ret;
+
+       sprd_timer_enable_interrupt(timer_of_base(&to));
+       clockevents_config_and_register(&to.clkevt, timer_of_rate(&to),
+                                       1, UINT_MAX);
+
+       return 0;
+}
+
+TIMER_OF_DECLARE(sc9860_timer, "sprd,sc9860-timer", sprd_timer_init);
index 8f2423789ba909bd68b74e4c18e0392aa7d8a826..e5cdc3af684cbbe2370406c79af63df143776cd3 100644 (file)
@@ -9,6 +9,7 @@
 #include <linux/kernel.h>
 #include <linux/clocksource.h>
 #include <linux/clockchips.h>
+#include <linux/delay.h>
 #include <linux/irq.h>
 #include <linux/interrupt.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
 #include <linux/clk.h>
 #include <linux/reset.h>
+#include <linux/sched_clock.h>
+#include <linux/slab.h>
+
+#include "timer-of.h"
 
 #define TIM_CR1                0x00
 #define TIM_DIER       0x0c
 #define TIM_SR         0x10
 #define TIM_EGR                0x14
+#define TIM_CNT                0x24
 #define TIM_PSC                0x28
 #define TIM_ARR                0x2c
+#define TIM_CCR1       0x34
 
 #define TIM_CR1_CEN    BIT(0)
+#define TIM_CR1_UDIS   BIT(1)
 #define TIM_CR1_OPM    BIT(3)
 #define TIM_CR1_ARPE   BIT(7)
 
 #define TIM_DIER_UIE   BIT(0)
+#define TIM_DIER_CC1IE BIT(1)
 
 #define TIM_SR_UIF     BIT(0)
 
 #define TIM_EGR_UG     BIT(0)
 
-struct stm32_clock_event_ddata {
-       struct clock_event_device evtdev;
-       unsigned periodic_top;
-       void __iomem *base;
+#define TIM_PSC_MAX    USHRT_MAX
+#define TIM_PSC_CLKRATE        10000
+
+struct stm32_timer_private {
+       int bits;
 };
 
-static int stm32_clock_event_shutdown(struct clock_event_device *evtdev)
+/**
+ * stm32_timer_of_bits_set - set accessor helper
+ * @to: a timer_of structure pointer
+ * @bits: the number of bits (16 or 32)
+ *
+ * Accessor helper to set the number of bits in the timer-of private
+ * structure.
+ *
+ */
+static void stm32_timer_of_bits_set(struct timer_of *to, int bits)
 {
-       struct stm32_clock_event_ddata *data =
-               container_of(evtdev, struct stm32_clock_event_ddata, evtdev);
-       void *base = data->base;
+       struct stm32_timer_private *pd = to->private_data;
 
-       writel_relaxed(0, base + TIM_CR1);
-       return 0;
+       pd->bits = bits;
+}
+
+/**
+ * stm32_timer_of_bits_get - get accessor helper
+ * @to: a timer_of structure pointer
+ *
+ * Accessor helper to get the number of bits in the timer-of private
+ * structure.
+ *
+ * Returns an integer corresponding to the number of bits.
+ */
+static int stm32_timer_of_bits_get(struct timer_of *to)
+{
+       struct stm32_timer_private *pd = to->private_data;
+
+       return pd->bits;
+}
+
+static void __iomem *stm32_timer_cnt __read_mostly;
+
+static u64 notrace stm32_read_sched_clock(void)
+{
+       return readl_relaxed(stm32_timer_cnt);
+}
+
+static struct delay_timer stm32_timer_delay;
+
+static unsigned long stm32_read_delay(void)
+{
+       return readl_relaxed(stm32_timer_cnt);
 }
 
-static int stm32_clock_event_set_periodic(struct clock_event_device *evtdev)
+static void stm32_clock_event_disable(struct timer_of *to)
 {
-       struct stm32_clock_event_ddata *data =
-               container_of(evtdev, struct stm32_clock_event_ddata, evtdev);
-       void *base = data->base;
+       writel_relaxed(0, timer_of_base(to) + TIM_DIER);
+}
+
+/**
+ * stm32_timer_start - Start the counter without event
+ * @to: a timer_of structure pointer
+ *
+ * Start the timer in order to have the counter reset and start
+ * incrementing but disable interrupt event when there is a counter
+ * overflow. By default, the counter direction is used as upcounter.
+ */
+static void stm32_timer_start(struct timer_of *to)
+{
+       writel_relaxed(TIM_CR1_UDIS | TIM_CR1_CEN, timer_of_base(to) + TIM_CR1);
+}
+
+static int stm32_clock_event_shutdown(struct clock_event_device *clkevt)
+{
+       struct timer_of *to = to_timer_of(clkevt);
+
+       stm32_clock_event_disable(to);
 
-       writel_relaxed(data->periodic_top, base + TIM_ARR);
-       writel_relaxed(TIM_CR1_ARPE | TIM_CR1_CEN, base + TIM_CR1);
        return 0;
 }
 
 static int stm32_clock_event_set_next_event(unsigned long evt,
-                                           struct clock_event_device *evtdev)
+                                           struct clock_event_device *clkevt)
 {
-       struct stm32_clock_event_ddata *data =
-               container_of(evtdev, struct stm32_clock_event_ddata, evtdev);
+       struct timer_of *to = to_timer_of(clkevt);
+       unsigned long now, next;
+
+       next = readl_relaxed(timer_of_base(to) + TIM_CNT) + evt;
+       writel_relaxed(next, timer_of_base(to) + TIM_CCR1);
+       now = readl_relaxed(timer_of_base(to) + TIM_CNT);
+
+       if ((next - now) > evt)
+               return -ETIME;
 
-       writel_relaxed(evt, data->base + TIM_ARR);
-       writel_relaxed(TIM_CR1_ARPE | TIM_CR1_OPM | TIM_CR1_CEN,
-                      data->base + TIM_CR1);
+       writel_relaxed(TIM_DIER_CC1IE, timer_of_base(to) + TIM_DIER);
+
+       return 0;
+}
+
+static int stm32_clock_event_set_periodic(struct clock_event_device *clkevt)
+{
+       struct timer_of *to = to_timer_of(clkevt);
+
+       stm32_timer_start(to);
+
+       return stm32_clock_event_set_next_event(timer_of_period(to), clkevt);
+}
+
+static int stm32_clock_event_set_oneshot(struct clock_event_device *clkevt)
+{
+       struct timer_of *to = to_timer_of(clkevt);
+
+       stm32_timer_start(to);
 
        return 0;
 }
 
 static irqreturn_t stm32_clock_event_handler(int irq, void *dev_id)
 {
-       struct stm32_clock_event_ddata *data = dev_id;
+       struct clock_event_device *clkevt = (struct clock_event_device *)dev_id;
+       struct timer_of *to = to_timer_of(clkevt);
+
+       writel_relaxed(0, timer_of_base(to) + TIM_SR);
 
-       writel_relaxed(0, data->base + TIM_SR);
+       if (clockevent_state_periodic(clkevt))
+               stm32_clock_event_set_periodic(clkevt);
+       else
+               stm32_clock_event_shutdown(clkevt);
 
-       data->evtdev.event_handler(&data->evtdev);
+       clkevt->event_handler(clkevt);
 
        return IRQ_HANDLED;
 }
 
-static struct stm32_clock_event_ddata clock_event_ddata = {
-       .evtdev = {
-               .name = "stm32 clockevent",
-               .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC,
-               .set_state_shutdown = stm32_clock_event_shutdown,
-               .set_state_periodic = stm32_clock_event_set_periodic,
-               .set_state_oneshot = stm32_clock_event_shutdown,
-               .tick_resume = stm32_clock_event_shutdown,
-               .set_next_event = stm32_clock_event_set_next_event,
-               .rating = 200,
-       },
-};
+/**
+ * stm32_timer_width - Sort out the timer width (32/16)
+ * @to: a pointer to a timer-of structure
+ *
+ * Write the 32-bit max value and read/return the result. If the timer
+ * is 32 bits wide, the result will be UINT_MAX, otherwise it will
+ * be truncated by the 16-bit register to USHRT_MAX.
+ *
+ */
+static void __init stm32_timer_set_width(struct timer_of *to)
+{
+       u32 width;
+
+       writel_relaxed(UINT_MAX, timer_of_base(to) + TIM_ARR);
+
+       width = readl_relaxed(timer_of_base(to) + TIM_ARR);
+
+       stm32_timer_of_bits_set(to, width == UINT_MAX ? 32 : 16);
+}
 
-static int __init stm32_clockevent_init(struct device_node *np)
+/**
+ * stm32_timer_set_prescaler - Compute and set the prescaler register
+ * @to: a pointer to a timer-of structure
+ *
+ * Depending on the timer width, compute the prescaler to always
+ * target a 10MHz timer rate for 16 bits. 32-bit timers are
+ * considered precise and long enough to not use the prescaler.
+ */
+static void __init stm32_timer_set_prescaler(struct timer_of *to)
 {
-       struct stm32_clock_event_ddata *data = &clock_event_ddata;
-       struct clk *clk;
-       struct reset_control *rstc;
-       unsigned long rate, max_delta;
-       int irq, ret, bits, prescaler = 1;
-
-       clk = of_clk_get(np, 0);
-       if (IS_ERR(clk)) {
-               ret = PTR_ERR(clk);
-               pr_err("failed to get clock for clockevent (%d)\n", ret);
-               goto err_clk_get;
+       int prescaler = 1;
+
+       if (stm32_timer_of_bits_get(to) != 32) {
+               prescaler = DIV_ROUND_CLOSEST(timer_of_rate(to),
+                                             TIM_PSC_CLKRATE);
+               /*
+                * The prescaler register is an u16, the variable
+                * can't be greater than TIM_PSC_MAX, let's cap it in
+                * this case.
+                */
+               prescaler = prescaler < TIM_PSC_MAX ? prescaler : TIM_PSC_MAX;
        }
 
-       ret = clk_prepare_enable(clk);
-       if (ret) {
-               pr_err("failed to enable timer clock for clockevent (%d)\n",
-                      ret);
-               goto err_clk_enable;
-       }
+       writel_relaxed(prescaler - 1, timer_of_base(to) + TIM_PSC);
+       writel_relaxed(TIM_EGR_UG, timer_of_base(to) + TIM_EGR);
+       writel_relaxed(0, timer_of_base(to) + TIM_SR);
 
-       rate = clk_get_rate(clk);
+       /* Adjust rate and period given the prescaler value */
+       to->of_clk.rate = DIV_ROUND_CLOSEST(to->of_clk.rate, prescaler);
+       to->of_clk.period = DIV_ROUND_UP(to->of_clk.rate, HZ);
+}
 
-       rstc = of_reset_control_get(np, NULL);
-       if (!IS_ERR(rstc)) {
-               reset_control_assert(rstc);
-               reset_control_deassert(rstc);
+static int __init stm32_clocksource_init(struct timer_of *to)
+{
+        u32 bits = stm32_timer_of_bits_get(to);
+       const char *name = to->np->full_name;
+
+       /*
+        * This driver allows to register several timers and relies on
+        * the generic time framework to select the right one.
+        * However, nothing allows to do the same for the
+        * sched_clock. We are not interested in a sched_clock for the
+        * 16-bit timers but only for the 32-bit one, so if no 32-bit
+        * timer is registered yet, we select this 32-bit timer as a
+        * sched_clock.
+        */
+       if (bits == 32 && !stm32_timer_cnt) {
+
+               /*
+                * Start immediately the counter as we will be using
+                * it right after.
+                */
+               stm32_timer_start(to);
+
+               stm32_timer_cnt = timer_of_base(to) + TIM_CNT;
+               sched_clock_register(stm32_read_sched_clock, bits, timer_of_rate(to));
+               pr_info("%s: STM32 sched_clock registered\n", name);
+
+               stm32_timer_delay.read_current_timer = stm32_read_delay;
+               stm32_timer_delay.freq = timer_of_rate(to);
+               register_current_timer_delay(&stm32_timer_delay);
+               pr_info("%s: STM32 delay timer registered\n", name);
        }
 
-       data->base = of_iomap(np, 0);
-       if (!data->base) {
-               ret = -ENXIO;
-               pr_err("failed to map registers for clockevent\n");
-               goto err_iomap;
-       }
+       return clocksource_mmio_init(timer_of_base(to) + TIM_CNT, name,
+                                    timer_of_rate(to), bits == 32 ? 250 : 100,
+                                    bits, clocksource_mmio_readl_up);
+}
 
-       irq = irq_of_parse_and_map(np, 0);
-       if (!irq) {
-               ret = -EINVAL;
-               pr_err("%pOF: failed to get irq.\n", np);
-               goto err_get_irq;
-       }
+static void __init stm32_clockevent_init(struct timer_of *to)
+{
+       u32 bits = stm32_timer_of_bits_get(to);
 
-       /* Detect whether the timer is 16 or 32 bits */
-       writel_relaxed(~0U, data->base + TIM_ARR);
-       max_delta = readl_relaxed(data->base + TIM_ARR);
-       if (max_delta == ~0U) {
-               prescaler = 1;
-               bits = 32;
-       } else {
-               prescaler = 1024;
-               bits = 16;
-       }
-       writel_relaxed(0, data->base + TIM_ARR);
+       to->clkevt.name = to->np->full_name;
+       to->clkevt.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT;
+       to->clkevt.set_state_shutdown = stm32_clock_event_shutdown;
+       to->clkevt.set_state_periodic = stm32_clock_event_set_periodic;
+       to->clkevt.set_state_oneshot = stm32_clock_event_set_oneshot;
+       to->clkevt.tick_resume = stm32_clock_event_shutdown;
+       to->clkevt.set_next_event = stm32_clock_event_set_next_event;
+       to->clkevt.rating = bits == 32 ? 250 : 100;
+
+       clockevents_config_and_register(&to->clkevt, timer_of_rate(to), 0x1,
+                                       (1 <<  bits) - 1);
+
+       pr_info("%pOF: STM32 clockevent driver initialized (%d bits)\n",
+               to->np, bits);
+}
+
+static int __init stm32_timer_init(struct device_node *node)
+{
+       struct reset_control *rstc;
+       struct timer_of *to;
+       int ret;
+
+       to = kzalloc(sizeof(*to), GFP_KERNEL);
+       if (!to)
+               return -ENOMEM;
 
-       writel_relaxed(prescaler - 1, data->base + TIM_PSC);
-       writel_relaxed(TIM_EGR_UG, data->base + TIM_EGR);
-       writel_relaxed(TIM_DIER_UIE, data->base + TIM_DIER);
-       writel_relaxed(0, data->base + TIM_SR);
+       to->flags = TIMER_OF_IRQ | TIMER_OF_CLOCK | TIMER_OF_BASE;
+       to->of_irq.handler = stm32_clock_event_handler;
 
-       data->periodic_top = DIV_ROUND_CLOSEST(rate, prescaler * HZ);
+       ret = timer_of_init(node, to);
+       if (ret)
+               goto err;
 
-       clockevents_config_and_register(&data->evtdev,
-                                       DIV_ROUND_CLOSEST(rate, prescaler),
-                                       0x1, max_delta);
+       to->private_data = kzalloc(sizeof(struct stm32_timer_private),
+                                  GFP_KERNEL);
+       if (!to->private_data)
+               goto deinit;
 
-       ret = request_irq(irq, stm32_clock_event_handler, IRQF_TIMER,
-                       "stm32 clockevent", data);
-       if (ret) {
-               pr_err("%pOF: failed to request irq.\n", np);
-               goto err_get_irq;
+       rstc = of_reset_control_get(node, NULL);
+       if (!IS_ERR(rstc)) {
+               reset_control_assert(rstc);
+               reset_control_deassert(rstc);
        }
 
-       pr_info("%pOF: STM32 clockevent driver initialized (%d bits)\n",
-                       np, bits);
+       stm32_timer_set_width(to);
 
-       return ret;
+       stm32_timer_set_prescaler(to);
+
+       ret = stm32_clocksource_init(to);
+       if (ret)
+               goto deinit;
+
+       stm32_clockevent_init(to);
+       return 0;
 
-err_get_irq:
-       iounmap(data->base);
-err_iomap:
-       clk_disable_unprepare(clk);
-err_clk_enable:
-       clk_put(clk);
-err_clk_get:
+deinit:
+       timer_of_cleanup(to);
+err:
+       kfree(to);
        return ret;
 }
 
-TIMER_OF_DECLARE(stm32, "st,stm32-timer", stm32_clockevent_init);
+TIMER_OF_DECLARE(stm32, "st,stm32-timer", stm32_timer_init);
index c5bce8e2298312bf2c73edc37f80af72b03ae34e..5780fba620ab13014369ec0304910d9ee317e1bf 100644 (file)
@@ -73,9 +73,7 @@ struct f_ncm {
        struct sk_buff                  *skb_tx_ndp;
        u16                             ndp_dgram_count;
        bool                            timer_force_tx;
-       struct tasklet_struct           tx_tasklet;
        struct hrtimer                  task_timer;
-
        bool                            timer_stopping;
 };
 
@@ -1104,7 +1102,7 @@ static struct sk_buff *ncm_wrap_ntb(struct gether *port,
 
                /* Delay the timer. */
                hrtimer_start(&ncm->task_timer, TX_TIMEOUT_NSECS,
-                             HRTIMER_MODE_REL);
+                             HRTIMER_MODE_REL_SOFT);
 
                /* Add the datagram position entries */
                ntb_ndp = skb_put_zero(ncm->skb_tx_ndp, dgram_idx_len);
@@ -1148,17 +1146,15 @@ err:
 }
 
 /*
- * This transmits the NTB if there are frames waiting.
+ * The transmit should only be run if no skb data has been sent
+ * for a certain duration.
  */
-static void ncm_tx_tasklet(unsigned long data)
+static enum hrtimer_restart ncm_tx_timeout(struct hrtimer *data)
 {
-       struct f_ncm    *ncm = (void *)data;
-
-       if (ncm->timer_stopping)
-               return;
+       struct f_ncm *ncm = container_of(data, struct f_ncm, task_timer);
 
        /* Only send if data is available. */
-       if (ncm->skb_tx_data) {
+       if (!ncm->timer_stopping && ncm->skb_tx_data) {
                ncm->timer_force_tx = true;
 
                /* XXX This allowance of a NULL skb argument to ndo_start_xmit
@@ -1171,16 +1167,6 @@ static void ncm_tx_tasklet(unsigned long data)
 
                ncm->timer_force_tx = false;
        }
-}
-
-/*
- * The transmit should only be run if no skb data has been sent
- * for a certain duration.
- */
-static enum hrtimer_restart ncm_tx_timeout(struct hrtimer *data)
-{
-       struct f_ncm *ncm = container_of(data, struct f_ncm, task_timer);
-       tasklet_schedule(&ncm->tx_tasklet);
        return HRTIMER_NORESTART;
 }
 
@@ -1513,8 +1499,7 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f)
        ncm->port.open = ncm_open;
        ncm->port.close = ncm_close;
 
-       tasklet_init(&ncm->tx_tasklet, ncm_tx_tasklet, (unsigned long) ncm);
-       hrtimer_init(&ncm->task_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+       hrtimer_init(&ncm->task_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
        ncm->task_timer.function = ncm_tx_timeout;
 
        DBG(cdev, "CDC Network: %s speed IN/%s OUT/%s NOTIFY/%s\n",
@@ -1623,7 +1608,6 @@ static void ncm_unbind(struct usb_configuration *c, struct usb_function *f)
        DBG(c->cdev, "ncm unbind\n");
 
        hrtimer_cancel(&ncm->task_timer);
-       tasklet_kill(&ncm->tx_tasklet);
 
        ncm_string_defs[0].id = 0;
        usb_free_all_descriptors(f);
index 012c37fdb688cb2c169530eb821c02abed2bcdd5..c7902ca7c9f4cc78aaa0ddce0684efbbe2ba1986 100644 (file)
@@ -28,13 +28,29 @@ struct hrtimer_cpu_base;
 
 /*
  * Mode arguments of xxx_hrtimer functions:
+ *
+ * HRTIMER_MODE_ABS            - Time value is absolute
+ * HRTIMER_MODE_REL            - Time value is relative to now
+ * HRTIMER_MODE_PINNED         - Timer is bound to CPU (is only considered
+ *                               when starting the timer)
+ * HRTIMER_MODE_SOFT           - Timer callback function will be executed in
+ *                               soft irq context
  */
 enum hrtimer_mode {
-       HRTIMER_MODE_ABS = 0x0,         /* Time value is absolute */
-       HRTIMER_MODE_REL = 0x1,         /* Time value is relative to now */
-       HRTIMER_MODE_PINNED = 0x02,     /* Timer is bound to CPU */
-       HRTIMER_MODE_ABS_PINNED = 0x02,
-       HRTIMER_MODE_REL_PINNED = 0x03,
+       HRTIMER_MODE_ABS        = 0x00,
+       HRTIMER_MODE_REL        = 0x01,
+       HRTIMER_MODE_PINNED     = 0x02,
+       HRTIMER_MODE_SOFT       = 0x04,
+
+       HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
+       HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
+
+       HRTIMER_MODE_ABS_SOFT   = HRTIMER_MODE_ABS | HRTIMER_MODE_SOFT,
+       HRTIMER_MODE_REL_SOFT   = HRTIMER_MODE_REL | HRTIMER_MODE_SOFT,
+
+       HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
+       HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
+
 };
 
 /*
@@ -87,6 +103,7 @@ enum hrtimer_restart {
  * @base:      pointer to the timer base (per cpu and per clock)
  * @state:     state information (See bit values above)
  * @is_rel:    Set if the timer was armed relative
+ * @is_soft:   Set if hrtimer will be expired in soft interrupt context.
  *
  * The hrtimer structure must be initialized by hrtimer_init()
  */
@@ -97,6 +114,7 @@ struct hrtimer {
        struct hrtimer_clock_base       *base;
        u8                              state;
        u8                              is_rel;
+       u8                              is_soft;
 };
 
 /**
@@ -112,9 +130,9 @@ struct hrtimer_sleeper {
 };
 
 #ifdef CONFIG_64BIT
-# define HRTIMER_CLOCK_BASE_ALIGN      64
+# define __hrtimer_clock_base_align    ____cacheline_aligned
 #else
-# define HRTIMER_CLOCK_BASE_ALIGN      32
+# define __hrtimer_clock_base_align
 #endif
 
 /**
@@ -123,48 +141,57 @@ struct hrtimer_sleeper {
  * @index:             clock type index for per_cpu support when moving a
  *                     timer to a base on another cpu.
  * @clockid:           clock id for per_cpu support
+ * @seq:               seqcount around __run_hrtimer
+ * @running:           pointer to the currently running hrtimer
  * @active:            red black tree root node for the active timers
  * @get_time:          function to retrieve the current time of the clock
  * @offset:            offset of this clock to the monotonic base
  */
 struct hrtimer_clock_base {
        struct hrtimer_cpu_base *cpu_base;
-       int                     index;
+       unsigned int            index;
        clockid_t               clockid;
+       seqcount_t              seq;
+       struct hrtimer          *running;
        struct timerqueue_head  active;
        ktime_t                 (*get_time)(void);
        ktime_t                 offset;
-} __attribute__((__aligned__(HRTIMER_CLOCK_BASE_ALIGN)));
+} __hrtimer_clock_base_align;
 
 enum  hrtimer_base_type {
        HRTIMER_BASE_MONOTONIC,
        HRTIMER_BASE_REALTIME,
        HRTIMER_BASE_BOOTTIME,
        HRTIMER_BASE_TAI,
+       HRTIMER_BASE_MONOTONIC_SOFT,
+       HRTIMER_BASE_REALTIME_SOFT,
+       HRTIMER_BASE_BOOTTIME_SOFT,
+       HRTIMER_BASE_TAI_SOFT,
        HRTIMER_MAX_CLOCK_BASES,
 };
 
-/*
+/**
  * struct hrtimer_cpu_base - the per cpu clock bases
  * @lock:              lock protecting the base and associated clock bases
  *                     and timers
- * @seq:               seqcount around __run_hrtimer
- * @running:           pointer to the currently running hrtimer
  * @cpu:               cpu number
  * @active_bases:      Bitfield to mark bases with active timers
  * @clock_was_set_seq: Sequence counter of clock was set events
- * @migration_enabled: The migration of hrtimers to other cpus is enabled
- * @nohz_active:       The nohz functionality is enabled
- * @expires_next:      absolute time of the next event which was scheduled
- *                     via clock_set_next_event()
- * @next_timer:                Pointer to the first expiring timer
- * @in_hrtirq:         hrtimer_interrupt() is currently executing
  * @hres_active:       State of high resolution mode
+ * @in_hrtirq:         hrtimer_interrupt() is currently executing
  * @hang_detected:     The last hrtimer interrupt detected a hang
+ * @softirq_activated: displays, if the softirq is raised - update of softirq
+ *                     related settings is not required then.
  * @nr_events:         Total number of hrtimer interrupt events
  * @nr_retries:                Total number of hrtimer interrupt retries
  * @nr_hangs:          Total number of hrtimer interrupt hangs
  * @max_hang_time:     Maximum time spent in hrtimer_interrupt
+ * @expires_next:      absolute time of the next event, is required for remote
+ *                     hrtimer enqueue; it is the total first expiry time (hard
+ *                     and soft hrtimer are taken into account)
+ * @next_timer:                Pointer to the first expiring timer
+ * @softirq_expires_next: Time to check, if soft queues needs also to be expired
+ * @softirq_next_timer: Pointer to the first expiring softirq based timer
  * @clock_base:                array of clock bases for this cpu
  *
  * Note: next_timer is just an optimization for __remove_hrtimer().
@@ -173,31 +200,28 @@ enum  hrtimer_base_type {
  */
 struct hrtimer_cpu_base {
        raw_spinlock_t                  lock;
-       seqcount_t                      seq;
-       struct hrtimer                  *running;
        unsigned int                    cpu;
        unsigned int                    active_bases;
        unsigned int                    clock_was_set_seq;
-       bool                            migration_enabled;
-       bool                            nohz_active;
+       unsigned int                    hres_active             : 1,
+                                       in_hrtirq               : 1,
+                                       hang_detected           : 1,
+                                       softirq_activated       : 1;
 #ifdef CONFIG_HIGH_RES_TIMERS
-       unsigned int                    in_hrtirq       : 1,
-                                       hres_active     : 1,
-                                       hang_detected   : 1;
-       ktime_t                         expires_next;
-       struct hrtimer                  *next_timer;
        unsigned int                    nr_events;
-       unsigned int                    nr_retries;
-       unsigned int                    nr_hangs;
+       unsigned short                  nr_retries;
+       unsigned short                  nr_hangs;
        unsigned int                    max_hang_time;
 #endif
+       ktime_t                         expires_next;
+       struct hrtimer                  *next_timer;
+       ktime_t                         softirq_expires_next;
+       struct hrtimer                  *softirq_next_timer;
        struct hrtimer_clock_base       clock_base[HRTIMER_MAX_CLOCK_BASES];
 } ____cacheline_aligned;
 
 static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
 {
-       BUILD_BUG_ON(sizeof(struct hrtimer_clock_base) > HRTIMER_CLOCK_BASE_ALIGN);
-
        timer->node.expires = time;
        timer->_softexpires = time;
 }
@@ -266,16 +290,17 @@ static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
        return timer->base->get_time();
 }
 
+static inline int hrtimer_is_hres_active(struct hrtimer *timer)
+{
+       return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
+               timer->base->cpu_base->hres_active : 0;
+}
+
 #ifdef CONFIG_HIGH_RES_TIMERS
 struct clock_event_device;
 
 extern void hrtimer_interrupt(struct clock_event_device *dev);
 
-static inline int hrtimer_is_hres_active(struct hrtimer *timer)
-{
-       return timer->base->cpu_base->hres_active;
-}
-
 /*
  * The resolution of the clocks. The resolution value is returned in
  * the clock_getres() system call to give application programmers an
@@ -298,11 +323,6 @@ extern unsigned int hrtimer_resolution;
 
 #define hrtimer_resolution     (unsigned int)LOW_RES_NSEC
 
-static inline int hrtimer_is_hres_active(struct hrtimer *timer)
-{
-       return 0;
-}
-
 static inline void clock_was_set_delayed(void) { }
 
 #endif
@@ -365,11 +385,12 @@ extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
                                   u64 range_ns, const enum hrtimer_mode mode);
 
 /**
- * hrtimer_start - (re)start an hrtimer on the current CPU
+ * hrtimer_start - (re)start an hrtimer
  * @timer:     the timer to be added
  * @tim:       expiry time
- * @mode:      expiry mode: absolute (HRTIMER_MODE_ABS) or
- *             relative (HRTIMER_MODE_REL)
+ * @mode:      timer mode: absolute (HRTIMER_MODE_ABS) or
+ *             relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
+ *             softirq based mode is considered for debug purpose only!
  */
 static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
                                 const enum hrtimer_mode mode)
@@ -422,7 +443,7 @@ static inline int hrtimer_is_queued(struct hrtimer *timer)
  */
 static inline int hrtimer_callback_running(struct hrtimer *timer)
 {
-       return timer->base->cpu_base->running == timer;
+       return timer->base->running == timer;
 }
 
 /* Forward a hrtimer so it expires after now: */
@@ -466,7 +487,7 @@ extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
 extern int schedule_hrtimeout_range_clock(ktime_t *expires,
                                          u64 delta,
                                          const enum hrtimer_mode mode,
-                                         int clock);
+                                         clockid_t clock_id);
 extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
 
 /* Soft interrupt function to run the hrtimer queues: */
index 672c4f32311e2f569e99fac56448009d5efbe3ac..c85704fcdbd2189b517f407ad871b02055924df5 100644 (file)
@@ -42,13 +42,26 @@ struct cpu_timer_list {
 #define CLOCKFD                        CPUCLOCK_MAX
 #define CLOCKFD_MASK           (CPUCLOCK_PERTHREAD_MASK|CPUCLOCK_CLOCK_MASK)
 
-#define MAKE_PROCESS_CPUCLOCK(pid, clock) \
-       ((~(clockid_t) (pid) << 3) | (clockid_t) (clock))
-#define MAKE_THREAD_CPUCLOCK(tid, clock) \
-       MAKE_PROCESS_CPUCLOCK((tid), (clock) | CPUCLOCK_PERTHREAD_MASK)
+static inline clockid_t make_process_cpuclock(const unsigned int pid,
+               const clockid_t clock)
+{
+       return ((~pid) << 3) | clock;
+}
+static inline clockid_t make_thread_cpuclock(const unsigned int tid,
+               const clockid_t clock)
+{
+       return make_process_cpuclock(tid, clock | CPUCLOCK_PERTHREAD_MASK);
+}
 
-#define FD_TO_CLOCKID(fd)      ((~(clockid_t) (fd) << 3) | CLOCKFD)
-#define CLOCKID_TO_FD(clk)     ((unsigned int) ~((clk) >> 3))
+static inline clockid_t fd_to_clockid(const int fd)
+{
+       return make_process_cpuclock((unsigned int) fd, CLOCKFD);
+}
+
+static inline int clockid_to_fd(const clockid_t clk)
+{
+       return ~(clk >> 3);
+}
 
 #define REQUEUE_PENDING 1
 
index 16e305e69f34ddf1d9a9ed87d70f7e716e052707..a57e4ee989d6229476471c5527a44b230014926c 100644 (file)
@@ -136,6 +136,24 @@ DEFINE_EVENT(timer_class, timer_cancel,
        TP_ARGS(timer)
 );
 
+#define decode_clockid(type)                                           \
+       __print_symbolic(type,                                          \
+               { CLOCK_REALTIME,       "CLOCK_REALTIME"        },      \
+               { CLOCK_MONOTONIC,      "CLOCK_MONOTONIC"       },      \
+               { CLOCK_BOOTTIME,       "CLOCK_BOOTTIME"        },      \
+               { CLOCK_TAI,            "CLOCK_TAI"             })
+
+#define decode_hrtimer_mode(mode)                                      \
+       __print_symbolic(mode,                                          \
+               { HRTIMER_MODE_ABS,             "ABS"           },      \
+               { HRTIMER_MODE_REL,             "REL"           },      \
+               { HRTIMER_MODE_ABS_PINNED,      "ABS|PINNED"    },      \
+               { HRTIMER_MODE_REL_PINNED,      "REL|PINNED"    },      \
+               { HRTIMER_MODE_ABS_SOFT,        "ABS|SOFT"      },      \
+               { HRTIMER_MODE_REL_SOFT,        "REL|SOFT"      },      \
+               { HRTIMER_MODE_ABS_PINNED_SOFT, "ABS|PINNED|SOFT" },    \
+               { HRTIMER_MODE_REL_PINNED_SOFT, "REL|PINNED|SOFT" })
+
 /**
  * hrtimer_init - called when the hrtimer is initialized
  * @hrtimer:   pointer to struct hrtimer
@@ -162,10 +180,8 @@ TRACE_EVENT(hrtimer_init,
        ),
 
        TP_printk("hrtimer=%p clockid=%s mode=%s", __entry->hrtimer,
-                 __entry->clockid == CLOCK_REALTIME ?
-                       "CLOCK_REALTIME" : "CLOCK_MONOTONIC",
-                 __entry->mode == HRTIMER_MODE_ABS ?
-                       "HRTIMER_MODE_ABS" : "HRTIMER_MODE_REL")
+                 decode_clockid(__entry->clockid),
+                 decode_hrtimer_mode(__entry->mode))
 );
 
 /**
@@ -174,15 +190,16 @@ TRACE_EVENT(hrtimer_init,
  */
 TRACE_EVENT(hrtimer_start,
 
-       TP_PROTO(struct hrtimer *hrtimer),
+       TP_PROTO(struct hrtimer *hrtimer, enum hrtimer_mode mode),
 
-       TP_ARGS(hrtimer),
+       TP_ARGS(hrtimer, mode),
 
        TP_STRUCT__entry(
                __field( void *,        hrtimer         )
                __field( void *,        function        )
                __field( s64,           expires         )
                __field( s64,           softexpires     )
+               __field( enum hrtimer_mode,     mode    )
        ),
 
        TP_fast_assign(
@@ -190,12 +207,14 @@ TRACE_EVENT(hrtimer_start,
                __entry->function       = hrtimer->function;
                __entry->expires        = hrtimer_get_expires(hrtimer);
                __entry->softexpires    = hrtimer_get_softexpires(hrtimer);
+               __entry->mode           = mode;
        ),
 
-       TP_printk("hrtimer=%p function=%pf expires=%llu softexpires=%llu",
-                 __entry->hrtimer, __entry->function,
+       TP_printk("hrtimer=%p function=%pf expires=%llu softexpires=%llu "
+                 "mode=%s", __entry->hrtimer, __entry->function,
                  (unsigned long long) __entry->expires,
-                 (unsigned long long) __entry->softexpires)
+                 (unsigned long long) __entry->softexpires,
+                 decode_hrtimer_mode(__entry->mode))
 );
 
 /**
index aa9d2a2b12109b9f40eb258b37f2c6abf867cdd3..ae0c8a411fe72bba2e98b54408bd86959e7d5093 100644 (file)
 
 #include "tick-internal.h"
 
+/*
+ * Masks for selecting the soft and hard context timers from
+ * cpu_base->active
+ */
+#define MASK_SHIFT             (HRTIMER_BASE_MONOTONIC_SOFT)
+#define HRTIMER_ACTIVE_HARD    ((1U << MASK_SHIFT) - 1)
+#define HRTIMER_ACTIVE_SOFT    (HRTIMER_ACTIVE_HARD << MASK_SHIFT)
+#define HRTIMER_ACTIVE_ALL     (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
+
 /*
  * The timer bases:
  *
@@ -70,7 +79,6 @@
 DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
 {
        .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
-       .seq = SEQCNT_ZERO(hrtimer_bases.seq),
        .clock_base =
        {
                {
@@ -93,6 +101,26 @@ DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
                        .clockid = CLOCK_TAI,
                        .get_time = &ktime_get_clocktai,
                },
+               {
+                       .index = HRTIMER_BASE_MONOTONIC_SOFT,
+                       .clockid = CLOCK_MONOTONIC,
+                       .get_time = &ktime_get,
+               },
+               {
+                       .index = HRTIMER_BASE_REALTIME_SOFT,
+                       .clockid = CLOCK_REALTIME,
+                       .get_time = &ktime_get_real,
+               },
+               {
+                       .index = HRTIMER_BASE_BOOTTIME_SOFT,
+                       .clockid = CLOCK_BOOTTIME,
+                       .get_time = &ktime_get_boottime,
+               },
+               {
+                       .index = HRTIMER_BASE_TAI_SOFT,
+                       .clockid = CLOCK_TAI,
+                       .get_time = &ktime_get_clocktai,
+               },
        }
 };
 
@@ -118,7 +146,6 @@ static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
  * timer->base->cpu_base
  */
 static struct hrtimer_cpu_base migration_cpu_base = {
-       .seq = SEQCNT_ZERO(migration_cpu_base),
        .clock_base = { { .cpu_base = &migration_cpu_base, }, },
 };
 
@@ -156,45 +183,33 @@ struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
 }
 
 /*
- * With HIGHRES=y we do not migrate the timer when it is expiring
- * before the next event on the target cpu because we cannot reprogram
- * the target cpu hardware and we would cause it to fire late.
+ * We do not migrate the timer when it is expiring before the next
+ * event on the target cpu. When high resolution is enabled, we cannot
+ * reprogram the target cpu hardware and we would cause it to fire
+ * late. To keep it simple, we handle the high resolution enabled and
+ * disabled case similar.
  *
  * Called with cpu_base->lock of target cpu held.
  */
 static int
 hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
 {
-#ifdef CONFIG_HIGH_RES_TIMERS
        ktime_t expires;
 
-       if (!new_base->cpu_base->hres_active)
-               return 0;
-
        expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
-       return expires <= new_base->cpu_base->expires_next;
-#else
-       return 0;
-#endif
+       return expires < new_base->cpu_base->expires_next;
 }
 
-#ifdef CONFIG_NO_HZ_COMMON
-static inline
-struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
-                                        int pinned)
-{
-       if (pinned || !base->migration_enabled)
-               return base;
-       return &per_cpu(hrtimer_bases, get_nohz_timer_target());
-}
-#else
 static inline
 struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
                                         int pinned)
 {
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+       if (static_branch_likely(&timers_migration_enabled) && !pinned)
+               return &per_cpu(hrtimer_bases, get_nohz_timer_target());
+#endif
        return base;
 }
-#endif
 
 /*
  * We switch the timer base to a power-optimized selected CPU target,
@@ -396,7 +411,8 @@ static inline void debug_hrtimer_init(struct hrtimer *timer)
        debug_object_init(timer, &hrtimer_debug_descr);
 }
 
-static inline void debug_hrtimer_activate(struct hrtimer *timer)
+static inline void debug_hrtimer_activate(struct hrtimer *timer,
+                                         enum hrtimer_mode mode)
 {
        debug_object_activate(timer, &hrtimer_debug_descr);
 }
@@ -429,8 +445,10 @@ void destroy_hrtimer_on_stack(struct hrtimer *timer)
 EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
 
 #else
+
 static inline void debug_hrtimer_init(struct hrtimer *timer) { }
-static inline void debug_hrtimer_activate(struct hrtimer *timer) { }
+static inline void debug_hrtimer_activate(struct hrtimer *timer,
+                                         enum hrtimer_mode mode) { }
 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
 #endif
 
@@ -442,10 +460,11 @@ debug_init(struct hrtimer *timer, clockid_t clockid,
        trace_hrtimer_init(timer, clockid, mode);
 }
 
-static inline void debug_activate(struct hrtimer *timer)
+static inline void debug_activate(struct hrtimer *timer,
+                                 enum hrtimer_mode mode)
 {
-       debug_hrtimer_activate(timer);
-       trace_hrtimer_start(timer);
+       debug_hrtimer_activate(timer, mode);
+       trace_hrtimer_start(timer, mode);
 }
 
 static inline void debug_deactivate(struct hrtimer *timer)
@@ -454,35 +473,43 @@ static inline void debug_deactivate(struct hrtimer *timer)
        trace_hrtimer_cancel(timer);
 }
 
-#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
-static inline void hrtimer_update_next_timer(struct hrtimer_cpu_base *cpu_base,
-                                            struct hrtimer *timer)
+static struct hrtimer_clock_base *
+__next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)
 {
-#ifdef CONFIG_HIGH_RES_TIMERS
-       cpu_base->next_timer = timer;
-#endif
+       unsigned int idx;
+
+       if (!*active)
+               return NULL;
+
+       idx = __ffs(*active);
+       *active &= ~(1U << idx);
+
+       return &cpu_base->clock_base[idx];
 }
 
-static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base)
+#define for_each_active_base(base, cpu_base, active)   \
+       while ((base = __next_base((cpu_base), &(active))))
+
+static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
+                                        unsigned int active,
+                                        ktime_t expires_next)
 {
-       struct hrtimer_clock_base *base = cpu_base->clock_base;
-       unsigned int active = cpu_base->active_bases;
-       ktime_t expires, expires_next = KTIME_MAX;
+       struct hrtimer_clock_base *base;
+       ktime_t expires;
 
-       hrtimer_update_next_timer(cpu_base, NULL);
-       for (; active; base++, active >>= 1) {
+       for_each_active_base(base, cpu_base, active) {
                struct timerqueue_node *next;
                struct hrtimer *timer;
 
-               if (!(active & 0x01))
-                       continue;
-
                next = timerqueue_getnext(&base->active);
                timer = container_of(next, struct hrtimer, node);
                expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
                if (expires < expires_next) {
                        expires_next = expires;
-                       hrtimer_update_next_timer(cpu_base, timer);
+                       if (timer->is_soft)
+                               cpu_base->softirq_next_timer = timer;
+                       else
+                               cpu_base->next_timer = timer;
                }
        }
        /*
@@ -494,7 +521,47 @@ static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base)
                expires_next = 0;
        return expires_next;
 }
-#endif
+
+/*
+ * Recomputes cpu_base::*next_timer and returns the earliest expires_next but
+ * does not set cpu_base::*expires_next, that is done by hrtimer_reprogram.
+ *
+ * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
+ * those timers will get run whenever the softirq gets handled, at the end of
+ * hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
+ *
+ * Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.
+ * The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual
+ * softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.
+ *
+ * @active_mask must be one of:
+ *  - HRTIMER_ACTIVE_ALL,
+ *  - HRTIMER_ACTIVE_SOFT, or
+ *  - HRTIMER_ACTIVE_HARD.
+ */
+static ktime_t
+__hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
+{
+       unsigned int active;
+       struct hrtimer *next_timer = NULL;
+       ktime_t expires_next = KTIME_MAX;
+
+       if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
+               active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
+               cpu_base->softirq_next_timer = NULL;
+               expires_next = __hrtimer_next_event_base(cpu_base, active, KTIME_MAX);
+
+               next_timer = cpu_base->softirq_next_timer;
+       }
+
+       if (active_mask & HRTIMER_ACTIVE_HARD) {
+               active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
+               cpu_base->next_timer = next_timer;
+               expires_next = __hrtimer_next_event_base(cpu_base, active, expires_next);
+       }
+
+       return expires_next;
+}
 
 static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
 {
@@ -502,36 +569,14 @@ static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
        ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
        ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
 
-       return ktime_get_update_offsets_now(&base->clock_was_set_seq,
+       ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,
                                            offs_real, offs_boot, offs_tai);
-}
-
-/* High resolution timer related functions */
-#ifdef CONFIG_HIGH_RES_TIMERS
 
-/*
- * High resolution timer enabled ?
- */
-static bool hrtimer_hres_enabled __read_mostly  = true;
-unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
-EXPORT_SYMBOL_GPL(hrtimer_resolution);
+       base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
+       base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
+       base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
 
-/*
- * Enable / Disable high resolution mode
- */
-static int __init setup_hrtimer_hres(char *str)
-{
-       return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
-}
-
-__setup("highres=", setup_hrtimer_hres);
-
-/*
- * hrtimer_high_res_enabled - query, if the highres mode is enabled
- */
-static inline int hrtimer_is_hres_enabled(void)
-{
-       return hrtimer_hres_enabled;
+       return now;
 }
 
 /*
@@ -539,7 +584,8 @@ static inline int hrtimer_is_hres_enabled(void)
  */
 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
 {
-       return cpu_base->hres_active;
+       return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
+               cpu_base->hres_active : 0;
 }
 
 static inline int hrtimer_hres_active(void)
@@ -557,10 +603,23 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
 {
        ktime_t expires_next;
 
-       if (!cpu_base->hres_active)
-               return;
+       /*
+        * Find the current next expiration time.
+        */
+       expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
 
-       expires_next = __hrtimer_get_next_event(cpu_base);
+       if (cpu_base->next_timer && cpu_base->next_timer->is_soft) {
+               /*
+                * When the softirq is activated, hrtimer has to be
+                * programmed with the first hard hrtimer because soft
+                * timer interrupt could occur too late.
+                */
+               if (cpu_base->softirq_activated)
+                       expires_next = __hrtimer_get_next_event(cpu_base,
+                                                               HRTIMER_ACTIVE_HARD);
+               else
+                       cpu_base->softirq_expires_next = expires_next;
+       }
 
        if (skip_equal && expires_next == cpu_base->expires_next)
                return;
@@ -568,6 +627,9 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
        cpu_base->expires_next = expires_next;
 
        /*
+        * If hres is not active, hardware does not have to be
+        * reprogrammed yet.
+        *
         * If a hang was detected in the last timer interrupt then we
         * leave the hang delay active in the hardware. We want the
         * system to make progress. That also prevents the following
@@ -581,83 +643,38 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
         * set. So we'd effectivly block all timers until the T2 event
         * fires.
         */
-       if (cpu_base->hang_detected)
+       if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
                return;
 
        tick_program_event(cpu_base->expires_next, 1);
 }
 
+/* High resolution timer related functions */
+#ifdef CONFIG_HIGH_RES_TIMERS
+
 /*
- * When a timer is enqueued and expires earlier than the already enqueued
- * timers, we have to check, whether it expires earlier than the timer for
- * which the clock event device was armed.
- *
- * Called with interrupts disabled and base->cpu_base.lock held
+ * High resolution timer enabled ?
  */
-static void hrtimer_reprogram(struct hrtimer *timer,
-                             struct hrtimer_clock_base *base)
-{
-       struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
-       ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
-
-       WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
-
-       /*
-        * If the timer is not on the current cpu, we cannot reprogram
-        * the other cpus clock event device.
-        */
-       if (base->cpu_base != cpu_base)
-               return;
-
-       /*
-        * If the hrtimer interrupt is running, then it will
-        * reevaluate the clock bases and reprogram the clock event
-        * device. The callbacks are always executed in hard interrupt
-        * context so we don't need an extra check for a running
-        * callback.
-        */
-       if (cpu_base->in_hrtirq)
-               return;
-
-       /*
-        * CLOCK_REALTIME timer might be requested with an absolute
-        * expiry time which is less than base->offset. Set it to 0.
-        */
-       if (expires < 0)
-               expires = 0;
-
-       if (expires >= cpu_base->expires_next)
-               return;
-
-       /* Update the pointer to the next expiring timer */
-       cpu_base->next_timer = timer;
-
-       /*
-        * If a hang was detected in the last timer interrupt then we
-        * do not schedule a timer which is earlier than the expiry
-        * which we enforced in the hang detection. We want the system
-        * to make progress.
-        */
-       if (cpu_base->hang_detected)
-               return;
+static bool hrtimer_hres_enabled __read_mostly  = true;
+unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
+EXPORT_SYMBOL_GPL(hrtimer_resolution);
 
-       /*
-        * Program the timer hardware. We enforce the expiry for
-        * events which are already in the past.
-        */
-       cpu_base->expires_next = expires;
-       tick_program_event(expires, 1);
+/*
+ * Enable / Disable high resolution mode
+ */
+static int __init setup_hrtimer_hres(char *str)
+{
+       return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
 }
 
+__setup("highres=", setup_hrtimer_hres);
+
 /*
- * Initialize the high resolution related parts of cpu_base
+ * hrtimer_high_res_enabled - query, if the highres mode is enabled
  */
-static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base)
+static inline int hrtimer_is_hres_enabled(void)
 {
-       base->expires_next = KTIME_MAX;
-       base->hang_detected = 0;
-       base->hres_active = 0;
-       base->next_timer = NULL;
+       return hrtimer_hres_enabled;
 }
 
 /*
@@ -669,7 +686,7 @@ static void retrigger_next_event(void *arg)
 {
        struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
 
-       if (!base->hres_active)
+       if (!__hrtimer_hres_active(base))
                return;
 
        raw_spin_lock(&base->lock);
@@ -716,22 +733,101 @@ void clock_was_set_delayed(void)
 
 #else
 
-static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *b) { return 0; }
-static inline int hrtimer_hres_active(void) { return 0; }
 static inline int hrtimer_is_hres_enabled(void) { return 0; }
 static inline void hrtimer_switch_to_hres(void) { }
-static inline void
-hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { }
-static inline int hrtimer_reprogram(struct hrtimer *timer,
-                                   struct hrtimer_clock_base *base)
-{
-       return 0;
-}
-static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
 static inline void retrigger_next_event(void *arg) { }
 
 #endif /* CONFIG_HIGH_RES_TIMERS */
 
+/*
+ * When a timer is enqueued and expires earlier than the already enqueued
+ * timers, we have to check, whether it expires earlier than the timer for
+ * which the clock event device was armed.
+ *
+ * Called with interrupts disabled and base->cpu_base.lock held
+ */
+static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
+{
+       struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
+       struct hrtimer_clock_base *base = timer->base;
+       ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
+
+       WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
+
+       /*
+        * CLOCK_REALTIME timer might be requested with an absolute
+        * expiry time which is less than base->offset. Set it to 0.
+        */
+       if (expires < 0)
+               expires = 0;
+
+       if (timer->is_soft) {
+               /*
+                * soft hrtimer could be started on a remote CPU. In this
+                * case softirq_expires_next needs to be updated on the
+                * remote CPU. The soft hrtimer will not expire before the
+                * first hard hrtimer on the remote CPU -
+                * hrtimer_check_target() prevents this case.
+                */
+               struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
+
+               if (timer_cpu_base->softirq_activated)
+                       return;
+
+               if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))
+                       return;
+
+               timer_cpu_base->softirq_next_timer = timer;
+               timer_cpu_base->softirq_expires_next = expires;
+
+               if (!ktime_before(expires, timer_cpu_base->expires_next) ||
+                   !reprogram)
+                       return;
+       }
+
+       /*
+        * If the timer is not on the current cpu, we cannot reprogram
+        * the other cpus clock event device.
+        */
+       if (base->cpu_base != cpu_base)
+               return;
+
+       /*
+        * If the hrtimer interrupt is running, then it will
+        * reevaluate the clock bases and reprogram the clock event
+        * device. The callbacks are always executed in hard interrupt
+        * context so we don't need an extra check for a running
+        * callback.
+        */
+       if (cpu_base->in_hrtirq)
+               return;
+
+       if (expires >= cpu_base->expires_next)
+               return;
+
+       /* Update the pointer to the next expiring timer */
+       cpu_base->next_timer = timer;
+       cpu_base->expires_next = expires;
+
+       /*
+        * If hres is not active, hardware does not have to be
+        * programmed yet.
+        *
+        * If a hang was detected in the last timer interrupt then we
+        * do not schedule a timer which is earlier than the expiry
+        * which we enforced in the hang detection. We want the system
+        * to make progress.
+        */
+       if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
+               return;
+
+       /*
+        * Program the timer hardware. We enforce the expiry for
+        * events which are already in the past.
+        */
+       tick_program_event(expires, 1);
+}
+
 /*
  * Clock realtime was set
  *
@@ -837,9 +933,10 @@ EXPORT_SYMBOL_GPL(hrtimer_forward);
  * Returns 1 when the new timer is the leftmost timer in the tree.
  */
 static int enqueue_hrtimer(struct hrtimer *timer,
-                          struct hrtimer_clock_base *base)
+                          struct hrtimer_clock_base *base,
+                          enum hrtimer_mode mode)
 {
-       debug_activate(timer);
+       debug_activate(timer, mode);
 
        base->cpu_base->active_bases |= 1 << base->index;
 
@@ -872,7 +969,6 @@ static void __remove_hrtimer(struct hrtimer *timer,
        if (!timerqueue_del(&base->active, &timer->node))
                cpu_base->active_bases &= ~(1 << base->index);
 
-#ifdef CONFIG_HIGH_RES_TIMERS
        /*
         * Note: If reprogram is false we do not update
         * cpu_base->next_timer. This happens when we remove the first
@@ -883,7 +979,6 @@ static void __remove_hrtimer(struct hrtimer *timer,
         */
        if (reprogram && timer == cpu_base->next_timer)
                hrtimer_force_reprogram(cpu_base, 1);
-#endif
 }
 
 /*
@@ -932,22 +1027,36 @@ static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
        return tim;
 }
 
-/**
- * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU
- * @timer:     the timer to be added
- * @tim:       expiry time
- * @delta_ns:  "slack" range for the timer
- * @mode:      expiry mode: absolute (HRTIMER_MODE_ABS) or
- *             relative (HRTIMER_MODE_REL)
- */
-void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
-                           u64 delta_ns, const enum hrtimer_mode mode)
+static void
+hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
 {
-       struct hrtimer_clock_base *base, *new_base;
-       unsigned long flags;
-       int leftmost;
+       ktime_t expires;
 
-       base = lock_hrtimer_base(timer, &flags);
+       /*
+        * Find the next SOFT expiration.
+        */
+       expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
+
+       /*
+        * reprogramming needs to be triggered, even if the next soft
+        * hrtimer expires at the same time than the next hard
+        * hrtimer. cpu_base->softirq_expires_next needs to be updated!
+        */
+       if (expires == KTIME_MAX)
+               return;
+
+       /*
+        * cpu_base->*next_timer is recomputed by __hrtimer_get_next_event()
+        * cpu_base->*expires_next is only set by hrtimer_reprogram()
+        */
+       hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
+}
+
+static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
+                                   u64 delta_ns, const enum hrtimer_mode mode,
+                                   struct hrtimer_clock_base *base)
+{
+       struct hrtimer_clock_base *new_base;
 
        /* Remove an active timer from the queue: */
        remove_hrtimer(timer, base, true);
@@ -962,21 +1071,35 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
        /* Switch the timer base, if necessary: */
        new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);
 
-       leftmost = enqueue_hrtimer(timer, new_base);
-       if (!leftmost)
-               goto unlock;
+       return enqueue_hrtimer(timer, new_base, mode);
+}
+
+/**
+ * hrtimer_start_range_ns - (re)start an hrtimer
+ * @timer:     the timer to be added
+ * @tim:       expiry time
+ * @delta_ns:  "slack" range for the timer
+ * @mode:      timer mode: absolute (HRTIMER_MODE_ABS) or
+ *             relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
+ *             softirq based mode is considered for debug purpose only!
+ */
+void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
+                           u64 delta_ns, const enum hrtimer_mode mode)
+{
+       struct hrtimer_clock_base *base;
+       unsigned long flags;
+
+       /*
+        * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
+        * match.
+        */
+       WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
+
+       base = lock_hrtimer_base(timer, &flags);
+
+       if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
+               hrtimer_reprogram(timer, true);
 
-       if (!hrtimer_is_hres_active(timer)) {
-               /*
-                * Kick to reschedule the next tick to handle the new timer
-                * on dynticks target.
-                */
-               if (new_base->cpu_base->nohz_active)
-                       wake_up_nohz_cpu(new_base->cpu_base->cpu);
-       } else {
-               hrtimer_reprogram(timer, new_base);
-       }
-unlock:
        unlock_hrtimer_base(timer, &flags);
 }
 EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
@@ -1074,7 +1197,7 @@ u64 hrtimer_get_next_event(void)
        raw_spin_lock_irqsave(&cpu_base->lock, flags);
 
        if (!__hrtimer_hres_active(cpu_base))
-               expires = __hrtimer_get_next_event(cpu_base);
+               expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
 
        raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
 
@@ -1097,17 +1220,24 @@ static inline int hrtimer_clockid_to_base(clockid_t clock_id)
 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
                           enum hrtimer_mode mode)
 {
+       bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
+       int base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
        struct hrtimer_cpu_base *cpu_base;
-       int base;
 
        memset(timer, 0, sizeof(struct hrtimer));
 
        cpu_base = raw_cpu_ptr(&hrtimer_bases);
 
-       if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS)
+       /*
+        * POSIX magic: Relative CLOCK_REALTIME timers are not affected by
+        * clock modifications, so they needs to become CLOCK_MONOTONIC to
+        * ensure POSIX compliance.
+        */
+       if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
                clock_id = CLOCK_MONOTONIC;
 
-       base = hrtimer_clockid_to_base(clock_id);
+       base += hrtimer_clockid_to_base(clock_id);
+       timer->is_soft = softtimer;
        timer->base = &cpu_base->clock_base[base];
        timerqueue_init(&timer->node);
 }
@@ -1116,7 +1246,13 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
  * hrtimer_init - initialize a timer to the given clock
  * @timer:     the timer to be initialized
  * @clock_id:  the clock to be used
- * @mode:      timer mode abs/rel
+ * @mode:       The modes which are relevant for intitialization:
+ *              HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
+ *              HRTIMER_MODE_REL_SOFT
+ *
+ *              The PINNED variants of the above can be handed in,
+ *              but the PINNED bit is ignored as pinning happens
+ *              when the hrtimer is started
  */
 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
                  enum hrtimer_mode mode)
@@ -1135,19 +1271,19 @@ EXPORT_SYMBOL_GPL(hrtimer_init);
  */
 bool hrtimer_active(const struct hrtimer *timer)
 {
-       struct hrtimer_cpu_base *cpu_base;
+       struct hrtimer_clock_base *base;
        unsigned int seq;
 
        do {
-               cpu_base = READ_ONCE(timer->base->cpu_base);
-               seq = raw_read_seqcount_begin(&cpu_base->seq);
+               base = READ_ONCE(timer->base);
+               seq = raw_read_seqcount_begin(&base->seq);
 
                if (timer->state != HRTIMER_STATE_INACTIVE ||
-                   cpu_base->running == timer)
+                   base->running == timer)
                        return true;
 
-       } while (read_seqcount_retry(&cpu_base->seq, seq) ||
-                cpu_base != READ_ONCE(timer->base->cpu_base));
+       } while (read_seqcount_retry(&base->seq, seq) ||
+                base != READ_ONCE(timer->base));
 
        return false;
 }
@@ -1173,7 +1309,8 @@ EXPORT_SYMBOL_GPL(hrtimer_active);
 
 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
                          struct hrtimer_clock_base *base,
-                         struct hrtimer *timer, ktime_t *now)
+                         struct hrtimer *timer, ktime_t *now,
+                         unsigned long flags)
 {
        enum hrtimer_restart (*fn)(struct hrtimer *);
        int restart;
@@ -1181,16 +1318,16 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
        lockdep_assert_held(&cpu_base->lock);
 
        debug_deactivate(timer);
-       cpu_base->running = timer;
+       base->running = timer;
 
        /*
         * Separate the ->running assignment from the ->state assignment.
         *
         * As with a regular write barrier, this ensures the read side in
-        * hrtimer_active() cannot observe cpu_base->running == NULL &&
+        * hrtimer_active() cannot observe base->running == NULL &&
         * timer->state == INACTIVE.
         */
-       raw_write_seqcount_barrier(&cpu_base->seq);
+       raw_write_seqcount_barrier(&base->seq);
 
        __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
        fn = timer->function;
@@ -1204,15 +1341,15 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
                timer->is_rel = false;
 
        /*
-        * Because we run timers from hardirq context, there is no chance
-        * they get migrated to another cpu, therefore its safe to unlock
-        * the timer base.
+        * The timer is marked as running in the CPU base, so it is
+        * protected against migration to a different CPU even if the lock
+        * is dropped.
         */
-       raw_spin_unlock(&cpu_base->lock);
+       raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
        trace_hrtimer_expire_entry(timer, now);
        restart = fn(timer);
        trace_hrtimer_expire_exit(timer);
-       raw_spin_lock(&cpu_base->lock);
+       raw_spin_lock_irq(&cpu_base->lock);
 
        /*
         * Note: We clear the running state after enqueue_hrtimer and
@@ -1225,33 +1362,31 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
         */
        if (restart != HRTIMER_NORESTART &&
            !(timer->state & HRTIMER_STATE_ENQUEUED))
-               enqueue_hrtimer(timer, base);
+               enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);
 
        /*
         * Separate the ->running assignment from the ->state assignment.
         *
         * As with a regular write barrier, this ensures the read side in
-        * hrtimer_active() cannot observe cpu_base->running == NULL &&
+        * hrtimer_active() cannot observe base->running.timer == NULL &&
         * timer->state == INACTIVE.
         */
-       raw_write_seqcount_barrier(&cpu_base->seq);
+       raw_write_seqcount_barrier(&base->seq);
 
-       WARN_ON_ONCE(cpu_base->running != timer);
-       cpu_base->running = NULL;
+       WARN_ON_ONCE(base->running != timer);
+       base->running = NULL;
 }
 
-static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now)
+static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
+                                unsigned long flags, unsigned int active_mask)
 {
-       struct hrtimer_clock_base *base = cpu_base->clock_base;
-       unsigned int active = cpu_base->active_bases;
+       struct hrtimer_clock_base *base;
+       unsigned int active = cpu_base->active_bases & active_mask;
 
-       for (; active; base++, active >>= 1) {
+       for_each_active_base(base, cpu_base, active) {
                struct timerqueue_node *node;
                ktime_t basenow;
 
-               if (!(active & 0x01))
-                       continue;
-
                basenow = ktime_add(now, base->offset);
 
                while ((node = timerqueue_getnext(&base->active))) {
@@ -1274,11 +1409,28 @@ static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now)
                        if (basenow < hrtimer_get_softexpires_tv64(timer))
                                break;
 
-                       __run_hrtimer(cpu_base, base, timer, &basenow);
+                       __run_hrtimer(cpu_base, base, timer, &basenow, flags);
                }
        }
 }
 
+static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
+{
+       struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
+       unsigned long flags;
+       ktime_t now;
+
+       raw_spin_lock_irqsave(&cpu_base->lock, flags);
+
+       now = hrtimer_update_base(cpu_base);
+       __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
+
+       cpu_base->softirq_activated = 0;
+       hrtimer_update_softirq_timer(cpu_base, true);
+
+       raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
+}
+
 #ifdef CONFIG_HIGH_RES_TIMERS
 
 /*
@@ -1289,13 +1441,14 @@ void hrtimer_interrupt(struct clock_event_device *dev)
 {
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
        ktime_t expires_next, now, entry_time, delta;
+       unsigned long flags;
        int retries = 0;
 
        BUG_ON(!cpu_base->hres_active);
        cpu_base->nr_events++;
        dev->next_event = KTIME_MAX;
 
-       raw_spin_lock(&cpu_base->lock);
+       raw_spin_lock_irqsave(&cpu_base->lock, flags);
        entry_time = now = hrtimer_update_base(cpu_base);
 retry:
        cpu_base->in_hrtirq = 1;
@@ -1308,17 +1461,23 @@ retry:
         */
        cpu_base->expires_next = KTIME_MAX;
 
-       __hrtimer_run_queues(cpu_base, now);
+       if (!ktime_before(now, cpu_base->softirq_expires_next)) {
+               cpu_base->softirq_expires_next = KTIME_MAX;
+               cpu_base->softirq_activated = 1;
+               raise_softirq_irqoff(HRTIMER_SOFTIRQ);
+       }
+
+       __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
 
        /* Reevaluate the clock bases for the next expiry */
-       expires_next = __hrtimer_get_next_event(cpu_base);
+       expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
        /*
         * Store the new expiry value so the migration code can verify
         * against it.
         */
        cpu_base->expires_next = expires_next;
        cpu_base->in_hrtirq = 0;
-       raw_spin_unlock(&cpu_base->lock);
+       raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
 
        /* Reprogramming necessary ? */
        if (!tick_program_event(expires_next, 0)) {
@@ -1339,7 +1498,7 @@ retry:
         * Acquire base lock for updating the offsets and retrieving
         * the current time.
         */
-       raw_spin_lock(&cpu_base->lock);
+       raw_spin_lock_irqsave(&cpu_base->lock, flags);
        now = hrtimer_update_base(cpu_base);
        cpu_base->nr_retries++;
        if (++retries < 3)
@@ -1352,7 +1511,8 @@ retry:
         */
        cpu_base->nr_hangs++;
        cpu_base->hang_detected = 1;
-       raw_spin_unlock(&cpu_base->lock);
+       raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
+
        delta = ktime_sub(now, entry_time);
        if ((unsigned int)delta > cpu_base->max_hang_time)
                cpu_base->max_hang_time = (unsigned int) delta;
@@ -1394,6 +1554,7 @@ static inline void __hrtimer_peek_ahead_timers(void) { }
 void hrtimer_run_queues(void)
 {
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
+       unsigned long flags;
        ktime_t now;
 
        if (__hrtimer_hres_active(cpu_base))
@@ -1411,10 +1572,17 @@ void hrtimer_run_queues(void)
                return;
        }
 
-       raw_spin_lock(&cpu_base->lock);
+       raw_spin_lock_irqsave(&cpu_base->lock, flags);
        now = hrtimer_update_base(cpu_base);
-       __hrtimer_run_queues(cpu_base, now);
-       raw_spin_unlock(&cpu_base->lock);
+
+       if (!ktime_before(now, cpu_base->softirq_expires_next)) {
+               cpu_base->softirq_expires_next = KTIME_MAX;
+               cpu_base->softirq_activated = 1;
+               raise_softirq_irqoff(HRTIMER_SOFTIRQ);
+       }
+
+       __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
+       raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
 }
 
 /*
@@ -1591,9 +1759,14 @@ int hrtimers_prepare_cpu(unsigned int cpu)
                timerqueue_init_head(&cpu_base->clock_base[i].active);
        }
 
-       cpu_base->active_bases = 0;
        cpu_base->cpu = cpu;
-       hrtimer_init_hres(cpu_base);
+       cpu_base->active_bases = 0;
+       cpu_base->hres_active = 0;
+       cpu_base->hang_detected = 0;
+       cpu_base->next_timer = NULL;
+       cpu_base->softirq_next_timer = NULL;
+       cpu_base->expires_next = KTIME_MAX;
+       cpu_base->softirq_expires_next = KTIME_MAX;
        return 0;
 }
 
@@ -1625,7 +1798,7 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
                 * sort out already expired timers and reprogram the
                 * event device.
                 */
-               enqueue_hrtimer(timer, new_base);
+               enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);
        }
 }
 
@@ -1637,6 +1810,12 @@ int hrtimers_dead_cpu(unsigned int scpu)
        BUG_ON(cpu_online(scpu));
        tick_cancel_sched_timer(scpu);
 
+       /*
+        * this BH disable ensures that raise_softirq_irqoff() does
+        * not wakeup ksoftirqd (and acquire the pi-lock) while
+        * holding the cpu_base lock
+        */
+       local_bh_disable();
        local_irq_disable();
        old_base = &per_cpu(hrtimer_bases, scpu);
        new_base = this_cpu_ptr(&hrtimer_bases);
@@ -1652,12 +1831,19 @@ int hrtimers_dead_cpu(unsigned int scpu)
                                     &new_base->clock_base[i]);
        }
 
+       /*
+        * The migration might have changed the first expiring softirq
+        * timer on this CPU. Update it.
+        */
+       hrtimer_update_softirq_timer(new_base, false);
+
        raw_spin_unlock(&old_base->lock);
        raw_spin_unlock(&new_base->lock);
 
        /* Check, if we got expired work to do */
        __hrtimer_peek_ahead_timers();
        local_irq_enable();
+       local_bh_enable();
        return 0;
 }
 
@@ -1666,18 +1852,19 @@ int hrtimers_dead_cpu(unsigned int scpu)
 void __init hrtimers_init(void)
 {
        hrtimers_prepare_cpu(smp_processor_id());
+       open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
 }
 
 /**
  * schedule_hrtimeout_range_clock - sleep until timeout
  * @expires:   timeout value (ktime_t)
  * @delta:     slack in expires timeout (ktime_t)
- * @mode:      timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
- * @clock:     timer clock, CLOCK_MONOTONIC or CLOCK_REALTIME
+ * @mode:      timer mode
+ * @clock_id:  timer clock to be used
  */
 int __sched
 schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
-                              const enum hrtimer_mode mode, int clock)
+                              const enum hrtimer_mode mode, clockid_t clock_id)
 {
        struct hrtimer_sleeper t;
 
@@ -1698,7 +1885,7 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
                return -EINTR;
        }
 
-       hrtimer_init_on_stack(&t.timer, clock, mode);
+       hrtimer_init_on_stack(&t.timer, clock_id, mode);
        hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
 
        hrtimer_init_sleeper(&t, current);
@@ -1720,7 +1907,7 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
  * schedule_hrtimeout_range - sleep until timeout
  * @expires:   timeout value (ktime_t)
  * @delta:     slack in expires timeout (ktime_t)
- * @mode:      timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
+ * @mode:      timer mode
  *
  * Make the current task sleep until the given expiry time has
  * elapsed. The routine will return immediately unless
@@ -1759,7 +1946,7 @@ EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
 /**
  * schedule_hrtimeout - sleep until timeout
  * @expires:   timeout value (ktime_t)
- * @mode:      timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
+ * @mode:      timer mode
  *
  * Make the current task sleep until the given expiry time has
  * elapsed. The routine will return immediately unless
index 17cdc554c9fe82c56e9d8a5b68c48715425dcd09..cc91d90abd84d78b4af473e152b7d7d62f85796b 100644 (file)
@@ -216,7 +216,7 @@ struct posix_clock_desc {
 
 static int get_clock_desc(const clockid_t id, struct posix_clock_desc *cd)
 {
-       struct file *fp = fget(CLOCKID_TO_FD(id));
+       struct file *fp = fget(clockid_to_fd(id));
        int err = -EINVAL;
 
        if (!fp)
index 1f27887aa19487e82950498485bc83a295098104..ec9f5da6f163d40d3b2df9f4b9f76e783b304543 100644 (file)
@@ -1189,9 +1189,8 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx,
        u64 now;
 
        WARN_ON_ONCE(clock_idx == CPUCLOCK_SCHED);
-       cpu_timer_sample_group(clock_idx, tsk, &now);
 
-       if (oldval) {
+       if (oldval && cpu_timer_sample_group(clock_idx, tsk, &now) != -EINVAL) {
                /*
                 * We are setting itimer. The *oldval is absolute and we update
                 * it to be relative, *newval argument is relative and we update
@@ -1363,8 +1362,8 @@ static long posix_cpu_nsleep_restart(struct restart_block *restart_block)
        return do_cpu_nanosleep(which_clock, TIMER_ABSTIME, &t);
 }
 
-#define PROCESS_CLOCK  MAKE_PROCESS_CPUCLOCK(0, CPUCLOCK_SCHED)
-#define THREAD_CLOCK   MAKE_THREAD_CPUCLOCK(0, CPUCLOCK_SCHED)
+#define PROCESS_CLOCK  make_process_cpuclock(0, CPUCLOCK_SCHED)
+#define THREAD_CLOCK   make_thread_cpuclock(0, CPUCLOCK_SCHED)
 
 static int process_cpu_clock_getres(const clockid_t which_clock,
                                    struct timespec64 *tp)
index f8e1845aa464b2c0b11dead0ad9baa37cae62822..e277284c2831c9c1dae2219c1a135e1f5dc8945d 100644 (file)
@@ -150,16 +150,15 @@ static inline void tick_nohz_init(void) { }
 
 #ifdef CONFIG_NO_HZ_COMMON
 extern unsigned long tick_nohz_active;
-#else
+extern void timers_update_nohz(void);
+# ifdef CONFIG_SMP
+extern struct static_key_false timers_migration_enabled;
+# endif
+#else /* CONFIG_NO_HZ_COMMON */
+static inline void timers_update_nohz(void) { }
 #define tick_nohz_active (0)
 #endif
 
-#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
-extern void timers_update_migration(bool update_nohz);
-#else
-static inline void timers_update_migration(bool update_nohz) { }
-#endif
-
 DECLARE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases);
 
 extern u64 get_next_timer_interrupt(unsigned long basej, u64 basem);
index f7cc7abfcf252f0cb6fadb3b9437b9fd096b8baf..29a5733eff83ec9e40432a2e9353bf19253d10b7 100644 (file)
@@ -1107,7 +1107,7 @@ static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
        ts->nohz_mode = mode;
        /* One update is enough */
        if (!test_and_set_bit(0, &tick_nohz_active))
-               timers_update_migration(true);
+               timers_update_nohz();
 }
 
 /**
index 0bcf00e3ce482b17392e30904decf86d459c4928..48150ab42de96449051b5e441cb405ef1a09445c 100644 (file)
@@ -200,8 +200,6 @@ struct timer_base {
        unsigned long           clk;
        unsigned long           next_expiry;
        unsigned int            cpu;
-       bool                    migration_enabled;
-       bool                    nohz_active;
        bool                    is_idle;
        bool                    must_forward_clk;
        DECLARE_BITMAP(pending_map, WHEEL_SIZE);
@@ -210,45 +208,64 @@ struct timer_base {
 
 static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]);
 
-#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+#ifdef CONFIG_NO_HZ_COMMON
+
+static DEFINE_STATIC_KEY_FALSE(timers_nohz_active);
+static DEFINE_MUTEX(timer_keys_mutex);
+
+static void timer_update_keys(struct work_struct *work);
+static DECLARE_WORK(timer_update_work, timer_update_keys);
+
+#ifdef CONFIG_SMP
 unsigned int sysctl_timer_migration = 1;
 
-void timers_update_migration(bool update_nohz)
+DEFINE_STATIC_KEY_FALSE(timers_migration_enabled);
+
+static void timers_update_migration(void)
 {
-       bool on = sysctl_timer_migration && tick_nohz_active;
-       unsigned int cpu;
+       if (sysctl_timer_migration && tick_nohz_active)
+               static_branch_enable(&timers_migration_enabled);
+       else
+               static_branch_disable(&timers_migration_enabled);
+}
+#else
+static inline void timers_update_migration(void) { }
+#endif /* !CONFIG_SMP */
 
-       /* Avoid the loop, if nothing to update */
-       if (this_cpu_read(timer_bases[BASE_STD].migration_enabled) == on)
-               return;
+static void timer_update_keys(struct work_struct *work)
+{
+       mutex_lock(&timer_keys_mutex);
+       timers_update_migration();
+       static_branch_enable(&timers_nohz_active);
+       mutex_unlock(&timer_keys_mutex);
+}
 
-       for_each_possible_cpu(cpu) {
-               per_cpu(timer_bases[BASE_STD].migration_enabled, cpu) = on;
-               per_cpu(timer_bases[BASE_DEF].migration_enabled, cpu) = on;
-               per_cpu(hrtimer_bases.migration_enabled, cpu) = on;
-               if (!update_nohz)
-                       continue;
-               per_cpu(timer_bases[BASE_STD].nohz_active, cpu) = true;
-               per_cpu(timer_bases[BASE_DEF].nohz_active, cpu) = true;
-               per_cpu(hrtimer_bases.nohz_active, cpu) = true;
-       }
+void timers_update_nohz(void)
+{
+       schedule_work(&timer_update_work);
 }
 
 int timer_migration_handler(struct ctl_table *table, int write,
                            void __user *buffer, size_t *lenp,
                            loff_t *ppos)
 {
-       static DEFINE_MUTEX(mutex);
        int ret;
 
-       mutex_lock(&mutex);
+       mutex_lock(&timer_keys_mutex);
        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
        if (!ret && write)
-               timers_update_migration(false);
-       mutex_unlock(&mutex);
+               timers_update_migration();
+       mutex_unlock(&timer_keys_mutex);
        return ret;
 }
-#endif
+
+static inline bool is_timers_nohz_active(void)
+{
+       return static_branch_unlikely(&timers_nohz_active);
+}
+#else
+static inline bool is_timers_nohz_active(void) { return false; }
+#endif /* NO_HZ_COMMON */
 
 static unsigned long round_jiffies_common(unsigned long j, int cpu,
                bool force_up)
@@ -534,7 +551,7 @@ __internal_add_timer(struct timer_base *base, struct timer_list *timer)
 static void
 trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)
 {
-       if (!IS_ENABLED(CONFIG_NO_HZ_COMMON) || !base->nohz_active)
+       if (!is_timers_nohz_active())
                return;
 
        /*
@@ -849,21 +866,20 @@ static inline struct timer_base *get_timer_base(u32 tflags)
        return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK);
 }
 
-#ifdef CONFIG_NO_HZ_COMMON
 static inline struct timer_base *
 get_target_base(struct timer_base *base, unsigned tflags)
 {
-#ifdef CONFIG_SMP
-       if ((tflags & TIMER_PINNED) || !base->migration_enabled)
-               return get_timer_this_cpu_base(tflags);
-       return get_timer_cpu_base(tflags, get_nohz_timer_target());
-#else
-       return get_timer_this_cpu_base(tflags);
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+       if (static_branch_likely(&timers_migration_enabled) &&
+           !(tflags & TIMER_PINNED))
+               return get_timer_cpu_base(tflags, get_nohz_timer_target());
 #endif
+       return get_timer_this_cpu_base(tflags);
 }
 
 static inline void forward_timer_base(struct timer_base *base)
 {
+#ifdef CONFIG_NO_HZ_COMMON
        unsigned long jnow;
 
        /*
@@ -887,16 +903,8 @@ static inline void forward_timer_base(struct timer_base *base)
                base->clk = jnow;
        else
                base->clk = base->next_expiry;
-}
-#else
-static inline struct timer_base *
-get_target_base(struct timer_base *base, unsigned tflags)
-{
-       return get_timer_this_cpu_base(tflags);
-}
-
-static inline void forward_timer_base(struct timer_base *base) { }
 #endif
+}
 
 
 /*
index 69db45bc0197e398ce298ed2c7d9f71b0539f9fb..8fb9a54fe8ba879857ce61ec25d5d2408c9a21ad 100644 (file)
@@ -375,17 +375,9 @@ struct dummy_hrtimer_pcm {
        ktime_t period_time;
        atomic_t running;
        struct hrtimer timer;
-       struct tasklet_struct tasklet;
        struct snd_pcm_substream *substream;
 };
 
-static void dummy_hrtimer_pcm_elapsed(unsigned long priv)
-{
-       struct dummy_hrtimer_pcm *dpcm = (struct dummy_hrtimer_pcm *)priv;
-       if (atomic_read(&dpcm->running))
-               snd_pcm_period_elapsed(dpcm->substream);
-}
-
 static enum hrtimer_restart dummy_hrtimer_callback(struct hrtimer *timer)
 {
        struct dummy_hrtimer_pcm *dpcm;
@@ -393,7 +385,14 @@ static enum hrtimer_restart dummy_hrtimer_callback(struct hrtimer *timer)
        dpcm = container_of(timer, struct dummy_hrtimer_pcm, timer);
        if (!atomic_read(&dpcm->running))
                return HRTIMER_NORESTART;
-       tasklet_schedule(&dpcm->tasklet);
+       /*
+        * In cases of XRUN and draining, this calls .trigger to stop PCM
+        * substream.
+        */
+       snd_pcm_period_elapsed(dpcm->substream);
+       if (!atomic_read(&dpcm->running))
+               return HRTIMER_NORESTART;
+
        hrtimer_forward_now(timer, dpcm->period_time);
        return HRTIMER_RESTART;
 }
@@ -403,7 +402,7 @@ static int dummy_hrtimer_start(struct snd_pcm_substream *substream)
        struct dummy_hrtimer_pcm *dpcm = substream->runtime->private_data;
 
        dpcm->base_time = hrtimer_cb_get_time(&dpcm->timer);
-       hrtimer_start(&dpcm->timer, dpcm->period_time, HRTIMER_MODE_REL);
+       hrtimer_start(&dpcm->timer, dpcm->period_time, HRTIMER_MODE_REL_SOFT);
        atomic_set(&dpcm->running, 1);
        return 0;
 }
@@ -413,14 +412,14 @@ static int dummy_hrtimer_stop(struct snd_pcm_substream *substream)
        struct dummy_hrtimer_pcm *dpcm = substream->runtime->private_data;
 
        atomic_set(&dpcm->running, 0);
-       hrtimer_cancel(&dpcm->timer);
+       if (!hrtimer_callback_running(&dpcm->timer))
+               hrtimer_cancel(&dpcm->timer);
        return 0;
 }
 
 static inline void dummy_hrtimer_sync(struct dummy_hrtimer_pcm *dpcm)
 {
        hrtimer_cancel(&dpcm->timer);
-       tasklet_kill(&dpcm->tasklet);
 }
 
 static snd_pcm_uframes_t
@@ -465,12 +464,10 @@ static int dummy_hrtimer_create(struct snd_pcm_substream *substream)
        if (!dpcm)
                return -ENOMEM;
        substream->runtime->private_data = dpcm;
-       hrtimer_init(&dpcm->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+       hrtimer_init(&dpcm->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
        dpcm->timer.function = dummy_hrtimer_callback;
        dpcm->substream = substream;
        atomic_set(&dpcm->running, 0);
-       tasklet_init(&dpcm->tasklet, dummy_hrtimer_pcm_elapsed,
-                    (unsigned long)dpcm);
        return 0;
 }
 
index 5d2eae16f7ee5ef9ab2d9ecfc15deed3a3da97dd..a5d8f0ab0da000211fd750b10fef2386ae5cfa4b 100644 (file)
@@ -60,9 +60,7 @@ static int clock_adjtime(clockid_t id, struct timex *tx)
 static clockid_t get_clockid(int fd)
 {
 #define CLOCKFD 3
-#define FD_TO_CLOCKID(fd)      ((~(clockid_t) (fd) << 3) | CLOCKFD)
-
-       return FD_TO_CLOCKID(fd);
+       return (((unsigned int) ~fd) << 3) | CLOCKFD;
 }
 
 static void handle_alarm(int s)