Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git...

[sfrench/cifs-2.6.git] / kernel / rcu / tree.h

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index d1df192272fb9b8ffee48ca7cbde5c883d9241f6..c612f306fe89032a5561a0ccf7028d873f8d5c73 100644 (file)
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -194,26 +194,36 @@ struct rcu_data {
 
        /* 5) Callback offloading. */
 #ifdef CONFIG_RCU_NOCB_CPU
-       struct rcu_head *nocb_cb_head;  /* CBs ready to invoke. */
-       struct rcu_head **nocb_cb_tail;
        struct swait_queue_head nocb_cb_wq; /* For nocb kthreads to sleep on. */
        struct task_struct *nocb_gp_kthread;
        raw_spinlock_t nocb_lock;       /* Guard following pair of fields. */
+       atomic_t nocb_lock_contended;   /* Contention experienced. */
        int nocb_defer_wakeup;          /* Defer wakeup of nocb_kthread. */
        struct timer_list nocb_timer;   /* Enforce finite deferral. */
+       unsigned long nocb_gp_adv_time; /* Last call_rcu() CB adv (jiffies). */
+
+       /* The following fields are used by call_rcu, hence own cacheline. */
+       raw_spinlock_t nocb_bypass_lock ____cacheline_internodealigned_in_smp;
+       struct rcu_cblist nocb_bypass;  /* Lock-contention-bypass CB list. */
+       unsigned long nocb_bypass_first; /* Time (jiffies) of first enqueue. */
+       unsigned long nocb_nobypass_last; /* Last ->cblist enqueue (jiffies). */
+       int nocb_nobypass_count;        /* # ->cblist enqueues at ^^^ time. */
 
        /* The following fields are used by GP kthread, hence own cacheline. */
-       struct rcu_head *nocb_gp_head ____cacheline_internodealigned_in_smp;
-                                       /* CBs waiting for GP. */
-       struct rcu_head **nocb_gp_tail;
-       bool nocb_gp_sleep;             /* Is the nocb GP thread asleep? */
+       raw_spinlock_t nocb_gp_lock ____cacheline_internodealigned_in_smp;
+       struct timer_list nocb_bypass_timer; /* Force nocb_bypass flush. */
+       u8 nocb_gp_sleep;               /* Is the nocb GP thread asleep? */
+       u8 nocb_gp_bypass;              /* Found a bypass on last scan? */
+       u8 nocb_gp_gp;                  /* GP to wait for on last scan? */
+       unsigned long nocb_gp_seq;      /*  If so, ->gp_seq to wait for. */
+       unsigned long nocb_gp_loops;    /* # passes through wait code. */
        struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */
        bool nocb_cb_sleep;             /* Is the nocb CB thread asleep? */
        struct task_struct *nocb_cb_kthread;
        struct rcu_data *nocb_next_cb_rdp;
                                        /* Next rcu_data in wakeup chain. */
 
-       /* The following fields are used by CB kthread, hence new cachline. */
+       /* The following fields are used by CB kthread, hence new cacheline. */
        struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp;
                                        /* GP rdp takes GP-end wakeups. */
 #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
@@ -421,6 +431,10 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp);
 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);
 static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);
 static void rcu_init_one_nocb(struct rcu_node *rnp);
+static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
+                                 unsigned long j);
+static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
+                               bool *was_alldone, unsigned long flags);
 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,
                                 unsigned long flags);
 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp);
@@ -428,13 +442,25 @@ static void do_nocb_deferred_wakeup(struct rcu_data *rdp);
 static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
 static void rcu_spawn_cpu_nocb_kthread(int cpu);
 static void __init rcu_spawn_nocb_kthreads(void);
+static void show_rcu_nocb_state(struct rcu_data *rdp);
 static void rcu_nocb_lock(struct rcu_data *rdp);
 static void rcu_nocb_unlock(struct rcu_data *rdp);
 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
                                       unsigned long flags);
+static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp);
 #ifdef CONFIG_RCU_NOCB_CPU
 static void __init rcu_organize_nocb_kthreads(void);
-#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
+#define rcu_nocb_lock_irqsave(rdp, flags)                              \
+do {                                                                   \
+       if (!rcu_segcblist_is_offloaded(&(rdp)->cblist))                \
+               local_irq_save(flags);                                  \
+       else                                                            \
+               raw_spin_lock_irqsave(&(rdp)->nocb_lock, (flags));      \
+} while (0)
+#else /* #ifdef CONFIG_RCU_NOCB_CPU */
+#define rcu_nocb_lock_irqsave(rdp, flags) local_irq_save(flags)
+#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
+
 static void rcu_bind_gp_kthread(void);
 static bool rcu_nohz_full_cpu(void);
 static void rcu_dynticks_task_enter(void);