smp: Add source and destination CPUs to __call_single_data

This commit adds a destination CPU to __call_single_data, and is inspired
by an earlier commit by Peter Zijlstra.  This version adds #ifdef to
permit use by 32-bit systems and supplying the destination CPU for all
smp_call_function*() requests, not just smp_call_function_single().

If need be, 32-bit systems could be accommodated by shrinking the flags
field to 16 bits (the atomic_t variant is currently unused) and by
providing only eight bits for CPU on such systems.

It is not clear that the addition of the fields to __call_single_node
are really needed.

[ paulmck: Apply Boqun Feng feedback on 32-bit builds. ]
Link: https://lore.kernel.org/lkml/20200615164048.GC2531@hirez.programming.kicks-ass.net/
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/smp.h b/include/linux/smp.h
index 80d557ef8a11c34fd99dd3233c0b9425df150055..9f13966d3d92d36a57faf3926a71bfa923a78a8d 100644 (file)
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -26,6 +26,9 @@ struct __call_single_data {
                struct {
                        struct llist_node llist;
                        unsigned int flags;
+#ifdef CONFIG_64BIT
+                       u16 src, dst;
+#endif
                };
        };
        smp_call_func_t func;

diff --git a/include/linux/smp_types.h b/include/linux/smp_types.h
index 364b3ae3e41d4d51fd277196b9fdb6b3f2da2b39..2e8461af8df6eecb5996b57346f2fc2c7420860d 100644 (file)
--- a/include/linux/smp_types.h
+++ b/include/linux/smp_types.h
@@ -61,6 +61,9 @@ struct __call_single_node {
                unsigned int    u_flags;
                atomic_t        a_flags;
        };
+#ifdef CONFIG_64BIT
+       u16 src, dst;
+#endif
 };
 
 #endif /* __LINUX_SMP_TYPES_H */

diff --git a/kernel/smp.c b/kernel/smp.c
index d0ae8eb6bf8b41df14e24187bf42d2d97a078edc..865a876f83ce14bbfd5d791fecf2693da6c2a197 100644 (file)
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -375,6 +375,9 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
 
        csd->func = func;
        csd->info = info;
+#ifdef CONFIG_64BIT
+       csd->dst = cpu;
+#endif
 
        err = generic_exec_single(cpu, csd);
 
@@ -540,6 +543,9 @@ static void smp_call_function_many_cond(const struct cpumask *mask,
                        csd->flags |= CSD_TYPE_SYNC;
                csd->func = func;
                csd->info = info;
+#ifdef CONFIG_64BIT
+               csd->dst = cpu;
+#endif
                if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
                        __cpumask_set_cpu(cpu, cfd->cpumask_ipi);
        }