Attachment 139306 Details for Bug 106373 – Kick softirqs harder

[patch] Kick softirqs harder

0001-softirq-Prefer-low-latency-over-ksoftirqd-for-HI_SOF.patch (text/plain), 6.52 KB, created by Chris Wilson on 2018-05-03 12:55:34 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Chris Wilson

Created: 2018-05-03 12:55:34 UTC

Size: 6.52 KB

patch

obsolete

>From 4addd0b6c1bffd24f9fe6c3a8661050d770bcb78 Mon Sep 17 00:00:00 2001
>From: Chris Wilson <chris@chris-wilson.co.uk>
>Date: Thu, 11 Jan 2018 08:16:34 +0000
>Subject: [PATCH] softirq: Prefer low-latency over ksoftirqd for HI_SOFTIRQ
>
>References: 4cd13c21b207 ("softirq: Let ksoftirqd do its job")
>Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
>---
> arch/x86/include/asm/hardirq.h |  7 +++++++
> include/asm-generic/hardirq.h  |  1 +
> include/linux/interrupt.h      |  2 ++
> include/linux/irq_cpustat.h    |  3 +++
> kernel/softirq.c               | 39 +++++++++++++++++++++------------------
> 5 files changed, 34 insertions(+), 18 deletions(-)
>
>diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
>index 5ea2afd..76192e2c 100644
>--- a/arch/x86/include/asm/hardirq.h
>+++ b/arch/x86/include/asm/hardirq.h
>@@ -7,6 +7,7 @@
> 
> typedef struct {
> 	unsigned int __softirq_pending;
>+	unsigned int __ksoftirqd_overflow;
> 	unsigned int __nmi_count;	/* arch dependent */
> #ifdef CONFIG_X86_LOCAL_APIC
> 	unsigned int apic_timer_irqs;	/* arch dependent */
>@@ -51,6 +52,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
> #define inc_irq_stat(member)	this_cpu_inc(irq_stat.member)
> 
> #define local_softirq_pending()	this_cpu_read(irq_stat.__softirq_pending)
>+#define local_ksoftirqd_overflow()	this_cpu_read(irq_stat.__ksoftirqd_overflow)
> 
> #define __ARCH_SET_SOFTIRQ_PENDING
> 
>@@ -58,6 +60,11 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
> 		this_cpu_write(irq_stat.__softirq_pending, (x))
> #define or_softirq_pending(x)	this_cpu_or(irq_stat.__softirq_pending, (x))
> 
>+#define set_ksoftirqd_overflow(x)	\
>+		this_cpu_write(irq_stat.__ksoftirqd_overflow, (x))
>+#define or_ksoftirqd_overflow(x)	\
>+		this_cpu_or(irq_stat.__ksoftirqd_overflow, (x))
>+
> extern void ack_bad_irq(unsigned int irq);
> 
> extern u64 arch_irq_stat_cpu(unsigned int cpu);
>diff --git a/include/asm-generic/hardirq.h b/include/asm-generic/hardirq.h
>index d14214d..e506fb8 100644
>--- a/include/asm-generic/hardirq.h
>+++ b/include/asm-generic/hardirq.h
>@@ -7,6 +7,7 @@
> 
> typedef struct {
> 	unsigned int __softirq_pending;
>+	unsigned int __ksoftirqd_overflow;
> } ____cacheline_aligned irq_cpustat_t;
> 
> #include <linux/irq_cpustat.h>	/* Standard mappings for irq_cpustat_t above */
>diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
>index 5426627..1db7b7b 100644
>--- a/include/linux/interrupt.h
>+++ b/include/linux/interrupt.h
>@@ -435,6 +435,8 @@ extern bool force_irqthreads;
> #ifndef __ARCH_SET_SOFTIRQ_PENDING
> #define set_softirq_pending(x) (local_softirq_pending() = (x))
> #define or_softirq_pending(x)  (local_softirq_pending() |= (x))
>+#define set_ksoftirqd_overflow(x) (local_ksoftirqd_overflow() = (x))
>+#define or_ksoftirqd_overflow(x)  (local_ksoftirqd_overflow() |= (x))
> #endif
> 
> /* Some architectures might implement lazy enabling/disabling of
>diff --git a/include/linux/irq_cpustat.h b/include/linux/irq_cpustat.h
>index 4954948..ed76635 100644
>--- a/include/linux/irq_cpustat.h
>+++ b/include/linux/irq_cpustat.h
>@@ -26,6 +26,9 @@ extern irq_cpustat_t irq_stat[];		/* defined in asm/hardirq.h */
> #define local_softirq_pending() \
> 	__IRQ_STAT(smp_processor_id(), __softirq_pending)
> 
>+#define local_ksoftirqd_overflow() \
>+	__IRQ_STAT(smp_processor_id(), __ksoftirqd_overflow)
>+
>   /* arch dependent irq_stat fields */
> #define nmi_count(cpu)		__IRQ_STAT((cpu), __nmi_count)	/* i386 */
> 
>diff --git a/kernel/softirq.c b/kernel/softirq.c
>index 177de36..9d053c9 100644
>--- a/kernel/softirq.c
>+++ b/kernel/softirq.c
>@@ -68,12 +68,16 @@ const char * const softirq_to_name[NR_SOFTIRQS] = {
>  * to the pending events, so lets the scheduler to balance
>  * the softirq load for us.
>  */
>-static void wakeup_softirqd(void)
>+static void wakeup_softirqd(unsigned long pending)
> {
> 	/* Interrupts are disabled: no need to stop preemption */
> 	struct task_struct *tsk = __this_cpu_read(ksoftirqd);
> 
>-	if (tsk && tsk->state != TASK_RUNNING)
>+	if (!tsk || local_ksoftirqd_overflow() & pending)
>+		return;
>+
>+	or_ksoftirqd_overflow(pending);
>+	if (tsk->state != TASK_RUNNING)
> 		wake_up_process(tsk);
> }
> 
>@@ -81,11 +85,10 @@ static void wakeup_softirqd(void)
>  * If ksoftirqd is scheduled, we do not want to process pending softirqs
>  * right now. Let ksoftirqd handle this at its own rate, to get fairness.
>  */
>-static bool ksoftirqd_running(void)
>+static bool defer_to_ksoftirqd(__u32 pending)
> {
>-	struct task_struct *tsk = __this_cpu_read(ksoftirqd);
>-
>-	return tsk && (tsk->state == TASK_RUNNING);
>+	pending &= ~HI_SOFTIRQ;
>+	return local_ksoftirqd_overflow() & pending;
> }
> 
> /*
>@@ -303,7 +306,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void)
> 		    --max_restart)
> 			goto restart;
> 
>-		wakeup_softirqd();
>+		wakeup_softirqd(pending);
> 	}
> 
> 	lockdep_softirq_end(in_hardirq);
>@@ -324,8 +327,7 @@ asmlinkage __visible void do_softirq(void)
> 	local_irq_save(flags);
> 
> 	pending = local_softirq_pending();
>-
>-	if (pending && !ksoftirqd_running())
>+	if (pending && !defer_to_ksoftirqd(pending))
> 		do_softirq_own_stack();
> 
> 	local_irq_restore(flags);
>@@ -350,12 +352,12 @@ void irq_enter(void)
> 	__irq_enter();
> }
> 
>-static inline void invoke_softirq(void)
>+static inline void invoke_softirq(__u32 pending)
> {
>-	if (ksoftirqd_running())
>+	if (!pending)
> 		return;
> 
>-	if (!force_irqthreads) {
>+	if (!force_irqthreads && !defer_to_ksoftirqd(pending)) {
> #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
> 		/*
> 		 * We can safely execute softirq on the current stack if
>@@ -372,7 +374,7 @@ static inline void invoke_softirq(void)
> 		do_softirq_own_stack();
> #endif
> 	} else {
>-		wakeup_softirqd();
>+		wakeup_softirqd(pending);
> 	}
> }
> 
>@@ -401,8 +403,8 @@ void irq_exit(void)
> #endif
> 	account_irq_exit_time(current);
> 	preempt_count_sub(HARDIRQ_OFFSET);
>-	if (!in_interrupt() && local_softirq_pending())
>-		invoke_softirq();
>+	if (!in_interrupt())
>+		invoke_softirq(local_softirq_pending());
> 
> 	tick_irq_exit();
> 	rcu_irq_exit();
>@@ -426,7 +428,7 @@ inline void raise_softirq_irqoff(unsigned int nr)
> 	 * schedule the softirq soon.
> 	 */
> 	if (!in_interrupt())
>-		wakeup_softirqd();
>+		wakeup_softirqd(local_softirq_pending());
> }
> 
> void raise_softirq(unsigned int nr)
>@@ -632,13 +634,14 @@ void __init softirq_init(void)
> 
> static int ksoftirqd_should_run(unsigned int cpu)
> {
>-	return local_softirq_pending();
>+	return local_ksoftirqd_overflow();
> }
> 
> static void run_ksoftirqd(unsigned int cpu)
> {
> 	local_irq_disable();
>-	if (local_softirq_pending()) {
>+	if (local_ksoftirqd_overflow()) {
>+		set_ksoftirqd_overflow(0);
> 		/*
> 		 * We can safely run softirq on inline stack, as we are not deep
> 		 * in the task stack here.
>-- 
>2.7.4
>

Actions: View | Splinter Review

Attachments on bug 106373: 139306

Use of freedesktop.org services, including Bugzilla, is subject to our Code of Conduct. How we collect and use information is described in our Privacy Policy.