From 4addd0b6c1bffd24f9fe6c3a8661050d770bcb78 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 11 Jan 2018 08:16:34 +0000 Subject: [PATCH] softirq: Prefer low-latency over ksoftirqd for HI_SOFTIRQ References: 4cd13c21b207 ("softirq: Let ksoftirqd do its job") Signed-off-by: Chris Wilson --- arch/x86/include/asm/hardirq.h | 7 +++++++ include/asm-generic/hardirq.h | 1 + include/linux/interrupt.h | 2 ++ include/linux/irq_cpustat.h | 3 +++ kernel/softirq.c | 39 +++++++++++++++++++++------------------ 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h index 5ea2afd..76192e2c 100644 --- a/arch/x86/include/asm/hardirq.h +++ b/arch/x86/include/asm/hardirq.h @@ -7,6 +7,7 @@ typedef struct { unsigned int __softirq_pending; + unsigned int __ksoftirqd_overflow; unsigned int __nmi_count; /* arch dependent */ #ifdef CONFIG_X86_LOCAL_APIC unsigned int apic_timer_irqs; /* arch dependent */ @@ -51,6 +52,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); #define inc_irq_stat(member) this_cpu_inc(irq_stat.member) #define local_softirq_pending() this_cpu_read(irq_stat.__softirq_pending) +#define local_ksoftirqd_overflow() this_cpu_read(irq_stat.__ksoftirqd_overflow) #define __ARCH_SET_SOFTIRQ_PENDING @@ -58,6 +60,11 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); this_cpu_write(irq_stat.__softirq_pending, (x)) #define or_softirq_pending(x) this_cpu_or(irq_stat.__softirq_pending, (x)) +#define set_ksoftirqd_overflow(x) \ + this_cpu_write(irq_stat.__ksoftirqd_overflow, (x)) +#define or_ksoftirqd_overflow(x) \ + this_cpu_or(irq_stat.__ksoftirqd_overflow, (x)) + extern void ack_bad_irq(unsigned int irq); extern u64 arch_irq_stat_cpu(unsigned int cpu); diff --git a/include/asm-generic/hardirq.h b/include/asm-generic/hardirq.h index d14214d..e506fb8 100644 --- a/include/asm-generic/hardirq.h +++ b/include/asm-generic/hardirq.h @@ -7,6 +7,7 @@ typedef struct { unsigned int __softirq_pending; + unsigned int __ksoftirqd_overflow; } ____cacheline_aligned irq_cpustat_t; #include /* Standard mappings for irq_cpustat_t above */ diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 5426627..1db7b7b 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -435,6 +435,8 @@ extern bool force_irqthreads; #ifndef __ARCH_SET_SOFTIRQ_PENDING #define set_softirq_pending(x) (local_softirq_pending() = (x)) #define or_softirq_pending(x) (local_softirq_pending() |= (x)) +#define set_ksoftirqd_overflow(x) (local_ksoftirqd_overflow() = (x)) +#define or_ksoftirqd_overflow(x) (local_ksoftirqd_overflow() |= (x)) #endif /* Some architectures might implement lazy enabling/disabling of diff --git a/include/linux/irq_cpustat.h b/include/linux/irq_cpustat.h index 4954948..ed76635 100644 --- a/include/linux/irq_cpustat.h +++ b/include/linux/irq_cpustat.h @@ -26,6 +26,9 @@ extern irq_cpustat_t irq_stat[]; /* defined in asm/hardirq.h */ #define local_softirq_pending() \ __IRQ_STAT(smp_processor_id(), __softirq_pending) +#define local_ksoftirqd_overflow() \ + __IRQ_STAT(smp_processor_id(), __ksoftirqd_overflow) + /* arch dependent irq_stat fields */ #define nmi_count(cpu) __IRQ_STAT((cpu), __nmi_count) /* i386 */ diff --git a/kernel/softirq.c b/kernel/softirq.c index 177de36..9d053c9 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -68,12 +68,16 @@ const char * const softirq_to_name[NR_SOFTIRQS] = { * to the pending events, so lets the scheduler to balance * the softirq load for us. */ -static void wakeup_softirqd(void) +static void wakeup_softirqd(unsigned long pending) { /* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk = __this_cpu_read(ksoftirqd); - if (tsk && tsk->state != TASK_RUNNING) + if (!tsk || local_ksoftirqd_overflow() & pending) + return; + + or_ksoftirqd_overflow(pending); + if (tsk->state != TASK_RUNNING) wake_up_process(tsk); } @@ -81,11 +85,10 @@ static void wakeup_softirqd(void) * If ksoftirqd is scheduled, we do not want to process pending softirqs * right now. Let ksoftirqd handle this at its own rate, to get fairness. */ -static bool ksoftirqd_running(void) +static bool defer_to_ksoftirqd(__u32 pending) { - struct task_struct *tsk = __this_cpu_read(ksoftirqd); - - return tsk && (tsk->state == TASK_RUNNING); + pending &= ~HI_SOFTIRQ; + return local_ksoftirqd_overflow() & pending; } /* @@ -303,7 +306,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void) --max_restart) goto restart; - wakeup_softirqd(); + wakeup_softirqd(pending); } lockdep_softirq_end(in_hardirq); @@ -324,8 +327,7 @@ asmlinkage __visible void do_softirq(void) local_irq_save(flags); pending = local_softirq_pending(); - - if (pending && !ksoftirqd_running()) + if (pending && !defer_to_ksoftirqd(pending)) do_softirq_own_stack(); local_irq_restore(flags); @@ -350,12 +352,12 @@ void irq_enter(void) __irq_enter(); } -static inline void invoke_softirq(void) +static inline void invoke_softirq(__u32 pending) { - if (ksoftirqd_running()) + if (!pending) return; - if (!force_irqthreads) { + if (!force_irqthreads && !defer_to_ksoftirqd(pending)) { #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK /* * We can safely execute softirq on the current stack if @@ -372,7 +374,7 @@ static inline void invoke_softirq(void) do_softirq_own_stack(); #endif } else { - wakeup_softirqd(); + wakeup_softirqd(pending); } } @@ -401,8 +403,8 @@ void irq_exit(void) #endif account_irq_exit_time(current); preempt_count_sub(HARDIRQ_OFFSET); - if (!in_interrupt() && local_softirq_pending()) - invoke_softirq(); + if (!in_interrupt()) + invoke_softirq(local_softirq_pending()); tick_irq_exit(); rcu_irq_exit(); @@ -426,7 +428,7 @@ inline void raise_softirq_irqoff(unsigned int nr) * schedule the softirq soon. */ if (!in_interrupt()) - wakeup_softirqd(); + wakeup_softirqd(local_softirq_pending()); } void raise_softirq(unsigned int nr) @@ -632,13 +634,14 @@ void __init softirq_init(void) static int ksoftirqd_should_run(unsigned int cpu) { - return local_softirq_pending(); + return local_ksoftirqd_overflow(); } static void run_ksoftirqd(unsigned int cpu) { local_irq_disable(); - if (local_softirq_pending()) { + if (local_ksoftirqd_overflow()) { + set_ksoftirqd_overflow(0); /* * We can safely run softirq on inline stack, as we are not deep * in the task stack here. -- 2.7.4