From: Nicholas Piggin <npiggin@gmail.com> Date: Thu, 17 Dec 2020 13:47:29 +0000 (+1000) Subject: powerpc/64s/radix: occasionally attempt to trim mm_cpumask X-Git-Url: https://git.baikalelectronics.ru/?a=commitdiff_plain;h=9393544842d6c85ebfc387c43a5059f8171d598f;p=kernel.git powerpc/64s/radix: occasionally attempt to trim mm_cpumask A single-threaded process that is flushing its own address space is so far the only case where the mm_cpumask is attempted to be trimmed. This patch expands that to flush in other situations, multi-threaded processes and external sources. For now it's a relatively simple occasional trim attempt. The main aim is to add the mechanism, tweaking and tuning can come with more data. Signed-off-by: Nicholas Piggin <npiggin@gmail.com> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://lore.kernel.org/r/20201217134731.488135-6-npiggin@gmail.com --- diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index c27ef2ff89504..1ef5d4e1a50a7 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -639,10 +639,8 @@ static bool mm_needs_flush_escalation(struct mm_struct *mm) return false; } -#ifdef CONFIG_SMP -static void do_exit_flush_lazy_tlb(void *arg) +static void exit_lazy_flush_tlb(struct mm_struct *mm) { - struct mm_struct *mm = arg; unsigned long pid = mm->context.id; int cpu = smp_processor_id(); @@ -682,6 +680,13 @@ out_flush: _tlbiel_pid(pid, RIC_FLUSH_ALL); } +#ifdef CONFIG_SMP +static void do_exit_flush_lazy_tlb(void *arg) +{ + struct mm_struct *mm = arg; + exit_lazy_flush_tlb(mm); +} + static void exit_flush_lazy_tlbs(struct mm_struct *mm) { /* @@ -694,10 +699,32 @@ static void exit_flush_lazy_tlbs(struct mm_struct *mm) smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb, (void *)mm, 1); } + #else /* CONFIG_SMP */ static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { } #endif /* CONFIG_SMP */ +static DEFINE_PER_CPU(unsigned int, mm_cpumask_trim_clock); + +/* + * Interval between flushes at which we send out IPIs to check whether the + * mm_cpumask can be trimmed for the case where it's not a single-threaded + * process flushing its own mm. The intent is to reduce the cost of later + * flushes. Don't want this to be so low that it adds noticable cost to TLB + * flushing, or so high that it doesn't help reduce global TLBIEs. + */ +static unsigned long tlb_mm_cpumask_trim_timer = 1073; + +static bool tick_and_test_trim_clock(void) +{ + if (__this_cpu_inc_return(mm_cpumask_trim_clock) == + tlb_mm_cpumask_trim_timer) { + __this_cpu_write(mm_cpumask_trim_clock, 0); + return true; + } + return false; +} + enum tlb_flush_type { FLUSH_TYPE_NONE, FLUSH_TYPE_LOCAL, @@ -711,8 +738,20 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm) if (active_cpus == 0) return FLUSH_TYPE_NONE; - if (active_cpus == 1 && cpumask_test_cpu(cpu, mm_cpumask(mm))) + if (active_cpus == 1 && cpumask_test_cpu(cpu, mm_cpumask(mm))) { + if (current->mm != mm) { + /* + * Asynchronous flush sources may trim down to nothing + * if the process is not running, so occasionally try + * to trim. + */ + if (tick_and_test_trim_clock()) { + exit_lazy_flush_tlb(mm); + return FLUSH_TYPE_NONE; + } + } return FLUSH_TYPE_LOCAL; + } /* Coprocessors require TLBIE to invalidate nMMU. */ if (atomic_read(&mm->context.copros) > 0) @@ -744,6 +783,19 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm) return FLUSH_TYPE_LOCAL; } + /* + * Occasionally try to trim down the cpumask. It's possible this can + * bring the mask to zero, which results in no flush. + */ + if (tick_and_test_trim_clock()) { + exit_flush_lazy_tlbs(mm); + if (current->mm == mm) + return FLUSH_TYPE_LOCAL; + if (cpumask_test_cpu(cpu, mm_cpumask(mm))) + exit_lazy_flush_tlb(mm); + return FLUSH_TYPE_NONE; + } + return FLUSH_TYPE_GLOBAL; }