printk: Wait for the global console lock when the system is going down

There are reports that the console kthreads block the global console
lock when the system is going down, for example, reboot, panic.

First part of the solution was to block kthreads in these problematic
system states so they stopped handling newly added messages.

Second part of the solution is to wait when for the kthreads when
they are actively printing. It solves the problem when a message
was printed before the system entered the problematic state and
the kthreads managed to step in.

A busy waiting has to be used because panic() can be called in any
context and in an unknown state of the scheduler.

There must be a timeout because the kthread might get stuck or sleeping
and never release the lock. The timeout 10s is an arbitrary value
inspired by the softlockup timeout.

Link: https://lore.kernel.org/r/20220610205038.GA3050413@paulmck-ThinkPad-P17-Gen-1
Link: https://lore.kernel.org/r/CAMdYzYpF4FNTBPZsEFeWRuEwSies36QM_As8osPWZSr2q-viEA@mail.gmail.com
Signed-off-by: Petr Mladek <pmladek@suse.com>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://lore.kernel.org/r/20220615162805.27962-3-pmladek@suse.com

diff --git a/include/linux/printk.h b/include/linux/printk.h
index cd26aab0ab2a094d7fbb276cfa95f8486e9363b5..c1e07c0652c7c227f896a59dcf9dace05e8300e7 100644 (file)
--- a/include/linux/printk.h
+++ b/include/linux/printk.h
@@ -174,6 +174,7 @@ extern void printk_prefer_direct_enter(void);
 extern void printk_prefer_direct_exit(void);
 
 extern bool pr_flush(int timeout_ms, bool reset_on_progress);
+extern void try_block_console_kthreads(int timeout_ms);
 
 /*
  * Please don't use printk_ratelimit(), because it shares ratelimiting state
@@ -238,6 +239,10 @@ static inline bool pr_flush(int timeout_ms, bool reset_on_progress)
        return true;
 }
 
+static inline void try_block_console_kthreads(int timeout_ms)
+{
+}
+
 static inline int printk_ratelimit(void)
 {
        return 0;

diff --git a/kernel/panic.c b/kernel/panic.c
index 6737b23322753a2c89954a98ef4e6038fc09b3a1..fe73d18ecdf02b2ef4640e644cad8d0670df8ab6 100644 (file)
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -273,6 +273,7 @@ void panic(const char *fmt, ...)
                 * unfortunately means it may not be hardened to work in a
                 * panic situation.
                 */
+               try_block_console_kthreads(10000);
                smp_send_stop();
        } else {
                /*
@@ -280,6 +281,7 @@ void panic(const char *fmt, ...)
                 * kmsg_dump, we will need architecture dependent extra
                 * works in addition to stopping other CPUs.
                 */
+               try_block_console_kthreads(10000);
                crash_smp_send_stop();
        }
 

diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h
index d947ca6c84f997d3e4e657bc2010f0e22367efe2..e7d8578860adfcf75b63668394584802f7f24f4f 100644 (file)
--- a/kernel/printk/internal.h
+++ b/kernel/printk/internal.h
@@ -20,6 +20,8 @@ enum printk_info_flags {
        LOG_CONT        = 8,    /* text is a fragment of a continuation line */
 };
 
+extern bool block_console_kthreads;
+
 __printf(4, 0)
 int vprintk_store(int facility, int level,
                  const struct dev_printk_info *dev_info,

diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 45c6c2b0b1044cc54888a4e5b647775edd282a22..b095fb5f5f61b7b2f7882986009fe890e48d49d8 100644 (file)
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -250,6 +250,9 @@ static atomic_t console_kthreads_active = ATOMIC_INIT(0);
 #define console_kthread_printing_exit() \
        atomic_dec(&console_kthreads_active)
 
+/* Block console kthreads to avoid processing new messages. */
+bool block_console_kthreads;
+
 /*
  * Helper macros to handle lockdep when locking/unlocking console_sem. We use
  * macros instead of functions so that _RET_IP_ contains useful information.
@@ -3730,6 +3733,7 @@ static bool printer_should_wake(struct console *con, u64 seq)
 
        if (con->blocked ||
            console_kthreads_atomically_blocked() ||
+           block_console_kthreads ||
            system_state > SYSTEM_RUNNING ||
            oops_in_progress) {
                return false;

diff --git a/kernel/printk/printk_safe.c b/kernel/printk/printk_safe.c
index ef0f9a2044da10739e4ec865ab35e97db9c02d5f..caac4de1ea59a00544c19f5b2f9434972eec9f86 100644 (file)
--- a/kernel/printk/printk_safe.c
+++ b/kernel/printk/printk_safe.c
@@ -8,7 +8,9 @@
 #include <linux/smp.h>
 #include <linux/cpumask.h>
 #include <linux/printk.h>
+#include <linux/console.h>
 #include <linux/kprobes.h>
+#include <linux/delay.h>
 
 #include "internal.h"
 
@@ -50,3 +52,33 @@ asmlinkage int vprintk(const char *fmt, va_list args)
        return vprintk_default(fmt, args);
 }
 EXPORT_SYMBOL(vprintk);
+
+/**
+ * try_block_console_kthreads() - Try to block console kthreads and
+ *     make the global console_lock() avaialble
+ *
+ * @timeout_ms:        The maximum time (in ms) to wait.
+ *
+ * Prevent console kthreads from starting processing new messages. Wait
+ * until the global console_lock() become available.
+ *
+ * Context: Can be called in any context.
+ */
+void try_block_console_kthreads(int timeout_ms)
+{
+       block_console_kthreads = true;
+
+       /* Do not wait when the console lock could not be safely taken. */
+       if (this_cpu_read(printk_context) || in_nmi())
+               return;
+
+       while (timeout_ms > 0) {
+               if (console_trylock()) {
+                       console_unlock();
+                       return;
+               }
+
+               udelay(1000);
+               timeout_ms -= 1;
+       }
+}

diff --git a/kernel/reboot.c b/kernel/reboot.c
index 4177645e74d6af001695d4443fc5d288e67a7ca3..31036368550207195f204201cf90415a833bc010 100644 (file)
--- a/kernel/reboot.c
+++ b/kernel/reboot.c
@@ -74,6 +74,7 @@ void kernel_restart_prepare(char *cmd)
 {
        blocking_notifier_call_chain(&reboot_notifier_list, SYS_RESTART, cmd);
        system_state = SYSTEM_RESTART;
+       try_block_console_kthreads(10000);
        usermodehelper_disable();
        device_shutdown();
 }
@@ -262,6 +263,7 @@ static void kernel_shutdown_prepare(enum system_states state)
        blocking_notifier_call_chain(&reboot_notifier_list,
                (state == SYSTEM_HALT) ? SYS_HALT : SYS_POWER_OFF, NULL);
        system_state = state;
+       try_block_console_kthreads(10000);
        usermodehelper_disable();
        device_shutdown();
 }