[PATCH] Directed yield: cpu_relax variants for spinlocks and rw-locks

On systems running with virtual cpus there is optimization potential in
regard to spinlocks and rw-locks.  If the virtual cpu that has taken a lock
is known to a cpu that wants to acquire the same lock it is beneficial to
yield the timeslice of the virtual cpu in favour of the cpu that has the
lock (directed yield).

With CONFIG_PREEMPT="n" this can be implemented by the architecture without
common code changes.  Powerpc already does this.

With CONFIG_PREEMPT="y" the lock loops are coded with _raw_spin_trylock,
_raw_read_trylock and _raw_write_trylock in kernel/spinlock.c.  If the lock
could not be taken cpu_relax is called.  A directed yield is not possible
because cpu_relax doesn't know anything about the lock.  To be able to
yield the lock in favour of the current lock holder variants of cpu_relax
for spinlocks and rw-locks are needed.  The new _raw_spin_relax,
_raw_read_relax and _raw_write_relax primitives differ from cpu_relax
insofar that they have an argument: a pointer to the lock structure.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Haavard Skinnemoen <hskinnemoen@atmel.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/include/asm-alpha/spinlock.h b/include/asm-alpha/spinlock.h
index 0c294c9b0c558a8a45f080b8e0b825fadbdf859f..aeeb125f68513df03fe1d625de0cf1883b46c869 100644 (file)
--- a/include/asm-alpha/spinlock.h
+++ b/include/asm-alpha/spinlock.h
@@ -166,4 +166,8 @@ static inline void __raw_write_unlock(raw_rwlock_t * lock)
        lock->lock = 0;
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* _ALPHA_SPINLOCK_H */

diff --git a/include/asm-arm/spinlock.h b/include/asm-arm/spinlock.h
index 01b7c26a30386951905de5ae46d3bc31fdde30cd..861092fbaa53283953bd41c9dbb6e840bb42ff02 100644 (file)
--- a/include/asm-arm/spinlock.h
+++ b/include/asm-arm/spinlock.h
@@ -218,4 +218,8 @@ static inline int __raw_read_trylock(raw_rwlock_t *rw)
 /* read_can_lock - would read_trylock() succeed? */
 #define __raw_read_can_lock(x)         ((x)->lock < 0x80000000)
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* __ASM_SPINLOCK_H */

diff --git a/include/asm-cris/arch-v32/spinlock.h b/include/asm-cris/arch-v32/spinlock.h
index 52df72a622329659d27524f57fc1d2f95556bc13..5f43df0a5fb42df32847984e49e6a2d8db070f2c 100644 (file)
--- a/include/asm-cris/arch-v32/spinlock.h
+++ b/include/asm-cris/arch-v32/spinlock.h
@@ -160,4 +160,8 @@ static __inline__ int is_write_locked(rwlock_t *rw)
        return rw->counter < 0;
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* __ASM_ARCH_SPINLOCK_H */

diff --git a/include/asm-i386/spinlock.h b/include/asm-i386/spinlock.h
index b0b3043f05e154742738f5ff03b7c409d265f2f4..c18b71fae6b38f6ec0a524509deb4a229de002ad 100644 (file)
--- a/include/asm-i386/spinlock.h
+++ b/include/asm-i386/spinlock.h
@@ -205,4 +205,8 @@ static inline void __raw_write_unlock(raw_rwlock_t *rw)
                                 : "+m" (rw->lock) : : "memory");
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* __ASM_SPINLOCK_H */

diff --git a/include/asm-ia64/spinlock.h b/include/asm-ia64/spinlock.h
index 9e83210dc31257adc4c5356b64f37b24e83bf1e4..ff857e31738a00dfbed4534ee1832528e231fb33 100644 (file)
--- a/include/asm-ia64/spinlock.h
+++ b/include/asm-ia64/spinlock.h
@@ -213,4 +213,8 @@ static inline int __raw_read_trylock(raw_rwlock_t *x)
        return (u32)ia64_cmpxchg4_acq((__u32 *)(x), new.word, old.word) == old.word;
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /*  _ASM_IA64_SPINLOCK_H */

diff --git a/include/asm-m32r/spinlock.h b/include/asm-m32r/spinlock.h
index f9f90727a4a1f9c88485150e380db6113d461218..f5cfba81ee10c1f59b007dcf34fd9db508a9bff5 100644 (file)
--- a/include/asm-m32r/spinlock.h
+++ b/include/asm-m32r/spinlock.h
@@ -316,4 +316,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *lock)
        return 0;
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* _ASM_M32R_SPINLOCK_H */

diff --git a/include/asm-mips/spinlock.h b/include/asm-mips/spinlock.h
index 4c1a1b53aeaf1c3db24c8f01fab27147600e0020..c8d5587467bbf6181ba2950fa66bf63c8dcc3858 100644 (file)
--- a/include/asm-mips/spinlock.h
+++ b/include/asm-mips/spinlock.h
@@ -328,4 +328,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)
 }
 
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* _ASM_SPINLOCK_H */

diff --git a/include/asm-parisc/spinlock.h b/include/asm-parisc/spinlock.h
index a93960e232cffabb43de39d78a97dc9d7542f154..e1825530365d3d1724b925fabb54a125429cd564 100644 (file)
--- a/include/asm-parisc/spinlock.h
+++ b/include/asm-parisc/spinlock.h
@@ -152,4 +152,8 @@ static __inline__ int __raw_write_can_lock(raw_rwlock_t *rw)
        return !rw->counter;
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* __ASM_SPINLOCK_H */

diff --git a/include/asm-powerpc/spinlock.h b/include/asm-powerpc/spinlock.h
index c31e4382a7759dba58f18f7925e675538a951e5f..eaccac8327f2d23ffbed6dbc5c303bb669f3fb60 100644 (file)
--- a/include/asm-powerpc/spinlock.h
+++ b/include/asm-powerpc/spinlock.h
@@ -285,5 +285,9 @@ static __inline__ void __raw_write_unlock(raw_rwlock_t *rw)
        rw->lock = 0;
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* __KERNEL__ */
 #endif /* __ASM_SPINLOCK_H */

diff --git a/include/asm-ppc/spinlock.h b/include/asm-ppc/spinlock.h
index 5c64b75f0295bb0eec7697dd79a6b4f171269a2a..fccaf5531e579444b83d46192ef4280245a1a8ae 100644 (file)
--- a/include/asm-ppc/spinlock.h
+++ b/include/asm-ppc/spinlock.h
@@ -161,4 +161,8 @@ static __inline__ void __raw_write_unlock(raw_rwlock_t *rw)
        rw->lock = 0;
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* __ASM_SPINLOCK_H */

diff --git a/include/asm-s390/spinlock.h b/include/asm-s390/spinlock.h
index ce3edf6d63b3bdf4bac03a982d00151577843d4c..5f00feaf1be62865c8a10edffc159f85b4cbf976 100644 (file)
--- a/include/asm-s390/spinlock.h
+++ b/include/asm-s390/spinlock.h
@@ -154,4 +154,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)
        return _raw_write_trylock_retry(rw);
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* __ASM_SPINLOCK_H */

diff --git a/include/asm-sh/spinlock.h b/include/asm-sh/spinlock.h
index 846322d4c35d489daf86e337518dbee79f5c738f..54458fd2498150ce964e159748faac8339c1f235 100644 (file)
--- a/include/asm-sh/spinlock.h
+++ b/include/asm-sh/spinlock.h
@@ -100,4 +100,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)
        return 0;
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* __ASM_SH_SPINLOCK_H */

diff --git a/include/asm-sparc/spinlock.h b/include/asm-sparc/spinlock.h
index 1c75474ba1df706b1abb5e9c96b50f8fc18dfc5b..557d08959d2f319ce90b491672c9d4ba518ec471 100644 (file)
--- a/include/asm-sparc/spinlock.h
+++ b/include/asm-sparc/spinlock.h
@@ -154,6 +154,10 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)
 #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
 #define __raw_read_trylock(lock) generic__raw_read_trylock(lock)
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #define __raw_read_can_lock(rw) (!((rw)->lock & 0xff))
 #define __raw_write_can_lock(rw) (!(rw)->lock)
 

diff --git a/include/asm-sparc64/spinlock.h b/include/asm-sparc64/spinlock.h
index bd5ffc76bc7e6774274b596ca8b43154c894ed08..0006fe9f8c7a03cab556cec5119ad5c016608e74 100644 (file)
--- a/include/asm-sparc64/spinlock.h
+++ b/include/asm-sparc64/spinlock.h
@@ -241,6 +241,10 @@ static int inline __write_trylock(raw_rwlock_t *lock)
 #define __raw_read_can_lock(rw)                (!((rw)->lock & 0x80000000UL))
 #define __raw_write_can_lock(rw)       (!(rw)->lock)
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* !(__ASSEMBLY__) */
 
 #endif /* !(__SPARC64_SPINLOCK_H) */

diff --git a/include/asm-x86_64/spinlock.h b/include/asm-x86_64/spinlock.h
index 3daf5b0059054994c04ba2bce41a8dfa81f701fc..05ef097ba55b26dccd45d997417228ad79a110ba 100644 (file)
--- a/include/asm-x86_64/spinlock.h
+++ b/include/asm-x86_64/spinlock.h
@@ -133,4 +133,8 @@ static inline void __raw_write_unlock(raw_rwlock_t *rw)
                                : "=m" (rw->lock) : : "memory");
 }
 
+#define _raw_spin_relax(lock)  cpu_relax()
+#define _raw_read_relax(lock)  cpu_relax()
+#define _raw_write_relax(lock) cpu_relax()
+
 #endif /* __ASM_SPINLOCK_H */

diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index d48143eafbfde6388717e67c1c3b74011ea8a747..476c3741511b31af1717f90e052880e0e6c9b585 100644 (file)
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -215,7 +215,7 @@ void __lockfunc _##op##_lock(locktype##_t *lock)                    \
                if (!(lock)->break_lock)                                \
                        (lock)->break_lock = 1;                         \
                while (!op##_can_lock(lock) && (lock)->break_lock)      \
-                       cpu_relax();                                    \
+                       _raw_##op##_relax(&lock->raw_lock);             \
        }                                                               \
        (lock)->break_lock = 0;                                         \
 }                                                                      \
@@ -237,7 +237,7 @@ unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)   \
                if (!(lock)->break_lock)                                \
                        (lock)->break_lock = 1;                         \
                while (!op##_can_lock(lock) && (lock)->break_lock)      \
-                       cpu_relax();                                    \
+                       _raw_##op##_relax(&lock->raw_lock);             \
        }                                                               \
        (lock)->break_lock = 0;                                         \
        return flags;                                                   \