Add Requeue-PI support for x86 arch.

nptl/ChangeLog

@@ -1,3 +1,14 @@
+2009-12-01  Dinakar Guniguntala  <dino@in.ibm.com>
+
+	* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.h: Define
+	FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: If mutex
+	is a non robust PI mutex, then use FUTEX_CMP_REQUEUE_PI.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: If mutex
+	is a non robust PI mutex, then use FUTEX_WAIT_REQUEUE_PI.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
+
 2009-12-12  Ulrich Drepper  <drepper@redhat.com>
 
 	* sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S (sem_timedwait):

nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S

@@ -91,12 +91,17 @@ __pthread_cond_broadcast:
 8:	cmpl	$-1, %edi
 	je	9f
 
-	/* XXX: The kernel so far doesn't support requeue to PI futex.  */
+	/* Do not use requeue for pshared condvars.  */
-	/* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
+	testl	$PS_BIT, MUTEX_KIND(%edi)
-	   type of futex (private resp. shared).  */
-	testl	$(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
 	jne	9f
 
+	/* Requeue to a non-robust PI mutex if the PI bit is set and
+	   the robust bit is not set.  */
+	movl	MUTEX_KIND(%edi), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	je	81f
+
 	/* Wake up all threads.  */
 #ifdef __ASSUME_PRIVATE_FUTEX
 	movl	$(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
@@ -138,6 +143,23 @@ __pthread_cond_broadcast:
 
 	cfi_restore_state
 
+81:	movl	$(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+	movl	$SYS_futex, %eax
+	movl	$0x7fffffff, %esi
+	movl	$1, %edx
+	/* Get the address of the futex involved.  */
+# if MUTEX_FUTEX != 0
+	addl	$MUTEX_FUTEX, %edi
+# endif
+	int	$0x80
+
+	/* For any kind of error, which mainly is EAGAIN, we try again
+	with WAKE.  The general test also covers running on old
+	kernels.  */
+	cmpl	$0xfffff001, %eax
+	jb	6b
+	jmp	9f
+
 	/* Initial locking failed.  */
 1:
 #if cond_lock == 0

nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S

@@ -22,6 +22,7 @@
 #include <lowlevellock.h>
 #include <lowlevelcond.h>
 #include <kernel-features.h>
+#include <pthread-pi-defines.h>
 #include <pthread-errnos.h>
 
 
@@ -86,7 +87,17 @@ __pthread_cond_signal:
 #endif
 	cmpl	$-1, dep_mutex-cond_futex(%ebx)
 	sete	%cl
-	subl	$1, %ecx
+	je	8f
+
+	movl	dep_mutex-cond_futex(%ebx), %edx
+	/* Requeue to a non-robust PI mutex if the PI bit is set and
+	   the robust bit is not set.  */
+	movl	MUTEX_KIND(%edx), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	je	9f
+
+8:	subl	$1, %ecx
 #ifdef __ASSUME_PRIVATE_FUTEX
 	andl	$FUTEX_PRIVATE_FLAG, %ecx
 #else
@@ -124,8 +135,34 @@ __pthread_cond_signal:
 
 	cfi_restore_state
 
-7:	/* %ecx should be either FUTEX_WAKE_OP or
+9:	movl	$(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
-	   FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall.  */
+	movl	$SYS_futex, %eax
+	movl	$1, %edx
+	xorl	%esi, %esi
+	movl	dep_mutex-cond_futex(%ebx), %edi
+	movl	(%ebx), %ebp
+	/* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
+	   sysenter.
+	ENTER_KERNEL  */
+	int	$0x80
+	popl	%ebp
+	popl	%esi
+
+	leal	-cond_futex(%ebx), %edi
+
+	/* For any kind of error, we try again with WAKE.
+	   The general test also covers running on old kernels.  */
+	cmpl	$-4095, %eax
+	jb	4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+	andl	$FUTEX_PRIVATE_FLAG, %ecx
+#else
+	andl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	orl	$FUTEX_WAKE, %ecx
+
 	xorl	$(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
 	movl	$SYS_futex, %eax
 	/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.

nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S

@@ -22,6 +22,7 @@
 #include <lowlevellock.h>
 #include <lowlevelcond.h>
 #include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
 #include <kernel-features.h>
 
 
@@ -95,7 +96,7 @@ __pthread_cond_timedwait:
 	addl	$1, cond_futex(%ebx)
 	addl	$(1 << nwaiters_shift), cond_nwaiters(%ebx)
 
-#define FRAME_SIZE 24
+#define FRAME_SIZE 32
 	subl	$FRAME_SIZE, %esp
 	cfi_adjust_cfa_offset(FRAME_SIZE)
 
@@ -107,8 +108,10 @@ __pthread_cond_timedwait:
 	movl	%edx, 16(%esp)
 	movl	%eax, 20(%esp)
 
+	/* Reset the pi-requeued flag.  */
+8:	movl	$0, 24(%esp)
 	/* Get the current time.  */
-8:	movl	%ebx, %edx
+	movl	%ebx, %edx
 #ifdef __NR_clock_gettime
 	/* Get the clock number.  */
 	movl	cond_nwaiters(%ebx), %ebx
@@ -158,6 +161,7 @@ __pthread_cond_timedwait:
 	movl	%edx, 8(%esp)
 
 	movl	cond_futex(%ebx), %edi
+	movl	%edi, 28(%esp)
 
 	/* Unlock.  */
 	LOCK
@@ -172,13 +176,50 @@ __pthread_cond_timedwait:
 4:	call	__pthread_enable_asynccancel
 	movl	%eax, (%esp)
 
-	leal	4(%esp), %esi
 #if FUTEX_PRIVATE_FLAG > 255
 	xorl	%ecx, %ecx
 #endif
 	cmpl	$-1, dep_mutex(%ebx)
 	sete	%cl
-	subl	$1, %ecx
+	je	40f
+
+	movl	dep_mutex(%ebx), %edi
+	/* Requeue to a non-robust PI mutex if the PI bit is set and
+	   the robust bit is not set.  */
+	movl	MUTEX_KIND(%edi), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	jne	40f
+	
+	movl	$(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+	/* The following only works like this because we only support
+	   two clocks, represented using a single bit.  */
+	testl	$1, cond_nwaiters(%ebx)
+	/* XXX Need to implement using sete instead of a jump.  */
+	jne	42f
+	orl	$FUTEX_CLOCK_REALTIME, %ecx
+	
+	/* Requeue-PI uses absolute timeout */
+42:	leal	(%ebp), %esi
+	movl	28(%esp), %edx
+	addl	$cond_futex, %ebx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	subl	$cond_futex, %ebx
+	movl	%eax, %esi
+	/* Set the pi-requeued flag only if the kernel has returned 0. The
+	   kernel does not hold the mutex on ETIMEDOUT or any other error.  */
+	cmpl	$0, %eax
+	sete	24(%esp)
+	je	41f
+
+	/* Normal and PI futexes dont mix. Use normal futex functions only
+	   if the kernel does not support the PI futex functions.  */
+	cmpl	$-ENOSYS, %eax
+	jne	41f
+	xorl	%ecx, %ecx
+	
+40:	subl	$1, %ecx
 #ifdef __ASSUME_PRIVATE_FUTEX
 	andl	$FUTEX_PRIVATE_FLAG, %ecx
 #else
@@ -187,7 +228,8 @@ __pthread_cond_timedwait:
 #if FUTEX_WAIT != 0
 	addl	$FUTEX_WAIT, %ecx
 #endif
-	movl	%edi, %edx
+	leal	4(%esp), %esi
+	movl	28(%esp), %edx
 	addl	$cond_futex, %ebx
 .Ladd_cond_futex:
 	movl	$SYS_futex, %eax
@@ -196,7 +238,7 @@ __pthread_cond_timedwait:
 .Lsub_cond_futex:
 	movl	%eax, %esi
 
-	movl	(%esp), %eax
+41:	movl	(%esp), %eax
 	call	__pthread_disable_asynccancel
 .LcleanupEND:
 
@@ -284,10 +326,16 @@ __pthread_cond_timedwait:
 #endif
 	jne	10f
 
+11:	xorl	%eax, %eax
+	/* With requeue_pi, the mutex lock is held in the kernel.  */
+	movl	24(%esp), %ecx
+	testl	%ecx, %ecx
+	jnz	26f
+
 	/* Remove cancellation handler.  */
-11:	movl	24+FRAME_SIZE(%esp), %eax
+	movl	24+FRAME_SIZE(%esp), %eax
 	call	__pthread_mutex_cond_lock
-	addl	$FRAME_SIZE, %esp
+26:	addl	$FRAME_SIZE, %esp
 	cfi_adjust_cfa_offset(-FRAME_SIZE);
 
 	/* We return the result of the mutex_lock operation if it failed.  */
@@ -317,6 +365,9 @@ __pthread_cond_timedwait:
 
 	cfi_restore_state
 
+27:	call	__pthread_mutex_cond_lock_adjust
+	jmp	26b
+
 	/* Initial locking failed.  */
 1:
 #if cond_lock == 0

nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S

@@ -22,6 +22,8 @@
 #include <lowlevellock.h>
 #include <lowlevelcond.h>
 #include <tcb-offsets.h>
+#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
 #include <kernel-features.h>
 
 
@@ -43,6 +45,9 @@ __pthread_cond_wait:
 	cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
 #endif
 
+	pushl	%ebp
+	cfi_adjust_cfa_offset(4)
+	cfi_rel_offset(%ebp, 0)
 	pushl	%edi
 	cfi_adjust_cfa_offset(4)
 	cfi_rel_offset(%edi, 0)
@@ -55,7 +60,7 @@ __pthread_cond_wait:
 	cfi_remember_state
 
 	xorl	%esi, %esi
-	movl	16(%esp), %ebx
+	movl	20(%esp), %ebx
 
 	/* Get internal lock.  */
 	movl	$1, %edx
@@ -71,7 +76,7 @@ __pthread_cond_wait:
 	/* Store the reference to the mutex.  If there is already a
 	   different value in there this is a bad user bug.  */
 2:	cmpl	$-1, dep_mutex(%ebx)
-	movl	20(%esp), %eax
+	movl	24(%esp), %eax
 	je	15f
 	movl	%eax, dep_mutex(%ebx)
 
@@ -87,7 +92,7 @@ __pthread_cond_wait:
 	addl	$1, cond_futex(%ebx)
 	addl	$(1 << nwaiters_shift), cond_nwaiters(%ebx)
 
-#define FRAME_SIZE 16
+#define FRAME_SIZE 20
 	subl	$FRAME_SIZE, %esp
 	cfi_adjust_cfa_offset(FRAME_SIZE)
 
@@ -99,8 +104,10 @@ __pthread_cond_wait:
 	movl	%edx, 8(%esp)
 	movl	%eax, 12(%esp)
 
-8:	movl	cond_futex(%ebx), %edi
+	/* Reset the pi-requeued flag.  */
-
+8:	movl	$0, 16(%esp)
+ 	movl	cond_futex(%ebx), %ebp
+	
 	/* Unlock.  */
 	LOCK
 #if cond_lock == 0
@@ -114,12 +121,39 @@ __pthread_cond_wait:
 4:	call	__pthread_enable_asynccancel
 	movl	%eax, (%esp)
 
-#if FUTEX_PRIVATE_FLAG > 255
 	xorl	%ecx, %ecx
-#endif
 	cmpl	$-1, dep_mutex(%ebx)
 	sete	%cl
-	subl	$1, %ecx
+	je 	18f
+
+	movl	dep_mutex(%ebx), %edi
+	/* Requeue to a non-robust PI mutex if the PI bit is set and
+	   the robust bit is not set.  */
+	movl	MUTEX_KIND(%edi), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	jne	18f
+	
+	movl	$(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+	movl	%ebp, %edx
+	xorl	%esi, %esi
+	addl	$cond_futex, %ebx
+	movl	$SYS_futex, %eax
+	ENTER_KERNEL
+	subl	$cond_futex, %ebx
+	/* Set the pi-requeued flag only if the kernel has returned 0. The
+	   kernel does not hold the mutex on error.  */
+	cmpl	$0, %eax
+	sete	16(%esp)
+	je	19f
+
+	/* Normal and PI futexes dont mix. Use normal futex functions only
+	   if the kernel does not support the PI futex functions.  */
+	cmpl	$-ENOSYS, %eax
+	jne	19f
+	xorl	%ecx, %ecx
+	
+18:	subl	$1, %ecx
 #ifdef __ASSUME_PRIVATE_FUTEX
 	andl	$FUTEX_PRIVATE_FLAG, %ecx
 #else
@@ -128,7 +162,7 @@ __pthread_cond_wait:
 #if FUTEX_WAIT != 0
 	addl	$FUTEX_WAIT, %ecx
 #endif
-	movl	%edi, %edx
+	movl	%ebp, %edx
 	addl	$cond_futex, %ebx
 .Ladd_cond_futex:
 	movl	$SYS_futex, %eax
@@ -136,7 +170,7 @@ __pthread_cond_wait:
 	subl	$cond_futex, %ebx
 .Lsub_cond_futex:
 
-	movl	(%esp), %eax
+19:	movl	(%esp), %eax
 	call	__pthread_disable_asynccancel
 .LcleanupEND:
 
@@ -212,9 +246,15 @@ __pthread_cond_wait:
 #endif
 	jne	10f
 
-11:	movl	20+FRAME_SIZE(%esp), %eax
+	/* With requeue_pi, the mutex lock is held in the kernel.  */
+11:	xorl	%eax, %eax
+	movl	16(%esp), %ecx
+	testl	%ecx, %ecx
+	jnz	20f
+
+	movl	24+FRAME_SIZE(%esp), %eax
 	call	__pthread_mutex_cond_lock
-	addl	$FRAME_SIZE, %esp
+20:	addl	$FRAME_SIZE, %esp
 	cfi_adjust_cfa_offset(-FRAME_SIZE);
 
 14:	popl	%ebx
@@ -226,12 +266,19 @@ __pthread_cond_wait:
 	popl	%edi
 	cfi_adjust_cfa_offset(-4)
 	cfi_restore(%edi)
+	popl	%ebp
+	cfi_adjust_cfa_offset(-4)
+	cfi_restore(%ebp)
 
 	/* We return the result of the mutex_lock operation.  */
 	ret
 
 	cfi_restore_state
 
+21:	call	__pthread_mutex_cond_lock_adjust
+	xorl	%eax, %eax
+	jmp	20b
+
 	/* Initial locking failed.  */
 1:
 #if cond_lock == 0
@@ -484,7 +531,7 @@ __condvar_w_cleanup:
 	movl	$0x7fffffff, %edx
 	ENTER_KERNEL
 
-5:	movl	20+FRAME_SIZE(%esp), %eax
+5:	movl	24+FRAME_SIZE(%esp), %eax
 	call	__pthread_mutex_cond_lock
 
 	movl	%esi, (%esp)

nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h

@@ -54,6 +54,8 @@
 #define FUTEX_TRYLOCK_PI	8
 #define FUTEX_WAIT_BITSET	9
 #define FUTEX_WAKE_BITSET	10
+#define FUTEX_WAIT_REQUEUE_PI	11
+#define FUTEX_CMP_REQUEUE_PI	12
 #define FUTEX_PRIVATE_FLAG	128
 #define FUTEX_CLOCK_REALTIME	256