unify and overhaul timed futex waits

new features:

- FUTEX_WAIT_BITSET op will be used for timed waits if available. this
  saves a call to clock_gettime.

- error checking for the timespec struct is now inside __timedwait so
  it doesn't need to be duplicated everywhere. cond_timedwait still
  needs to duplicate it to avoid unlocking the mutex, though.

- pushing and popping the cancellation handler is delegated to
  __timedwait, and cancellable/non-cancellable waits are unified.

src/internal/futex.h

@@ -10,6 +10,9 @@
 #define FUTEX_LOCK_PI		6
 #define FUTEX_UNLOCK_PI		7
 #define FUTEX_TRYLOCK_PI	8
+#define FUTEX_WAIT_BITSET	9
+
+#define FUTEX_CLOCK_REALTIME 256
 
 int __futex(volatile int *, int, int, void *);
 

src/internal/pthread_impl.h

@@ -95,8 +95,7 @@ int __libc_sigprocmask(int, const sigset_t *, sigset_t *);
 void __lock(volatile int *);
 void __unmapself(void *, size_t);
 
-int __timedwait(volatile int *, int, clockid_t, const struct timespec *, int);
-int __timedwait_cp(volatile int *, int, clockid_t, const struct timespec *, int);
+int __timedwait(volatile int *, int, clockid_t, const struct timespec *, void (*)(void *), void *, int);
 void __wait(volatile int *, volatile int *, int, int);
 void __wake(volatile int *, int, int);
 

src/thread/__timedwait.c

@@ -1,23 +1,48 @@
+#include <pthread.h>
 #include <time.h>
 #include <errno.h>
 #include "futex.h"
 #include "syscall.h"
 
-int __timedwait(volatile int *addr, int val, clockid_t clk, const struct timespec *at, int priv)
+static int do_wait(volatile int *addr, int val, clockid_t clk, const struct timespec *at, int cp, int priv)
 {
-	int r;
-	struct timespec to;
-	if (at) {
-		clock_gettime(clk, &to);
+	int r, flag = 0;
+	struct timespec to, *top=0;
+
+	if (!at) goto notimeout;
+	if (at->tv_nsec >= 1000000000UL)
+		return EINVAL;
+	if (clk == CLOCK_REALTIME || clk == CLOCK_MONOTONIC) {
+		if (clk == CLOCK_REALTIME) flag = FUTEX_CLOCK_REALTIME;
+		if (cp) r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT_BITSET|flag, val, at, 0, -1);
+		else r = -__syscall(SYS_futex, addr, FUTEX_WAIT_BITSET|flag, val, at, 0, -1);
+		if (r != ENOSYS) goto done;
+	}
+	if (clock_gettime(clk, &to)) return EINVAL;
 	to.tv_sec = at->tv_sec - to.tv_sec;
 	if ((to.tv_nsec = at->tv_nsec - to.tv_nsec) < 0) {
 		to.tv_sec--;
 		to.tv_nsec += 1000000000;
 	}
 	if (to.tv_sec < 0) return ETIMEDOUT;
-	}
-	if (priv) priv = 128; priv=0;
-	r = -__syscall(SYS_futex, (long)addr, FUTEX_WAIT | priv, val, at ? (long)&to : 0);
-	if (r == ETIMEDOUT || r == EINTR) return r;
+	top = &to;
+notimeout:
+	if (cp) r = -__syscall_cp(SYS_futex, addr, FUTEX_WAIT, val, top);
+	else r = -__syscall(SYS_futex, addr, FUTEX_WAIT, val, top);
+done:
+	if (r == EINTR || r == EINVAL || r == ETIMEDOUT) return r;
 	return 0;
 }
+
+int __timedwait(volatile int *addr, int val, clockid_t clk, const struct timespec *at, void (*cleanup)(void *), void *arg, int priv)
+{
+	int r;
+	if (cleanup) {
+		pthread_cleanup_push(cleanup, arg);
+		r = do_wait(addr, val, clk, at, 1, priv);
+		pthread_cleanup_pop(0);
+	} else {
+		r = do_wait(addr, val, clk, at, 0, priv);
+	}
+	return r;
+}

src/thread/__timedwait_cp.c

@@ -1,23 +0,0 @@
-#include <time.h>
-#include <errno.h>
-#include "futex.h"
-#include "syscall.h"
-
-int __timedwait_cp(volatile int *addr, int val, clockid_t clk, const struct timespec *at, int priv)
-{
-	int r;
-	struct timespec to;
-	if (at) {
-		clock_gettime(clk, &to);
-		to.tv_sec = at->tv_sec - to.tv_sec;
-		if ((to.tv_nsec = at->tv_nsec - to.tv_nsec) < 0) {
-			to.tv_sec--;
-			to.tv_nsec += 1000000000;
-		}
-		if (to.tv_sec < 0) return ETIMEDOUT;
-	}
-	if (priv) priv = 128; priv=0;
-	r = -__syscall_cp(SYS_futex, (long)addr, FUTEX_WAIT | priv, val, at ? (long)&to : 0);
-	if (r == ETIMEDOUT || r == EINTR) return r;
-	return 0;
-}

src/thread/pthread_cond_timedwait.c

@@ -9,16 +9,17 @@ int pthread_cond_timedwait(pthread_cond_t *c, pthread_mutex_t *m, const struct t
 {
 	int r, e=0;
 
+	if (ts && ts->tv_nsec >= 1000000000UL)
+		return EINVAL;
+
 	pthread_testcancel();
 
-	pthread_cleanup_push(relock, m);
 	c->_c_block = 1;
 	if ((r=pthread_mutex_unlock(m))) return r;
 
-	do e = __timedwait_cp(&c->_c_block, 1, c->_c_clock, ts, 0);
+	do e = __timedwait(&c->_c_block, 1, c->_c_clock, ts, relock, m, 0);
 	while (e == EINTR);
 
-	pthread_cleanup_pop(0);
 	if ((r=pthread_mutex_lock(m))) return r;
 
 	pthread_testcancel();

src/thread/pthread_join.c

@@ -1,9 +1,13 @@
 #include "pthread_impl.h"
 
+static void dummy(void *p)
+{
+}
+
 int pthread_join(pthread_t t, void **res)
 {
 	int tmp = t->tid;
-	if (tmp) __timedwait_cp(&t->tid, tmp, 0, 0, 1);
+	if (tmp) __timedwait(&t->tid, tmp, 0, 0, dummy, 0, 1);
 	if (res) *res = t->result;
 	if (t->map_base) munmap(t->map_base, t->map_size);
 	return 0;

src/thread/pthread_mutex_timedlock.c

@@ -16,7 +16,7 @@ int pthread_mutex_timedlock(pthread_mutex_t *m, const struct timespec *at)
 		a_inc(&m->_m_waiters);
 		t = r | 0x80000000;
 		a_cas(&m->_m_lock, r, t);
-		r = __timedwait(&m->_m_lock, t, CLOCK_REALTIME, at, 0);
+		r = __timedwait(&m->_m_lock, t, CLOCK_REALTIME, at, 0, 0, 0);
 		a_dec(&m->_m_waiters);
 		if (r && r != EINTR) break;
 	}

src/thread/pthread_rwlock_timedrdlock.c

@@ -5,7 +5,7 @@ int pthread_rwlock_timedrdlock(pthread_rwlock_t *rw, const struct timespec *at)
 	int w=0;
 	while (pthread_rwlock_tryrdlock(rw)) {
 		if (!w) a_inc(&rw->_rw_waiters), w++;
-		if (__timedwait(&rw->_rw_wrlock, 1, CLOCK_REALTIME, at, 0)==ETIMEDOUT) {
+		if (__timedwait(&rw->_rw_wrlock, 1, CLOCK_REALTIME, at, 0, 0, 0)==ETIMEDOUT) {
 			if (w) a_dec(&rw->_rw_waiters);
 			return ETIMEDOUT;
 		}

src/thread/pthread_rwlock_timedwrlock.c

@@ -7,7 +7,7 @@ int pthread_rwlock_timedwrlock(pthread_rwlock_t *rw, const struct timespec *at)
 		if (!w) a_inc(&rw->_rw_waiters), w++;
 		if ((nr=rw->_rw_readers)) p = &rw->_rw_readers;
 		else nr=1, p = &rw->_rw_wrlock;
-		if (__timedwait(p, nr, CLOCK_REALTIME, at, 0)==ETIMEDOUT) {
+		if (__timedwait(p, nr, CLOCK_REALTIME, at, 0, 0, 0)==ETIMEDOUT) {
 			if (w) a_dec(&rw->_rw_waiters);
 			return ETIMEDOUT;
 		}

src/thread/sem_timedwait.c

@@ -10,15 +10,9 @@ int sem_timedwait(sem_t *sem, const struct timespec *at)
 {
 	while (sem_trywait(sem)) {
 		int r;
-		if (at && at->tv_nsec >= 1000000000UL) {
-			errno = EINVAL;
-			return -1;
-		}
 		a_inc(sem->__val+1);
 		a_cas(sem->__val, 0, -1);
-		pthread_cleanup_push(cleanup, sem->__val+1);
-		r = __timedwait_cp(sem->__val, -1, CLOCK_REALTIME, at, 0);
-		pthread_cleanup_pop(1);
+		r = __timedwait(sem->__val, -1, CLOCK_REALTIME, at, cleanup, sem->__val+1, 0);
 		if (r) {
 			errno = r;
 			return -1;