struct waiter {
struct waiter *prev, *next;
- int state, barrier, requeued, mutex_ret;
- int *notify;
- pthread_mutex_t *mutex;
- pthread_cond_t *cond;
- int shared;
+ volatile int state, barrier;
+ volatile int *notify;
};
/* Self-synchronized-destruction-safe lock functions */
__wake(l, 1, 1);
}
+static inline void unlock_requeue(volatile int *l, volatile int *r, int w)
+{
+ a_store(l, 0);
+ if (w) __wake(l, 1, 1);
+ else __syscall(SYS_futex, l, FUTEX_REQUEUE|FUTEX_PRIVATE, 0, 1, r) != -ENOSYS
+ || __syscall(SYS_futex, l, FUTEX_REQUEUE, 0, 1, r);
+}
+
enum {
WAITING,
SIGNALED,
LEAVING,
};
-static void unwait(void *arg)
-{
- struct waiter *node = arg;
-
- if (node->shared) {
- pthread_cond_t *c = node->cond;
- pthread_mutex_t *m = node->mutex;
- if (a_fetch_add(&c->_c_waiters, -1) == -0x7fffffff)
- __wake(&c->_c_waiters, 1, 0);
- node->mutex_ret = pthread_mutex_lock(m);
- return;
- }
-
- int oldstate = a_cas(&node->state, WAITING, LEAVING);
-
- if (oldstate == WAITING) {
- /* Access to cv object is valid because this waiter was not
- * yet signaled and a new signal/broadcast cannot return
- * after seeing a LEAVING waiter without getting notified
- * via the futex notify below. */
-
- pthread_cond_t *c = node->cond;
- lock(&c->_c_lock);
-
- if (c->_c_head == node) c->_c_head = node->next;
- else if (node->prev) node->prev->next = node->next;
- if (c->_c_tail == node) c->_c_tail = node->prev;
- else if (node->next) node->next->prev = node->prev;
-
- unlock(&c->_c_lock);
-
- if (node->notify) {
- if (a_fetch_add(node->notify, -1)==1)
- __wake(node->notify, 1, 1);
- }
- } else {
- /* Lock barrier first to control wake order. */
- lock(&node->barrier);
- }
-
- node->mutex_ret = pthread_mutex_lock(node->mutex);
-
- if (oldstate == WAITING) return;
-
- /* If this thread was requeued to the mutex, undo the extra
- * waiter count that was added to the mutex. */
- if (node->requeued) a_dec(&node->mutex->_m_waiters);
-
- /* Unlock the barrier that's holding back the next waiter,
- * and either wake it or requeue it to the mutex. */
- if (node->prev) {
- unlock(&node->prev->barrier);
- node->prev->requeued = 1;
- a_inc(&node->mutex->_m_waiters);
- /* The futex requeue command cannot requeue from
- * private to shared, so for process-shared mutexes,
- * simply wake the target. */
- int wake = node->mutex->_m_type & 128;
- __syscall(SYS_futex, &node->prev->state, FUTEX_REQUEUE|128,
- wake, 1, &node->mutex->_m_lock) != -EINVAL
- || __syscall(SYS_futex, &node->prev->state, FUTEX_REQUEUE,
- 0, 1, &node->mutex->_m_lock);
- }
-}
-
-int pthread_cond_timedwait(pthread_cond_t *restrict c, pthread_mutex_t *restrict m, const struct timespec *restrict ts)
+int __pthread_cond_timedwait(pthread_cond_t *restrict c, pthread_mutex_t *restrict m, const struct timespec *restrict ts)
{
- struct waiter node = { .cond = c, .mutex = m };
- int e, seq, *fut, clock = c->_c_clock;
+ struct waiter node = { 0 };
+ int e, seq, clock = c->_c_clock, cs, shared=0, oldstate, tmp;
+ volatile int *fut;
if ((m->_m_type&15) && (m->_m_lock&INT_MAX) != __pthread_self()->tid)
return EPERM;
if (ts && ts->tv_nsec >= 1000000000UL)
return EINVAL;
- pthread_testcancel();
+ __pthread_testcancel();
if (c->_c_shared) {
- node.shared = 1;
+ shared = 1;
fut = &c->_c_seq;
seq = c->_c_seq;
a_inc(&c->_c_waiters);
} else {
lock(&c->_c_lock);
- node.barrier = 1;
- fut = &node.state;
- seq = node.state = WAITING;
+ seq = node.barrier = 2;
+ fut = &node.barrier;
+ node.state = WAITING;
node.next = c->_c_head;
c->_c_head = &node;
if (!c->_c_tail) c->_c_tail = &node;
unlock(&c->_c_lock);
}
- pthread_mutex_unlock(m);
+ __pthread_mutex_unlock(m);
- do e = __timedwait(fut, seq, clock, ts, unwait, &node, !node.shared);
+ __pthread_setcancelstate(PTHREAD_CANCEL_MASKED, &cs);
+ if (cs == PTHREAD_CANCEL_DISABLE) __pthread_setcancelstate(cs, 0);
+
+ do e = __timedwait_cp(fut, seq, clock, ts, !shared);
while (*fut==seq && (!e || e==EINTR));
if (e == EINTR) e = 0;
- unwait(&node);
+ if (shared) {
+ /* Suppress cancellation if a signal was potentially
+ * consumed; this is a legitimate form of spurious
+ * wake even if not. */
+ if (e == ECANCELED && c->_c_seq != seq) e = 0;
+ if (a_fetch_add(&c->_c_waiters, -1) == -0x7fffffff)
+ __wake(&c->_c_waiters, 1, 0);
+ oldstate = WAITING;
+ goto relock;
+ }
+
+ oldstate = a_cas(&node.state, WAITING, LEAVING);
- return node.mutex_ret ? node.mutex_ret : e;
+ if (oldstate == WAITING) {
+ /* Access to cv object is valid because this waiter was not
+ * yet signaled and a new signal/broadcast cannot return
+ * after seeing a LEAVING waiter without getting notified
+ * via the futex notify below. */
+
+ lock(&c->_c_lock);
+
+ if (c->_c_head == &node) c->_c_head = node.next;
+ else if (node.prev) node.prev->next = node.next;
+ if (c->_c_tail == &node) c->_c_tail = node.prev;
+ else if (node.next) node.next->prev = node.prev;
+
+ unlock(&c->_c_lock);
+
+ if (node.notify) {
+ if (a_fetch_add(node.notify, -1)==1)
+ __wake(node.notify, 1, 1);
+ }
+ } else {
+ /* Lock barrier first to control wake order. */
+ lock(&node.barrier);
+ }
+
+relock:
+ /* Errors locking the mutex override any existing error or
+ * cancellation, since the caller must see them to know the
+ * state of the mutex. */
+ if ((tmp = pthread_mutex_lock(m))) e = tmp;
+
+ if (oldstate == WAITING) goto done;
+
+ if (!node.next && !(m->_m_type & 8))
+ a_inc(&m->_m_waiters);
+
+ /* Unlock the barrier that's holding back the next waiter, and
+ * either wake it or requeue it to the mutex. */
+ if (node.prev) {
+ int val = m->_m_lock;
+ if (val>0) a_cas(&m->_m_lock, val, val|0x80000000);
+ unlock_requeue(&node.prev->barrier, &m->_m_lock, m->_m_type & (8|128));
+ } else if (!(m->_m_type & 8)) {
+ a_dec(&m->_m_waiters);
+ }
+
+ /* Since a signal was consumed, cancellation is not permitted. */
+ if (e == ECANCELED) e = 0;
+
+done:
+ __pthread_setcancelstate(cs, 0);
+
+ if (e == ECANCELED) {
+ __pthread_testcancel();
+ __pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0);
+ }
+
+ return e;
}
int __private_cond_signal(pthread_cond_t *c, int n)
{
struct waiter *p, *first=0;
- int ref = 0, cur;
+ volatile int ref = 0;
+ int cur;
lock(&c->_c_lock);
for (p=c->_c_tail; n && p; p=p->prev) {
- /* The per-waiter-node barrier lock is held at this
- * point, so while the following CAS may allow forward
- * progress in the target thread, it doesn't allow
- * access to the waiter list yet. Ideally the target
- * does not run until the futex wake anyway. */
if (a_cas(&p->state, WAITING, SIGNALED) != WAITING) {
ref++;
p->notify = &ref;
while ((cur = ref)) __wait(&ref, 0, cur, 1);
/* Allow first signaled waiter, if any, to proceed. */
- if (first) {
- __wake(&first->state, 1, 1);
- unlock(&first->barrier);
- }
+ if (first) unlock(&first->barrier);
return 0;
}
+
+weak_alias(__pthread_cond_timedwait, pthread_cond_timedwait);