* degenerate list of one member.
*
* Waiter lists attached to the condition variable itself are
- * protected by the lock on the cv. Detached waiter lists are
- * protected by the associated mutex. The hand-off between protections
- * is handled by a "barrier" lock in each node, which disallows
- * signaled waiters from making forward progress to the code that will
- * access the list using the mutex until the list is in a consistent
- * state and the cv lock as been released.
+ * protected by the lock on the cv. Detached waiter lists are never
+ * modified again, but can only be traversed in reverse order, and are
+ * protected by the "barrier" locks in each node, which are unlocked
+ * in turn to control wake order.
*
* Since process-shared cond var semantics do not necessarily allow
* one thread to see another's automatic storage (they may be in
struct waiter {
struct waiter *prev, *next;
- int state, barrier, requeued, mutex_ret;
+ int state, barrier, mutex_ret;
int *notify;
pthread_mutex_t *mutex;
pthread_cond_t *cond;
__wake(l, 1, 1);
}
+static inline void unlock_requeue(volatile int *l, volatile int *r, int w)
+{
+ a_store(l, 0);
+ if (w) __wake(l, 1, 1);
+ else __syscall(SYS_futex, l, FUTEX_REQUEUE|128, 0, 1, r) != -ENOSYS
+ || __syscall(SYS_futex, l, FUTEX_REQUEUE, 0, 1, r);
+}
+
enum {
WAITING,
SIGNALED,
static void unwait(void *arg)
{
- struct waiter *node = arg, *p;
+ struct waiter *node = arg;
if (node->shared) {
pthread_cond_t *c = node->cond;
if (a_fetch_add(node->notify, -1)==1)
__wake(node->notify, 1, 1);
}
+ } else {
+ /* Lock barrier first to control wake order. */
+ lock(&node->barrier);
}
node->mutex_ret = pthread_mutex_lock(node->mutex);
if (oldstate == WAITING) return;
- /* If the mutex can't be locked, we're in big trouble because
- * it's all that protects access to the shared list state.
- * In order to prevent catastrophic stack corruption from
- * unsynchronized access, simply deadlock. */
- if (node->mutex_ret && node->mutex_ret != EOWNERDEAD)
- for (;;) lock(&(int){0});
-
- /* Wait until control of the list has been handed over from
- * the cv lock (signaling thread) to the mutex (waiters). */
- lock(&node->barrier);
-
- /* If this thread was requeued to the mutex, undo the extra
- * waiter count that was added to the mutex. */
- if (node->requeued) a_dec(&node->mutex->_m_waiters);
-
- /* Find a thread to requeue to the mutex, starting from the
- * end of the list (oldest waiters). */
- for (p=node; p->next; p=p->next);
- if (p==node) p=node->prev;
- for (; p && p->requeued; p=p->prev);
- if (p==node) p=node->prev;
- if (p) {
- p->requeued = 1;
- a_inc(&node->mutex->_m_waiters);
- /* The futex requeue command cannot requeue from
- * private to shared, so for process-shared mutexes,
- * simply wake the target. */
- int wake = node->mutex->_m_type & 128;
- __syscall(SYS_futex, &p->state, FUTEX_REQUEUE|128,
- wake, 1, &node->mutex->_m_lock) != -EINVAL
- || __syscall(SYS_futex, &p->state, FUTEX_REQUEUE,
- 0, 1, &node->mutex->_m_lock);
- }
+ if (!node->next) a_inc(&node->mutex->_m_waiters);
- /* Remove this thread from the list. */
- if (node->next) node->next->prev = node->prev;
- if (node->prev) node->prev->next = node->next;
+ /* Unlock the barrier that's holding back the next waiter, and
+ * either wake it or requeue it to the mutex. */
+ if (node->prev) {
+ unlock_requeue(&node->prev->barrier,
+ &node->mutex->_m_lock,
+ node->mutex->_m_type & 128);
+ } else {
+ a_dec(&node->mutex->_m_waiters);
+ }
}
int pthread_cond_timedwait(pthread_cond_t *restrict c, pthread_mutex_t *restrict m, const struct timespec *restrict ts)
} else {
lock(&c->_c_lock);
- node.barrier = 1;
- fut = &node.state;
- seq = node.state = WAITING;
+ seq = node.barrier = 2;
+ fut = &node.barrier;
+ node.state = WAITING;
node.next = c->_c_head;
c->_c_head = &node;
if (!c->_c_tail) c->_c_tail = &node;
int __private_cond_signal(pthread_cond_t *c, int n)
{
- struct waiter *p, *q=0;
+ struct waiter *p, *first=0;
int ref = 0, cur;
lock(&c->_c_lock);
for (p=c->_c_tail; n && p; p=p->prev) {
- /* The per-waiter-node barrier lock is held at this
- * point, so while the following CAS may allow forward
- * progress in the target thread, it doesn't allow
- * access to the waiter list yet. Ideally the target
- * does not run until the futex wake anyway. */
if (a_cas(&p->state, WAITING, SIGNALED) != WAITING) {
ref++;
p->notify = &ref;
} else {
n--;
- if (!q) q=p;
+ if (!first) first=p;
}
}
/* Split the list, leaving any remainder on the cv. */
* signaled threads to proceed. */
while ((cur = ref)) __wait(&ref, 0, cur, 1);
- /* Wake the first signaled thread and unlock the per-waiter
- * barriers preventing their forward progress. */
- for (p=q; p; p=q) {
- q = p->prev;
- if (!p->next) __wake(&p->state, 1, 1);
- unlock(&p->barrier);
- }
+ /* Allow first signaled waiter, if any, to proceed. */
+ if (first) unlock(&first->barrier);
+
return 0;
}
projects / musl / blobdiff