fix possible failure-to-wake deadlock with robust mutexes

[musl] / src / thread / pthread_mutex_trylock.c

diff --git a/src/thread/pthread_mutex_trylock.c b/src/thread/pthread_mutex_trylock.c
index f871e9e..e851517 100644 (file)
--- a/src/thread/pthread_mutex_trylock.c
+++ b/src/thread/pthread_mutex_trylock.c
@@ -9,6 +9,7 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m)
 
        if (!self->robust_list.off) {
                __syscall(SYS_set_robust_list, &self->robust_list, 3*sizeof(long));
+               self->robust_list.head = &self->robust_list.head;
                self->robust_list.off = (char*)&m->_m_lock-(char *)&m->_m_next;
        }
 
@@ -19,8 +20,12 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m)
                m->_m_count++;
                return 0;
        }
+       if (own == 0x40000000) return ENOTRECOVERABLE;
 
-       self->robust_list.pending = &m->_m_next;
+       if (m->_m_type & 128) {
+               if (m->_m_waiters) tid |= 0x80000000;
+               self->robust_list.pending = &m->_m_next;
+       }
 
        if ((own && (!(own & 0x40000000) || !(type & 4)))
            || a_cas(&m->_m_lock, old, tid) != old) {
@@ -28,23 +33,17 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m)
                return EBUSY;
        }
 
-       m->_m_next = self->robust_list.head;
+       volatile void *next = self->robust_list.head;
+       m->_m_next = next;
        m->_m_prev = &self->robust_list.head;
-       if (self->robust_list.head)
-               self->robust_list.head[-1] = &m->_m_next;
+       if (next != &self->robust_list.head) *(volatile void *volatile *)
+               ((char *)next - sizeof(void *)) = &m->_m_next;
        self->robust_list.head = &m->_m_next;
        self->robust_list.pending = 0;
 
-       if (type < 4) return 0;
-
-       if (type >= 8) {
-               m->_m_lock = 0;
-               return ENOTRECOVERABLE;
-       }
-
        if (own) {
                m->_m_count = 0;
-               m->_m_type += 8;
+               m->_m_type |= 8;
                return EOWNERDEAD;
        }