X-Git-Url: http://nsz.repo.hu/git/?p=musl;a=blobdiff_plain;f=src%2Fldso%2Fdynlink.c;h=39a276088fe106c8f33e653bb2eae4918adb756e;hp=3f9bc1451fefa2e4c2dbfb730288f4c443def4c9;hb=deb15b3cf23cb6093f439f7b37eaeb1263df5399;hpb=5f88c0edd5db0916cbbcdec9042d4986ca778ae8 diff --git a/src/ldso/dynlink.c b/src/ldso/dynlink.c index 3f9bc145..39a27608 100644 --- a/src/ldso/dynlink.c +++ b/src/ldso/dynlink.c @@ -19,7 +19,6 @@ #include #include "pthread_impl.h" #include "libc.h" -#undef libc static int errflag; static char errbuf[128]; @@ -40,6 +39,9 @@ typedef Elf64_Sym Sym; #define R_SYM(x) ((x)>>32) #endif +#define MAXP2(a,b) (-(-(a)&-(b))) +#define ALIGN(x,y) ((x)+(y)-1 & -(y)) + struct debug { int ver; void *head; @@ -72,6 +74,7 @@ struct dso { void **new_dtv; unsigned char *new_tls; int new_dtv_idx, new_tls_idx; + struct dso *fini_next; char *shortname; char buf[]; }; @@ -86,7 +89,7 @@ struct symdef { void __init_ssp(size_t *); void *__install_initial_tls(void *); -static struct dso *head, *tail, *libc; +static struct dso *head, *tail, *ldso, *fini_head; static char *env_path, *sys_path, *r_path; static int ssp_used; static int runtime; @@ -96,11 +99,12 @@ static jmp_buf rtld_fail; static pthread_rwlock_t lock; static struct debug debug; static size_t *auxv; -static size_t tls_cnt, tls_size; +static size_t tls_cnt, tls_offset, tls_align = 4*sizeof(size_t); +static pthread_mutex_t init_fini_lock = { ._m_type = PTHREAD_MUTEX_RECURSIVE }; struct debug *_dl_debug_addr = &debug; -#define AUX_CNT 24 +#define AUX_CNT 38 #define DYN_CNT 34 static void decode_vec(size_t *v, size_t *a, size_t cnt) @@ -204,14 +208,20 @@ static struct symdef find_sym(struct dso *dso, const char *s, int need_def) if (!h) h = sysv_hash(s); sym = sysv_lookup(s, h, dso); } - if (sym && (!need_def || sym->st_shndx) && sym->st_value - && (1<<(sym->st_info&0xf) & OK_TYPES) - && (1<<(sym->st_info>>4) & OK_BINDS)) { - if (def.sym && sym->st_info>>4 == STB_WEAK) continue; - def.sym = sym; - def.dso = dso; - if (sym->st_info>>4 == STB_GLOBAL) break; - } + if (!sym) continue; + if (!sym->st_shndx) + if (need_def || (sym->st_info&0xf) == STT_TLS) + continue; + if (!sym->st_value) + if ((sym->st_info&0xf) != STT_TLS) + continue; + if (!(1<<(sym->st_info&0xf) & OK_TYPES)) continue; + if (!(1<<(sym->st_info>>4) & OK_BINDS)) continue; + + if (def.sym && sym->st_info>>4 == STB_WEAK) continue; + def.sym = sym; + def.dso = dso; + if (sym->st_info>>4 == STB_GLOBAL) break; } return def; } @@ -435,12 +445,12 @@ static struct dso *load_library(const char *name) size_t l = z-name; for (rp=reserved; *rp && memcmp(name+3, rp, l-3); rp+=strlen(rp)+1); if (*rp) { - if (!libc->prev) { - tail->next = libc; - libc->prev = tail; - tail = libc->next ? libc->next : libc; + if (!ldso->prev) { + tail->next = ldso; + ldso->prev = tail; + tail = ldso->next ? ldso->next : ldso; } - return libc; + return ldso; } } } @@ -503,7 +513,7 @@ static struct dso *load_library(const char *name) if (runtime && temp_dso.tls_image) { size_t per_th = temp_dso.tls_size + temp_dso.tls_align + sizeof(void *) * (tls_cnt+3); - n_th = __libc.threads_minus_1 + 1; + n_th = libc.threads_minus_1 + 1; if (n_th > SSIZE_MAX / per_th) alloc_size = SIZE_MAX; else alloc_size += n_th * per_th; } @@ -522,9 +532,23 @@ static struct dso *load_library(const char *name) /* Add a shortname only if name arg was not an explicit pathname. */ if (pathname != name) p->shortname = strrchr(p->name, '/')+1; if (p->tls_image) { + if (runtime && !__pthread_self_init()) { + free(p); + munmap(map, map_len); + return 0; + } p->tls_id = ++tls_cnt; - tls_size += p->tls_size + p->tls_align + 8*sizeof(size_t) - 1 - & -4*sizeof(size_t); + tls_align = MAXP2(tls_align, p->tls_align); +#ifdef TLS_ABOVE_TP + p->tls_offset = tls_offset + ( (tls_align-1) & + -(tls_offset + (uintptr_t)p->tls_image) ); + tls_offset += p->tls_size; +#else + tls_offset += p->tls_size + p->tls_align - 1; + tls_offset -= (tls_offset + (uintptr_t)p->tls_image) + & (p->tls_align-1); + p->tls_offset = tls_offset; +#endif p->new_dtv = (void *)(-sizeof(size_t) & (uintptr_t)(p->name+strlen(p->name)+sizeof(size_t))); p->new_tls = (void *)(p->new_dtv + n_th*(tls_cnt+1)); @@ -608,16 +632,6 @@ static void reloc_all(struct dso *p) } } -static void free_all(struct dso *p) -{ - struct dso *n; - while (p) { - n = p->next; - if (p->map && p!=libc && p!=head) free(p); - p = n; - } -} - static size_t find_dyn(Phdr *ph, size_t cnt, size_t stride) { for (; cnt--; ph = (void *)((char *)ph + stride)) @@ -642,18 +656,37 @@ static void find_map_range(Phdr *ph, size_t cnt, size_t stride, struct dso *p) p->map_len = max_addr - min_addr; } +static void do_fini() +{ + struct dso *p; + size_t dyn[DYN_CNT] = {0}; + for (p=fini_head; p; p=p->fini_next) { + if (!p->constructed) continue; + decode_vec(p->dynv, dyn, DYN_CNT); + ((void (*)(void))(p->base + dyn[DT_FINI]))(); + } +} + static void do_init_fini(struct dso *p) { size_t dyn[DYN_CNT] = {0}; + int need_locking = libc.threads_minus_1; + /* Allow recursive calls that arise when a library calls + * dlopen from one of its constructors, but block any + * other threads until all ctors have finished. */ + if (need_locking) pthread_mutex_lock(&init_fini_lock); for (; p; p=p->prev) { - if (p->constructed) return; + if (p->constructed) continue; + p->constructed = 1; decode_vec(p->dynv, dyn, DYN_CNT); - if (dyn[0] & (1<base + dyn[DT_FINI])); + if (dyn[0] & (1<fini_next = fini_head; + fini_head = p; + } if (dyn[0] & (1<base + dyn[DT_INIT]))(); - p->constructed = 1; } + if (need_locking) pthread_mutex_unlock(&init_fini_lock); } void _dl_debug_state(void) @@ -662,21 +695,38 @@ void _dl_debug_state(void) void *__copy_tls(unsigned char *mem) { + pthread_t td; struct dso *p; + + if (!tls_cnt) return mem; + void **dtv = (void *)mem; dtv[0] = (void *)tls_cnt; - mem = (void *)(dtv + tls_cnt + 1); - for (p=tail; p; p=p->prev) { + +#ifdef TLS_ABOVE_TP + mem += sizeof(void *) * (tls_cnt+1); + mem += -((uintptr_t)mem + sizeof(struct pthread)) & (tls_align-1); + td = (pthread_t)mem; + mem += sizeof(struct pthread); + + for (p=head; p; p=p->next) { if (!p->tls_id) continue; - mem += -p->tls_len & (4*sizeof(size_t)-1); - mem += ((uintptr_t)p->tls_image - (uintptr_t)mem) - & (p->tls_align-1); - dtv[p->tls_id] = mem; - memcpy(mem, p->tls_image, p->tls_len); - mem += p->tls_size; + dtv[p->tls_id] = mem + p->tls_offset; + memcpy(dtv[p->tls_id], p->tls_image, p->tls_len); } - ((pthread_t)mem)->dtv = dtv; - return mem; +#else + mem += libc.tls_size - sizeof(struct pthread); + mem -= (uintptr_t)mem & (tls_align-1); + td = (pthread_t)mem; + + for (p=head; p; p=p->next) { + if (!p->tls_id) continue; + dtv[p->tls_id] = mem - p->tls_offset; + memcpy(dtv[p->tls_id], p->tls_image, p->tls_len); + } +#endif + td->dtv = dtv; + return td; } void *__tls_get_addr(size_t *v) @@ -687,8 +737,7 @@ void *__tls_get_addr(size_t *v) /* Block signals to make accessing new TLS async-signal-safe */ sigset_t set; - sigfillset(&set); - pthread_sigmask(SIG_BLOCK, &set, &set); + pthread_sigmask(SIG_BLOCK, SIGALL_SET, &set); if (self->dtv && v[0]<=(size_t)self->dtv[0] && self->dtv[v[0]]) { pthread_sigmask(SIG_SETMASK, &set, 0); return (char *)self->dtv[v[0]]+v[1]; @@ -716,11 +765,21 @@ void *__tls_get_addr(size_t *v) (p->tls_size + p->tls_align) * a_fetch_add(&p->new_tls_idx,1); mem += ((uintptr_t)p->tls_image - (uintptr_t)mem) & (p->tls_align-1); self->dtv[v[0]] = mem; - memcpy(mem, p->tls_image, p->tls_size); + memcpy(mem, p->tls_image, p->tls_len); pthread_sigmask(SIG_SETMASK, &set, 0); return mem + v[1]; } +static void update_tls_size() +{ + libc.tls_size = ALIGN( + (1+tls_cnt) * sizeof(void *) + + tls_offset + + sizeof(struct pthread) + + tls_align * 2, + tls_align); +} + void *__dynlink(int argc, char **argv) { size_t aux[AUX_CNT] = {0}; @@ -828,9 +887,18 @@ void *__dynlink(int argc, char **argv) aux[AT_ENTRY] = ehdr->e_entry; } if (app->tls_size) { - app->tls_id = ++tls_cnt; - tls_size += app->tls_size+app->tls_align + 8*sizeof(size_t)-1 - & -4*sizeof(size_t); + app->tls_id = tls_cnt = 1; +#ifdef TLS_ABOVE_TP + app->tls_offset = 0; + tls_offset = app->tls_size + + ( -((uintptr_t)app->tls_image + app->tls_size) + & (app->tls_align-1) ); +#else + tls_offset = app->tls_offset = app->tls_size + + ( -((uintptr_t)app->tls_image + app->tls_size) + & (app->tls_align-1) ); +#endif + tls_align = MAXP2(tls_align, app->tls_align); } app->global = 1; app->constructed = 1; @@ -858,7 +926,7 @@ void *__dynlink(int argc, char **argv) * restore the initial chain in preparation for loading third * party libraries (preload/needed). */ head = tail = app; - libc = lib; + ldso = lib; app->next = lib; reloc_all(lib); app->next = 0; @@ -879,36 +947,22 @@ void *__dynlink(int argc, char **argv) load_deps(app); make_global(app); - /* Make an initial pass setting up TLS before performing relocs. - * This provides the TP-based offset of each DSO's TLS for - * use in TP-relative relocations. After relocations, we need - * to copy the TLS images again in case they had relocs. */ - tls_size += sizeof(struct pthread) + 4*sizeof(size_t); - __libc.tls_size = tls_size; + reloc_all(app->next); + reloc_all(app); + + update_tls_size(); if (tls_cnt) { struct dso *p; - void *mem = mmap(0, __libc.tls_size, PROT_READ|PROT_WRITE, + void *mem = mmap(0, libc.tls_size, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); if (mem==MAP_FAILED || !__install_initial_tls(__copy_tls(mem))) { dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n", - argv[0], tls_size); + argv[0], libc.tls_size); _exit(127); } - for (p=head; p; p=p->next) { - if (!p->tls_id) continue; - p->tls_offset = (char *)__pthread_self() - - (char *)__pthread_self()->dtv[p->tls_id]; - } } - reloc_all(app->next); - reloc_all(app); - - /* The initial DTV is located at the base of the memory - * allocated for TLS. Repeat copying TLS to pick up relocs. */ - if (tls_cnt) __copy_tls((void *)__pthread_self()->dtv); - if (ldso_fail) _exit(127); if (ldd_mode) _exit(0); @@ -930,8 +984,9 @@ void *__dynlink(int argc, char **argv) debug.state = 0; _dl_debug_state(); - if (ssp_used) __init_ssp(auxv); + if (ssp_used) __init_ssp((void *)aux[AT_RANDOM]); + atexit(do_fini); do_init_fini(tail); errno = 0; @@ -941,7 +996,7 @@ void *__dynlink(int argc, char **argv) void *dlopen(const char *file, int mode) { struct dso *volatile p, *orig_tail, *next; - size_t orig_tls_cnt; + size_t orig_tls_cnt, orig_tls_offset, orig_tls_align; size_t i; int cs; @@ -953,6 +1008,8 @@ void *dlopen(const char *file, int mode) p = 0; orig_tls_cnt = tls_cnt; + orig_tls_offset = tls_offset; + orig_tls_align = tls_align; orig_tail = tail; if (setjmp(rtld_fail)) { @@ -967,7 +1024,8 @@ void *dlopen(const char *file, int mode) free(p); } tls_cnt = orig_tls_cnt; - tls_size = __libc.tls_size; + tls_offset = orig_tls_offset; + tls_align = orig_tls_align; tail = orig_tail; tail->next = 0; p = 0; @@ -1002,16 +1060,16 @@ void *dlopen(const char *file, int mode) p->global = 1; } - __libc.tls_size = tls_size; + update_tls_size(); if (ssp_used) __init_ssp(auxv); _dl_debug_state(); - - do_init_fini(tail); + orig_tail = tail; end: __release_ptc(); pthread_rwlock_unlock(&lock); + if (p) do_init_fini(orig_tail); pthread_setcancelstate(cs, 0); return p; } @@ -1022,11 +1080,14 @@ static void *do_dlsym(struct dso *p, const char *s, void *ra) uint32_t h = 0, gh = 0; Sym *sym; if (p == head || p == RTLD_DEFAULT || p == RTLD_NEXT) { - if (p == RTLD_NEXT) { + if (p == RTLD_DEFAULT) { + p = head; + } else if (p == RTLD_NEXT) { for (p=head; p && (unsigned char *)ra-p->map>p->map_len; p=p->next); if (!p) p=head; + p = p->next; } - struct symdef def = find_sym(p->next, s, 0); + struct symdef def = find_sym(p, s, 0); if (!def.sym) goto failed; return def.dso->base + def.sym->st_value; }