#define _GNU_SOURCE
#define SYSCALL_NO_TLS 1
-#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <stddef.h>
#include <semaphore.h>
#include <sys/membarrier.h>
#include "pthread_impl.h"
+#include "fork_impl.h"
#include "libc.h"
#include "dynlink.h"
-#include "malloc_impl.h"
+
+#define malloc __libc_malloc
+#define calloc __libc_calloc
+#define realloc __libc_realloc
+#define free __libc_free
static void error(const char *, ...);
struct dso **deps, *needed_by;
size_t ndeps_direct;
size_t next_dep;
- int ctor_visitor;
+ pthread_t ctor_visitor;
char *rpath_orig, *rpath;
struct tls_module tls;
size_t tls_id;
struct dso *dso;
};
+typedef void (*stage3_func)(size_t *, size_t *);
+
static struct builtin_tls {
char c;
struct pthread pt;
}
return (void *)(v - p->loadmap->segs[j].p_vaddr + p->loadmap->segs[j].addr);
}
-#define fpaddr(p, v) ((void (*)())&(struct funcdesc){ \
- laddr(p, v), (p)->got })
+static void (*fdbarrier(void *p))()
+{
+ void (*fd)();
+ __asm__("" : "=r"(fd) : "0"(p));
+ return fd;
+}
+#define fpaddr(p, v) fdbarrier((&(struct funcdesc){ \
+ laddr(p, v), (p)->got }))
#else
#define laddr(p, v) (void *)((p)->base + (v))
#define laddr_pg(p, v) laddr(p, v)
#define ARCH_SYM_REJECT_UND(s) 0
#endif
-static struct symdef find_sym(struct dso *dso, const char *s, int need_def)
+#if defined(__GNUC__)
+__attribute__((always_inline))
+#endif
+static inline struct symdef find_sym2(struct dso *dso, const char *s, int need_def, int use_deps)
{
uint32_t h = 0, gh = gnu_hash(s), gho = gh / (8*sizeof(size_t)), *ght;
size_t ghm = 1ul << gh % (8*sizeof(size_t));
struct symdef def = {0};
- for (; dso; dso=dso->syms_next) {
+ struct dso **deps = use_deps ? dso->deps : 0;
+ for (; dso; dso=use_deps ? *deps++ : dso->syms_next) {
Sym *sym;
if ((ght = dso->ghashtab)) {
sym = gnu_lookup_filtered(gh, ght, dso, s, gho, ghm);
return def;
}
+static struct symdef find_sym(struct dso *dso, const char *s, int need_def)
+{
+ return find_sym2(dso, s, need_def, 0);
+}
+
static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stride)
{
unsigned char *base = dso->base;
}
switch(type) {
- case REL_NONE:
- break;
case REL_OFFSET:
addend -= (size_t)reloc_addr;
case REL_SYMBOLIC:
}
}
+static ssize_t read_loop(int fd, void *p, size_t n)
+{
+ for (size_t i=0; i<n; ) {
+ ssize_t l = read(fd, (char *)p+i, n-i);
+ if (l<0) {
+ if (errno==EINTR) continue;
+ else return -1;
+ }
+ if (l==0) return i;
+ i += l;
+ }
+ return n;
+}
+
static void *mmap_fixed(void *p, size_t n, int prot, int flags, int fd, off_t off)
{
static int no_map_fixed;
char *q;
+ if (!n) return p;
if (!no_map_fixed) {
q = mmap(p, n, prot, flags|MAP_FIXED, fd, off);
if (!DL_NOMMU_SUPPORT || q != MAP_FAILED || errno != EINVAL)
snprintf(etc_ldso_path, sizeof etc_ldso_path,
"%.*s/etc/ld-musl-" LDSO_ARCH ".path",
(int)prefix_len, prefix);
- FILE *f = fopen(etc_ldso_path, "rbe");
- if (f) {
- if (getdelim(&sys_path, (size_t[1]){0}, 0, f) <= 0) {
+ fd = open(etc_ldso_path, O_RDONLY|O_CLOEXEC);
+ if (fd>=0) {
+ size_t n = 0;
+ if (!fstat(fd, &st)) n = st.st_size;
+ if ((sys_path = malloc(n+1)))
+ sys_path[n] = 0;
+ if (!sys_path || read_loop(fd, sys_path, n)<0) {
free(sys_path);
sys_path = "";
}
- fclose(f);
+ close(fd);
} else if (errno != ENOENT) {
sys_path = "";
}
{
struct dso *p;
size_t dyn[DYN_CNT];
- int self = __pthread_self()->tid;
+ pthread_t self = __pthread_self();
/* Take both locks before setting shutting_down, so that
* either lock is sufficient to read its value. The lock
}
}
+void __ldso_atfork(int who)
+{
+ if (who<0) {
+ pthread_rwlock_wrlock(&lock);
+ pthread_mutex_lock(&init_fini_lock);
+ } else {
+ pthread_mutex_unlock(&init_fini_lock);
+ pthread_rwlock_unlock(&lock);
+ }
+}
+
static struct dso **queue_ctors(struct dso *dso)
{
size_t cnt, qpos, spos, i;
}
queue[qpos] = 0;
for (i=0; i<qpos; i++) queue[i]->mark = 0;
+ for (i=0; i<qpos; i++)
+ if (queue[i]->ctor_visitor && queue[i]->ctor_visitor->tid < 0) {
+ error("State of %s is inconsistent due to multithreaded fork\n",
+ queue[i]->name);
+ free(queue);
+ if (runtime) longjmp(*rtld_fail, 1);
+ }
return queue;
}
{
struct dso *p;
size_t dyn[DYN_CNT], i;
- int self = __pthread_self()->tid;
+ pthread_t self = __pthread_self();
pthread_mutex_lock(&init_fini_lock);
for (i=0; (p=queue[i]); i++) {
/* Install new dtv for each thread. */
for (j=0, td=self; !j || td!=self; j++, td=td->next) {
- td->dtv = td->dtv_copy = newdtv[j];
+ td->dtv = newdtv[j];
}
__tl_unlock();
hidden void __dls2(unsigned char *base, size_t *sp)
{
+ size_t *auxv;
+ for (auxv=sp+1+*sp+1; *auxv; auxv++);
+ auxv++;
if (DL_FDPIC) {
void *p1 = (void *)sp[-2];
void *p2 = (void *)sp[-1];
if (!p1) {
- size_t *auxv, aux[AUX_CNT];
- for (auxv=sp+1+*sp+1; *auxv; auxv++);
- auxv++;
+ size_t aux[AUX_CNT];
decode_vec(auxv, aux, AUX_CNT);
if (aux[AT_BASE]) ldso.base = (void *)aux[AT_BASE];
else ldso.base = (void *)(aux[AT_PHDR] & -4096);
* symbolically as a barrier against moving the address
* load across the above relocation processing. */
struct symdef dls2b_def = find_sym(&ldso, "__dls2b", 0);
- if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls2b_def.sym-ldso.syms])(sp);
- else ((stage3_func)laddr(&ldso, dls2b_def.sym->st_value))(sp);
+ if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls2b_def.sym-ldso.syms])(sp, auxv);
+ else ((stage3_func)laddr(&ldso, dls2b_def.sym->st_value))(sp, auxv);
}
/* Stage 2b sets up a valid thread pointer, which requires relocations
* so that loads of the thread pointer and &errno can be pure/const and
* thereby hoistable. */
-void __dls2b(size_t *sp)
+void __dls2b(size_t *sp, size_t *auxv)
{
/* Setup early thread pointer in builtin_tls for ldso/libc itself to
* use during dynamic linking. If possible it will also serve as the
* thread pointer at runtime. */
+ search_vec(auxv, &__hwcap, AT_HWCAP);
+ libc.auxv = auxv;
libc.tls_size = sizeof builtin_tls;
libc.tls_align = tls_align;
if (__init_tp(__copy_tls((void *)builtin_tls)) < 0) {
}
struct symdef dls3_def = find_sym(&ldso, "__dls3", 0);
- if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls3_def.sym-ldso.syms])(sp);
- else ((stage3_func)laddr(&ldso, dls3_def.sym->st_value))(sp);
+ if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls3_def.sym-ldso.syms])(sp, auxv);
+ else ((stage3_func)laddr(&ldso, dls3_def.sym->st_value))(sp, auxv);
}
/* Stage 3 of the dynamic linker is called with the dynamic linker/libc
* process dependencies and relocations for the main application and
* transfer control to its entry point. */
-void __dls3(size_t *sp)
+void __dls3(size_t *sp, size_t *auxv)
{
static struct dso app, vdso;
- size_t aux[AUX_CNT], *auxv;
+ size_t aux[AUX_CNT];
size_t i;
char *env_preload=0;
char *replace_argv0=0;
/* Find aux vector just past environ[] and use it to initialize
* global data that may be needed before we can make syscalls. */
__environ = envp;
- for (i=argc+1; argv[i]; i++);
- libc.auxv = auxv = (void *)(argv+i+1);
decode_vec(auxv, aux, AUX_CNT);
- __hwcap = aux[AT_HWCAP];
search_vec(auxv, &__sysinfo, AT_SYSINFO);
__pthread_self()->sysinfo = __sysinfo;
libc.page_size = aux[AT_PAGESZ];
dprintf(2, "%s: cannot load %s: %s\n", ldname, argv[0], strerror(errno));
_exit(1);
}
- Ehdr *ehdr = (void *)map_library(fd, &app);
+ Ehdr *ehdr = map_library(fd, &app);
if (!ehdr) {
dprintf(2, "%s: %s: Not a valid dynamic program\n", ldname, argv[0]);
_exit(1);
/* Initial TLS must also be allocated before final relocations
* might result in calloc being a call to application code. */
update_tls_size();
+ void *initial_tls = builtin_tls;
if (libc.tls_size > sizeof builtin_tls || tls_align > MIN_TLS_ALIGN) {
- void *initial_tls = calloc(libc.tls_size, 1);
+ initial_tls = calloc(libc.tls_size, 1);
if (!initial_tls) {
dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
argv[0], libc.tls_size);
_exit(127);
}
+ }
+ static_tls_cnt = tls_cnt;
+
+ /* The main program must be relocated LAST since it may contain
+ * copy relocations which depend on libraries' relocations. */
+ reloc_all(app.next);
+ reloc_all(&app);
+
+ /* Actual copying to new TLS needs to happen after relocations,
+ * since the TLS images might have contained relocated addresses. */
+ if (initial_tls != builtin_tls) {
if (__init_tp(__copy_tls(initial_tls)) < 0) {
a_crash();
}
if (__copy_tls((void*)builtin_tls) != self) a_crash();
libc.tls_size = tmp_tls_size;
}
- static_tls_cnt = tls_cnt;
-
- /* The main program must be relocated LAST since it may contin
- * copy relocations which depend on libraries' relocations. */
- reloc_all(app.next);
- reloc_all(&app);
if (ldso_fail) _exit(127);
if (ldd_mode) _exit(0);
* possibility of incomplete replacement. */
if (find_sym(head, "malloc", 1).dso != &ldso)
__malloc_replaced = 1;
+ if (find_sym(head, "aligned_alloc", 1).dso != &ldso)
+ __aligned_alloc_replaced = 1;
/* Switch to runtime mode: any further failures in the dynamic
* linker are a reportable failure rather than a fatal startup
debug.bp = dl_debug_state;
debug.head = head;
debug.base = ldso.base;
- debug.state = 0;
+ debug.state = RT_CONSISTENT;
_dl_debug_state();
if (replace_argv0) argv[0] = replace_argv0;
pthread_rwlock_wrlock(&lock);
__inhibit_ptc();
+ debug.state = RT_ADD;
+ _dl_debug_state();
+
p = 0;
if (shutting_down) {
error("Cannot dlopen while program is exiting.");
load_deps(p);
extend_bfs_deps(p);
pthread_mutex_lock(&init_fini_lock);
- if (!p->constructed) ctor_queue = queue_ctors(p);
+ int constructed = p->constructed;
pthread_mutex_unlock(&init_fini_lock);
+ if (!constructed) ctor_queue = queue_ctors(p);
if (!p->relocated && (mode & RTLD_LAZY)) {
prepare_lazy(p);
for (i=0; p->deps[i]; i++)
update_tls_size();
if (tls_cnt != orig_tls_cnt)
install_new_tls();
- _dl_debug_state();
orig_tail = tail;
end:
+ debug.state = RT_CONSISTENT;
+ _dl_debug_state();
__release_ptc();
if (p) gencnt++;
pthread_rwlock_unlock(&lock);
static void *do_dlsym(struct dso *p, const char *s, void *ra)
{
- size_t i;
- uint32_t h = 0, gh = 0, *ght;
- Sym *sym;
- if (p == head || p == RTLD_DEFAULT || p == RTLD_NEXT) {
- if (p == RTLD_DEFAULT) {
- p = head;
- } else if (p == RTLD_NEXT) {
- p = addr2dso((size_t)ra);
- if (!p) p=head;
- p = p->next;
- }
- struct symdef def = find_sym(p, s, 0);
- if (!def.sym) goto failed;
- if ((def.sym->st_info&0xf) == STT_TLS)
- return __tls_get_addr((tls_mod_off_t []){def.dso->tls_id, def.sym->st_value-DTP_OFFSET});
- if (DL_FDPIC && (def.sym->st_info&0xf) == STT_FUNC)
- return def.dso->funcdescs + (def.sym - def.dso->syms);
- return laddr(def.dso, def.sym->st_value);
- }
- if (__dl_invalid_handle(p))
+ int use_deps = 0;
+ if (p == head || p == RTLD_DEFAULT) {
+ p = head;
+ } else if (p == RTLD_NEXT) {
+ p = addr2dso((size_t)ra);
+ if (!p) p=head;
+ p = p->next;
+ } else if (__dl_invalid_handle(p)) {
return 0;
- if ((ght = p->ghashtab)) {
- gh = gnu_hash(s);
- sym = gnu_lookup(gh, ght, p, s);
- } else {
- h = sysv_hash(s);
- sym = sysv_lookup(s, h, p);
- }
- if (sym && (sym->st_info&0xf) == STT_TLS)
- return __tls_get_addr((tls_mod_off_t []){p->tls_id, sym->st_value-DTP_OFFSET});
- if (DL_FDPIC && sym && sym->st_shndx && (sym->st_info&0xf) == STT_FUNC)
- return p->funcdescs + (sym - p->syms);
- if (sym && sym->st_value && (1<<(sym->st_info&0xf) & OK_TYPES))
- return laddr(p, sym->st_value);
- for (i=0; p->deps[i]; i++) {
- if ((ght = p->deps[i]->ghashtab)) {
- if (!gh) gh = gnu_hash(s);
- sym = gnu_lookup(gh, ght, p->deps[i], s);
- } else {
- if (!h) h = sysv_hash(s);
- sym = sysv_lookup(s, h, p->deps[i]);
- }
- if (sym && (sym->st_info&0xf) == STT_TLS)
- return __tls_get_addr((tls_mod_off_t []){p->deps[i]->tls_id, sym->st_value-DTP_OFFSET});
- if (DL_FDPIC && sym && sym->st_shndx && (sym->st_info&0xf) == STT_FUNC)
- return p->deps[i]->funcdescs + (sym - p->deps[i]->syms);
- if (sym && sym->st_value && (1<<(sym->st_info&0xf) & OK_TYPES))
- return laddr(p->deps[i], sym->st_value);
- }
-failed:
- error("Symbol not found: %s", s);
- return 0;
+ } else
+ use_deps = 1;
+ struct symdef def = find_sym2(p, s, 0, use_deps);
+ if (!def.sym) {
+ error("Symbol not found: %s", s);
+ return 0;
+ }
+ if ((def.sym->st_info&0xf) == STT_TLS)
+ return __tls_get_addr((tls_mod_off_t []){def.dso->tls_id, def.sym->st_value-DTP_OFFSET});
+ if (DL_FDPIC && (def.sym->st_info&0xf) == STT_FUNC)
+ return def.dso->funcdescs + (def.sym - def.dso->syms);
+ return laddr(def.dso, def.sym->st_value);
}
int dladdr(const void *addr_arg, Dl_info *info)
return res;
}
+hidden void *__dlsym_redir_time64(void *restrict p, const char *restrict s, void *restrict ra)
+{
+#if _REDIR_TIME64
+ const char *suffix, *suffix2 = "";
+ char redir[36];
+
+ /* Map the symbol name to a time64 version of itself according to the
+ * pattern used for naming the redirected time64 symbols. */
+ size_t l = strnlen(s, sizeof redir);
+ if (l<4 || l==sizeof redir) goto no_redir;
+ if (s[l-2]=='_' && s[l-1]=='r') {
+ l -= 2;
+ suffix2 = s+l;
+ }
+ if (l<4) goto no_redir;
+ if (!strcmp(s+l-4, "time")) suffix = "64";
+ else suffix = "_time64";
+
+ /* Use the presence of the remapped symbol name in libc to determine
+ * whether it's one that requires time64 redirection; replace if so. */
+ snprintf(redir, sizeof redir, "__%.*s%s%s", (int)l, s, suffix, suffix2);
+ if (find_sym(&ldso, redir, 1).sym) s = redir;
+no_redir:
+#endif
+ return __dlsym(p, s, ra);
+}
+
int dl_iterate_phdr(int(*callback)(struct dl_phdr_info *info, size_t size, void *data), void *data)
{
struct dso *current;
info.dlpi_adds = gencnt;
info.dlpi_subs = 0;
info.dlpi_tls_modid = current->tls_id;
- info.dlpi_tls_data = current->tls.image;
+ info.dlpi_tls_data = !current->tls_id ? 0 :
+ __tls_get_addr((tls_mod_off_t[]){current->tls_id,0});
ret = (callback)(&info, sizeof (info), data);