optimize hot paths of getc with manual shrink-wrapping

with these changes, in a program that has not created any threads
besides the main thread and that has not called f[try]lockfile, getc
performs indistinguishably from getc_unlocked. this was measured on
several i386 and x86_64 models, and should hold on other archs too
simply by the properties of the code generation.

the case where the caller already holds the lock (via flockfile) is
improved significantly as well (40-60% reduction in time on machines
tested) and the case where locking is needed is improved somewhat
(roughly 10%).

the key technique used here is forcing the non-hot path out-of-line
and enabling it to be a tail call. a static noinline function
(conditional on __GNUC__) is used rather than the extern hiddens used
elsewhere for this purpose, so that the compiler can choose
non-default calling conventions, making it possible to tail-call to a
callee that takes more arguments than the caller on archs where
arguments are passed on the stack or must have space reserved on the
stack for spilling the. the tid could just be reloaded via the thread
pointer in locking_getc, but that would be ridiculously expensive on
some archs where thread pointer load requires a trap or syscall.

diff --git a/src/stdio/fgetc.c b/src/stdio/fgetc.c
index e122416..2578afc 100644 (file)
--- a/src/stdio/fgetc.c
+++ b/src/stdio/fgetc.c
@@ -1,11 +1,7 @@
-#include "stdio_impl.h"
+#include <stdio.h>
+#include "getc.h"
 
 int fgetc(FILE *f)
 {
-       int c;
-       if (f->lock < 0 || !__lockfile(f))
-               return getc_unlocked(f);
-       c = getc_unlocked(f);
-       __unlockfile(f);
-       return c;
+       return do_getc(f);
 }

diff --git a/src/stdio/getc.c b/src/stdio/getc.c
index b3f351d..8409fc2 100644 (file)
--- a/src/stdio/getc.c
+++ b/src/stdio/getc.c
@@ -1,13 +1,9 @@
-#include "stdio_impl.h"
+#include <stdio.h>
+#include "getc.h"
 
 int getc(FILE *f)
 {
-       int c;
-       if (f->lock < 0 || !__lockfile(f))
-               return getc_unlocked(f);
-       c = getc_unlocked(f);
-       __unlockfile(f);
-       return c;
+       return do_getc(f);
 }
 
 weak_alias(getc, _IO_getc);

diff --git a/src/stdio/getc.h b/src/stdio/getc.h
new file mode 100644 (file)
index 0000000..0657ab6
--- /dev/null
+++ b/src/stdio/getc.h
@@ -0,0 +1,22 @@
+#include "stdio_impl.h"
+#include "pthread_impl.h"
+
+#ifdef __GNUC__
+__attribute__((__noinline__))
+#endif
+static int locking_getc(FILE *f, int tid)
+{
+       if (a_cas(&f->lock, 0, tid)) __lockfile(f);
+       int c = getc_unlocked(f);
+       if (a_swap(&f->lock, 0) & MAYBE_WAITERS)
+               __wake(&f->lock, 1, 1);
+       return c;
+}
+
+static inline int do_getc(FILE *f)
+{
+       int tid, l = f->lock;
+       if (l < 0 || (l & ~MAYBE_WAITERS) == (tid=__pthread_self()->tid))
+               return getc_unlocked(f);
+       return locking_getc(f, tid);
+}

diff --git a/src/stdio/getchar.c b/src/stdio/getchar.c
index c101265..df395ca 100644 (file)
--- a/src/stdio/getchar.c
+++ b/src/stdio/getchar.c
@@ -1,6 +1,7 @@
 #include <stdio.h>
+#include "getc.h"
 
 int getchar(void)
 {
-       return fgetc(stdin);
+       return do_getc(stdin);
 }