Hooking framework (Detours-like)

Question

I am looking for a C library/framework that allows me to replace functions in memory and redirect them to my own implementations, while still allowing my implementation to call the original implementation.

This seems to be a rather rare need on Linux-y systems, presumably because LD_PRELOAD covers most aspects of runtime-function-replacing-thingies.

Answer 1

The following approach seems to work on applications I have. I don't like proprietary blobs on my machines, so I don't know if it works with e.g. Steam. I'd be very interested to know, though; I don't see any reason why it shouldn't.

The following approach uses _dl_vsym() to look up correctly versioned dlsym() and dlvsym() from libdl.so prior to main() being executed. During application execution, the interposed dlsym() and dlvsym() call their original versions (not _dl_vsym()); I believe that should avoid any application-specific woes.

In case other dynamic libraries get initialized before this one, very careful initial versions of those functions are used. They use _dl_vsym() to obtain the references to the libdl dlsym() or dlvsym() function; any subsequent call will use the libdl dlsym() or dlvsym(). This limits the fragile time to the first call during library initialization -- but the priority 101 hopefully gets this library initialized first.

#define _GNU_SOURCE
#include <dlfcn.h>
#include <errno.h>
#include <string.h>
#include <GL/glx.h>
#include <EGL/egl.h>

#define UNUSED __attribute__((unused))

#define LIBDL_VERSION "GLIBC_2.2.5"
#define LIBDL_PATH    "libdl.so"

extern void *_dl_vsym(void *, const char *, const char *, void *);

static const struct {
    const char *const symbol;
    const char *const version;
    void *const       function;
} interposed[] = {
    { "dlsym",          LIBDL_VERSION,   dlsym  },
    { "dlvsym",         LIBDL_VERSION,   dlvsym },
    { "glXSwapBuffers", (const char *)0, glXSwapBuffers },
    { "eglSwapBuffers", (const char *)0, eglSwapBuffers },
    { (const char *)0,  (const char *)0, (void *)0 }
};

static void *       initial_dlsym(void *, const char *);
static void *       initial_dlvsym(void *, const char *, const char *);
static void         initial_glXSwapBuffers(Display *, GLXDrawable);
static EGLBoolean   initial_eglSwapBuffers(EGLDisplay, EGLSurface);

static void *     (*actual_dlsym)(void *, const char *)                = initial_dlsym;
static void *     (*actual_dlvsym)(void *, const char *, const char *) = initial_dlvsym;
static void       (*actual_glXSwapBuffers)(Display *, GLXDrawable)     = initial_glXSwapBuffers;
static EGLBoolean (*actual_eglSwapBuffers)(EGLDisplay, EGLSurface)     = initial_eglSwapBuffers;

static void initial_glXSwapBuffers(Display *display UNUSED, GLXDrawable drawable UNUSED)
{
    return;
}

static EGLBoolean initial_eglSwapBuffers(EGLDisplay display UNUSED, EGLSurface surface UNUSED)
{
    return 0;
}

static void *initial_dlsym(void *handle, const char *const symbol)
{
    void *(*call_dlsym)(void *, const char *);

    if (symbol) {
        size_t i;
        for (i = 0; interposed[i].symbol; i++)
            if (!strcmp(symbol, interposed[i].symbol))
                return interposed[i].function;
    }

    *(void **)(&call_dlsym) = __atomic_load_n((void **)(&actual_dlsym), __ATOMIC_SEQ_CST);
    if (!call_dlsym || call_dlsym == initial_dlsym) {
        const int saved_errno = errno;
        void     *handle;

        handle = dlopen(LIBDL_PATH, RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND | RTLD_NODELETE);
        call_dlsym = _dl_vsym(handle, "dlsym", LIBDL_VERSION, dlsym);
        dlclose(handle);

        if (!call_dlsym || call_dlsym == initial_dlsym || call_dlsym == dlsym) {
            errno = saved_errno;
            return (void *)0;
        }

        __atomic_store_n((void **)(&actual_dlsym), call_dlsym, __ATOMIC_SEQ_CST);
        errno = saved_errno;
    }

    return call_dlsym(handle, symbol);
}

static void *initial_dlvsym(void *handle, const char *const symbol, const char *const version)
{
    void *(*call_dlvsym)(void *, const char *, const char *);

    if (symbol) {
        size_t i;
        for (i = 0; interposed[i].symbol; i++)
            if (!strcmp(symbol, interposed[i].symbol))
                if (!interposed[i].version || !version || !strcmp(version, interposed[i].version))
                    return interposed[i].function;
    }

    *(void **)(&call_dlvsym) = __atomic_load_n((void **)(&actual_dlvsym), __ATOMIC_SEQ_CST);
    if (!call_dlvsym || call_dlvsym == initial_dlvsym) {
        const int saved_errno = errno;
        void     *handle;

        handle = dlopen(LIBDL_PATH, RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND | RTLD_NODELETE);
        call_dlvsym = _dl_vsym(handle, "dlvsym", LIBDL_VERSION, dlvsym);
        dlclose(handle);

        if (!call_dlvsym || call_dlvsym == initial_dlvsym || call_dlvsym == dlvsym) {
            errno = saved_errno;
            return (void *)0;
        }

        __atomic_store_n((void **)(&actual_dlvsym), call_dlvsym, __ATOMIC_SEQ_CST);
        errno = saved_errno;
    }

    return call_dlvsym(handle, symbol, version);
}

void *dlsym(void *handle, const char *const symbol)
{
    if (symbol) {
        size_t i;
        for (i = 0; interposed[i].symbol; i++)
            if (!strcmp(symbol, interposed[i].symbol))
                return interposed[i].function;
    }
    return actual_dlsym(handle, symbol);
}

void *dlvsym(void *handle, const char *const symbol, const char *version)
{
    if (symbol) {
        size_t i;
        for (i = 0; interposed[i].symbol; i++)
            if (!strcmp(symbol, interposed[i].symbol))
                if (!interposed[i].version || !version || !strcmp(version, interposed[i].version))
                    return interposed[i].function;
    }
    return actual_dlvsym(handle, symbol, version);
}

static void init(void) __attribute__((constructor (101)));
static void init(void)
{
    int    saved_errno;
    void  *handle;

    saved_errno = errno;

    handle = dlopen(LIBDL_PATH, RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND | RTLD_NODELETE);
    __atomic_store_n((void **)(&actual_dlsym),  _dl_vsym(handle, "dlsym",  LIBDL_VERSION, dlsym),  __ATOMIC_SEQ_CST);
    __atomic_store_n((void **)(&actual_dlvsym), _dl_vsym(handle, "dlvsym", LIBDL_VERSION, dlvsym), __ATOMIC_SEQ_CST);
    dlclose(handle);

    __atomic_store_n((void **)(&actual_glXSwapBuffers), actual_dlsym(RTLD_NEXT, "glXSwapBuffers"), __ATOMIC_SEQ_CST);
    __atomic_store_n((void **)(&actual_eglSwapBuffers), actual_dlsym(RTLD_NEXT, "eglSwapBuffers"), __ATOMIC_SEQ_CST);

    errno = saved_errno;
}

void glXSwapBuffers(Display *dpy, GLXDrawable drawable)
{
    /* TODO: Custom stuff before glXSwapBuffers() */
    actual_glXSwapBuffers(dpy, drawable);
    /* TODO: Custom stuff after glXSwapBuffers() */
}

EGLBoolean eglSwapBuffers(EGLDisplay dpy, EGLSurface surface)
{
    EGLBoolean result;
    /* TODO: Custom stuff before eglSwapBuffers() */
    result = actual_eglSwapBuffers(dpy, surface);
    /* TODO: Custom stuff after eglSwapBuffers() */
    return result;
}

If you save the above as example.c, you can compile it into libexample.so using

gcc -Wall -fPIC -shared `pkg-config --cflags gl egl` example.c -ldl -Wl,-soname,libexample.so `pkg-config --libs gl egl` -o libexample.so

In some cases, you need to modify the LIBDL_VERSION. Use

find /lib* /usr/ -name 'libdl.*' | while read FILE ; do echo "$FILE:" ; readelf -s "$FILE" | sed -ne '/ dlsym@/ s|^.*@@*|\t|p' ; done

to check which API version your libdl uses. (I've seen GLIBC_2.0 and GLIBC_2.2.5; it does not reflect the actual version of the library, but the API version of the dlsym() and dlvsym() calls.)

The interposed[] array contains the modified results for the interposed functions.

I have verified that the above example does not crash with any applications I tried -- including a simple dlsym() and dlvsym() stress test I wrote --, and that it also interposes the glXSwapBuffers() correctly (in glxgears and mpv).

Questions? Comments?