wintun:ERROR_INVALID_PARAMETER on registering ring buffers

Question

I am currently trying to get the wintun driver to work with my program for simple tunneling (see: https://www.wintun.net/ ).

I successfully find and open the network device, but when it comes to registering the buffer, I get the result ERROR_INVALID_PARAMETER (87). Like I said, opening works just fine and registering is done with SYSTEM privileges (if this is not done, I get ERROR_ACCESS_DENIED (5)).

First attempt was to malloc the ring buffers, but after that did not work I looked at how OpenVPN does it (yes, it added wintun support) and they seem to do with with CreateFileMapping.

First of all, here is my struct:

typedef struct _TUN_RING {
    volatile ULONG Head;
    volatile ULONG Tail;
    volatile LONG Alertable;
    UCHAR Data[(1024 * 1024) + 0x10000];
} TUN_RING;

which is according to the docs (https://git.zx2c4.com/wintun/about/ section "Ring Layout). Also its the same as OpenVPN does.

After that I create the file mapping

send_ring_handle_ = CreateFileMapping(INVALID_HANDLE_VALUE,
                                            nullptr,
                                            PAGE_READWRITE,
                                            0,
                                            sizeof(TUN_RING),
                                            nullptr);
recv_ring_handle_ = CreateFileMapping(INVALID_HANDLE_VALUE,
                                            nullptr,
                                            PAGE_READWRITE,
                                            0,
                                            sizeof(TUN_RING),
                                            nullptr);

Then I create the mappings:

send_ring_ = (TUN_RING *)MapViewOfFile(send_ring_handle_,
                                         FILE_MAP_ALL_ACCESS,
                                         0,
                                         0,
                                         sizeof(TUN_RING));
recv_ring_ = (TUN_RING *)MapViewOfFile(recv_ring_handle_,
                                         FILE_MAP_ALL_ACCESS,
                                         0,
                                         0,
                                         sizeof(TUN_RING));

and finally (after impersonating the system user) trying to register it with DeviceIoControl:

    TUN_REGISTER_RINGS reg_rings;
    memset(&reg_rings, 0, sizeof(TUN_REGISTER_RINGS));
    reg_rings.Send.RingSize = sizeof(TUN_RING);
    reg_rings.Send.Ring = send_ring_;
    reg_rings.Send.TailMoved = CreateEvent(0, TRUE, FALSE, 0);
    reg_rings.Receive.RingSize = sizeof(TUN_RING);
    reg_rings.Receive.Ring = recv_ring_;
    reg_rings.Receive.TailMoved = CreateEvent(0, TRUE, FALSE, 0);

    DWORD len;
    if (!DeviceIoControl(tun_fd_,
                            TUN_IOCTL_REGISTER_RINGS,
                            &reg_rings,
                            sizeof(reg_rings),
                            nullptr,
                            0,
                            &len,
                            nullptr))
    {
        printf("Could not register ring buffers (%d).", ::GetLastError());
        return false;
    }

Can anybody point me to where I am wrong? Like I said, with malloc instead of the file mapping the same error arieses.

I have written a complete example by now using malloc:

#include <windows.h>
#include <winioctl.h>
#include <IPHlpApi.h>
#include <ndisguid.h>
#include <TlHelp32.h>
#include <tchar.h>
#include <securitybaseapi.h>
#include <cfgmgr32.h>

#include <stdint.h>
#include <stdio.h>
#include <string>
#include <assert.h>

#pragma pack(push, 1)
typedef struct _TUN_PACKET_PROTO {
    ULONG Size;
    UCHAR Data[]; // max packet size as defined by the driver.
} TUN_PACKET_PROTO;

typedef struct _TUN_RING_PROTO {
    volatile ULONG Head;
    volatile ULONG Tail;
    volatile LONG Alertable;
    UCHAR Data[];
} TUN_RING_PROTO;


#define TUN_IOCTL_REGISTER_RINGS CTL_CODE(51820U, 0x970U, METHOD_BUFFERED, FILE_READ_DATA | FILE_WRITE_DATA)
#define TUN_IOCTL_FORCE_CLOSE_HANDLES CTL_CODE(51820U, 0x971U, METHOD_NEITHER, FILE_READ_DATA | FILE_WRITE_DATA)

#define WINTUN_RING_CAPACITY        0x800000
#define WINTUN_RING_TRAILING_BYTES  0x10000
#define WINTUN_MAX_PACKET_SIZE      0xffff
#define WINTUN_PACKET_ALIGN         4

/* Memory alignment of packets and rings */
#define TUN_ALIGNMENT sizeof(ULONG)
#define TUN_ALIGN(Size) (((ULONG)(Size) + ((ULONG)TUN_ALIGNMENT - 1)) & ~((ULONG)TUN_ALIGNMENT - 1))
#define TUN_IS_ALIGNED(Size) (!((ULONG)(Size) & ((ULONG)TUN_ALIGNMENT - 1)))
/* Maximum IP packet size */
#define TUN_MAX_IP_PACKET_SIZE 0xFFFF
/* Maximum packet size */
#define TUN_MAX_PACKET_SIZE TUN_ALIGN(sizeof(TUN_PACKET_PROTO) + TUN_MAX_IP_PACKET_SIZE)
/* Minimum ring capacity. */
#define TUN_MIN_RING_CAPACITY 0x20000 /* 128kiB */
/* Maximum ring capacity. */
#define TUN_MAX_RING_CAPACITY 0x4000000 /* 64MiB */
/* Calculates ring capacity */
#define TUN_RING_CAPACITY(Size) ((Size) - sizeof(TUN_RING_PROTO) - (TUN_MAX_PACKET_SIZE - TUN_ALIGNMENT))
/* Calculates ring offset modulo capacity */
#define TUN_RING_WRAP(Value, Capacity) ((Value) & (Capacity - 1))

#define IS_POW2(x) ((x) && !((x) & ((x)-1)))



typedef struct _TUN_RING {
    volatile ULONG Head;
    volatile ULONG Tail;
    volatile LONG Alertable;
    UCHAR Data[WINTUN_RING_CAPACITY + (TUN_MAX_PACKET_SIZE-TUN_ALIGNMENT)];
} TUN_RING;

typedef struct _TUN_PACKET {
    ULONG Size;
    UCHAR Data[WINTUN_MAX_PACKET_SIZE]; // max packet size as defined by the driver.
} TUN_PACKET;

typedef struct _TUN_REGISTER_RINGS {
    struct {
        ULONG RingSize;
        TUN_RING *Ring;
        HANDLE TailMoved;
    } Send, Receive;
} TUN_REGISTER_RINGS;
#pragma pack(pop)

class regkey_t
{
public:
    regkey_t(void);
    regkey_t(HKEY handle);
    ~regkey_t(void);

    void attach(HKEY handle);
    void release(void);

    HKEY detach(void);
    operator HKEY (void) const;
    HKEY &get(void);
    HKEY *operator &(void);

private:
    regkey_t(const regkey_t &);
    regkey_t &operator = (const regkey_t &);

    HKEY handle_;
};

regkey_t::regkey_t():
    handle_(0)
{
}

regkey_t::regkey_t(HKEY handle):
    handle_(handle)
{
}

regkey_t::~regkey_t(void)
{
    release();
}

void regkey_t::attach(HKEY handle)
{
    release();
    handle_ = handle;
}

void regkey_t::release(void)
{
    if (handle_)
    {
        const LONG res (::RegCloseKey(handle_));
        if (res != ERROR_SUCCESS)
        {
            printf("Couldn't close a reg handle (%lu).\n", res);
        }

        handle_ = 0;
    }
}

HKEY regkey_t::detach(void)
{
    const HKEY result (handle_);
    handle_ = 0;
    return result;
}

HKEY &regkey_t::get(void)
{
    return handle_;
}

HKEY *regkey_t::operator &(void)
{
    return &handle_;
}

regkey_t::operator HKEY(void) const
{
    return handle_;
}

bool impersonate_as_system()
{
    HANDLE thread_token, process_snapshot, winlogon_process, winlogon_token, duplicated_token;
    PROCESSENTRY32 entry;
    BOOL ret;
    DWORD pid = 0;
    TOKEN_PRIVILEGES privileges;

    ::memset(&entry, 0, sizeof(entry));
    ::memset(&privileges, 0, sizeof(privileges));

    entry.dwSize = sizeof(PROCESSENTRY32);

    privileges.PrivilegeCount = 1;
    privileges.Privileges->Attributes = SE_PRIVILEGE_ENABLED;

    if (!LookupPrivilegeValue(NULL, SE_DEBUG_NAME, &privileges.Privileges[0].Luid))
    {
        return false;
    }

    if (!ImpersonateSelf(SecurityImpersonation))
    {
        return false;
    }

    if (!OpenThreadToken(GetCurrentThread(), TOKEN_ADJUST_PRIVILEGES, FALSE, &thread_token))
    {
        RevertToSelf();
        return false;
    }
    if (!AdjustTokenPrivileges(thread_token, FALSE, &privileges, sizeof(privileges), NULL, NULL))
    {
        CloseHandle(thread_token);
        RevertToSelf();
        return false;
    }
    CloseHandle(thread_token);

    process_snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
    if (process_snapshot == INVALID_HANDLE_VALUE)
    {
        RevertToSelf();
        return false;
    }
    for (ret = Process32First(process_snapshot, &entry); ret; ret = Process32Next(process_snapshot, &entry))
    {
        if (::strcmp(entry.szExeFile, "winlogon.exe") == 0)
        {
            pid = entry.th32ProcessID;
            break;
        }
    }
    CloseHandle(process_snapshot);
    if (!pid)
    {
        RevertToSelf();
        return false;
    }

    winlogon_process = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid);
    if (!winlogon_process)
    {
        RevertToSelf();
        return false;
    }

    if (!OpenProcessToken(winlogon_process, TOKEN_IMPERSONATE | TOKEN_DUPLICATE, &winlogon_token))
    {
        CloseHandle(winlogon_process);
        RevertToSelf();
        return false;
    }
    CloseHandle(winlogon_process);

    if (!DuplicateToken(winlogon_token, SecurityImpersonation, &duplicated_token))
    {
        CloseHandle(winlogon_token);
        RevertToSelf();
        return false;
    }
    CloseHandle(winlogon_token);

    if (!SetThreadToken(NULL, duplicated_token))
    {
        CloseHandle(duplicated_token);
        RevertToSelf();
        return false;
    }
    CloseHandle(duplicated_token);

    return true;
}

std::string get_instance_id(uint32_t device_index)
{
    const std::string key_name("SYSTEM\\CurrentControlSet\\Control\\Class\\{4D36E972-E325-11CE-BFC1-08002BE10318}");

    std::string device_id("");

    regkey_t adapters;
    DWORD ret = ::RegOpenKeyEx(HKEY_LOCAL_MACHINE, key_name.c_str(), 0, KEY_READ, &adapters);
    if (ret != ERROR_SUCCESS)
    {
        printf("Could not open registry key %s (%d).\n", key_name.c_str(), ret);

        return device_id;
    }

    DWORD sub_keys(0);
    ret = ::RegQueryInfoKey(adapters, NULL, NULL, NULL, &sub_keys, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
    if (ret != ERROR_SUCCESS)
    {
        printf("Could not get info from %s (%d).\n", key_name.c_str(), ret);
        return device_id;
    }

    if (sub_keys <= 0)
    {
        printf("Wrong registry key %s.\n", key_name.c_str());
        return device_id;
    }

    if (device_index >= sub_keys)
    {
        return device_id;
    }

    uint32_t index(0);
    for (DWORD i = 0; i < sub_keys; i++)
    {
        const uint32_t max_key_length = 255;
        TCHAR key[max_key_length];
        DWORD keylen(max_key_length);

        // Get the adapter name
        ret = ::RegEnumKeyEx(adapters, i, key, &keylen, NULL, NULL, NULL, NULL);
        if (ret != ERROR_SUCCESS)
        {
            continue;
        }

        // Append it to NETWORK_ADAPTERS and open it
        regkey_t device;
        const std::string new_key(key_name + "\\" + std::string(key));

        ret = ::RegOpenKeyEx(HKEY_LOCAL_MACHINE, new_key.c_str(), 0, KEY_READ, &device);
        if (ret != ERROR_SUCCESS)
        {
            continue;
        }

        TCHAR data[256];
        DWORD len(sizeof(data));
        ret = ::RegQueryValueEx(device, "ComponentId", NULL, NULL, (LPBYTE)data, &len);
        if (ret != ERROR_SUCCESS)
        {
            continue;
        }

        std::string device_name("wintun");
        if (::_tcsnccmp(data, device_name.c_str(), sizeof(TCHAR) * device_name.length()) == 0)
        {
            if (device_index != index)
            {
                index++;
                continue;
            }

            DWORD type;
            len = sizeof(data);

            ret = ::RegQueryValueEx(device, "DeviceInstanceID", NULL, &type, (LPBYTE)data, &len);
            if (ret != ERROR_SUCCESS)
            {
                printf("Could not get info from %s (%d).\n", key_name.c_str(), ret);

            }

            device_id = data;
            break;
        }
    }

    return device_id;
}

bool open_tun_device()
{
    HANDLE tun_fd_ = INVALID_HANDLE_VALUE;
    std::string device_id;
    uint32_t device_index;
    {
        TCHAR *interface_list = nullptr;
        for (device_index = 0; device_index < 256; ++device_index)
        {
            device_id = get_instance_id(device_index);
            if (device_id.empty())
            {
                continue;
            }

            CONFIGRET status = CR_SUCCESS;

            // This loop is recommended as "robust code" by MSDN. See the Remarks of CM_Get_Device_Interface_list.
            do
            {
                DWORD required_chars(0);
                if ((status = ::CM_Get_Device_Interface_List_Size(&required_chars,
                                                        (LPGUID)&GUID_DEVINTERFACE_NET,
                                                        (char *)device_id.c_str(),
                                                        CM_GET_DEVICE_INTERFACE_LIST_PRESENT)) != CR_SUCCESS || !required_chars)
                {
                    break;
                }

                assert(required_chars > 0);
                interface_list = (TCHAR *)::malloc(sizeof(TCHAR) * required_chars);

                status = ::CM_Get_Device_Interface_List((LPGUID)&GUID_DEVINTERFACE_NET,
                                                                  (char *)device_id.c_str(),
                                                                  interface_list,
                                                                  required_chars,
                                                                  CM_GET_DEVICE_INTERFACE_LIST_PRESENT);

                if (status == CR_SUCCESS)
                {
                    break;
                }

                ::free(interface_list);
                interface_list = nullptr;
            } while(status == CR_BUFFER_SMALL);

            if (interface_list)
            {
                break;
            }
        }

        if (!interface_list)
        {
            printf("Could not find wintun interface.\n");
            return false;
        }
        else
        {
            tun_fd_ = ::CreateFile(interface_list,
                                   GENERIC_READ | GENERIC_WRITE,
                                   FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
                                   nullptr,
                                   OPEN_EXISTING, 0, nullptr);
        }


        ::free(interface_list);
    }

    if (!tun_fd_ || tun_fd_ == INVALID_HANDLE_VALUE)
    {
        printf("Could not open wintun device.\n");
        return false;
    }

    printf("Opened wintun device.\n");
    ::Sleep(1000);

    TUN_RING * send_ring_ = (TUN_RING *)::malloc(sizeof(TUN_RING));
    TUN_RING * recv_ring_ = (TUN_RING *)::malloc(sizeof(TUN_RING));

    if (!recv_ring_ || !send_ring_)
    {
        printf("Could not malloc.\n");
        return false;
    }

    ::memset(send_ring_, 0, sizeof(*send_ring_));
    ::memset(recv_ring_, 0, sizeof(*recv_ring_));

    recv_ring_->Alertable = TRUE;
    recv_ring_->Head = 0;
    recv_ring_->Tail = 0;
    send_ring_->Alertable = TRUE;
    send_ring_->Head = 0;
    send_ring_->Tail = 0;

    HANDLE send_event = ::CreateEvent(0, FALSE, FALSE, 0);
    HANDLE recv_event = ::CreateEvent(0, FALSE, FALSE, 0);
    // register the rings
    if (impersonate_as_system())
    {
        TUN_REGISTER_RINGS reg_rings;
        ::memset(&reg_rings, 0, sizeof(TUN_REGISTER_RINGS));
        reg_rings.Send.RingSize = sizeof(TUN_RING);
        reg_rings.Send.Ring = send_ring_;
        reg_rings.Send.TailMoved = send_event;
        reg_rings.Receive.RingSize = sizeof(TUN_RING);
        reg_rings.Receive.Ring = recv_ring_;
        reg_rings.Receive.TailMoved = recv_event;


        int send_capacity = TUN_RING_CAPACITY(reg_rings.Send.RingSize);

        if ((send_capacity < TUN_MIN_RING_CAPACITY || send_capacity > TUN_MAX_RING_CAPACITY ||
             !IS_POW2(send_capacity) || !reg_rings.Send.TailMoved || !reg_rings.Send.Ring))
        {
            printf("Fuck this shit I am out...\n");
        }


        DWORD len;

        DWORD fuckyou = 0;
        if (!::DeviceIoControl(tun_fd_, TUN_IOCTL_FORCE_CLOSE_HANDLES,
                               &fuckyou, sizeof(fuckyou), nullptr, 0, &len, nullptr))
        {
            printf("Error releasing handles (%d).\n", ::GetLastError());
        }


        if (!::DeviceIoControl(tun_fd_,
                                TUN_IOCTL_REGISTER_RINGS,
                                &reg_rings,
                                sizeof(reg_rings),
                                nullptr,
                                0,
                                &len,
                                nullptr))
        {
            printf("Could not register ring buffers (%d).\n", ::GetLastError());
            ::Sleep(10000);
            RevertToSelf();
            return false;
        }
        ::Sleep(10000);
        RevertToSelf();
    }
    else
    {
        printf("Could not elevate to SYSTEM\n");
        return false;
    }



    return true;
}

int main()
{
    if (!open_tun_device())
    {
        printf("Experiment failed.\n");
    }

    printf("Size TUNRING: %d (%d)\n", sizeof(TUN_RING),  0x800000 + 0x010000 + 0x0C);
    printf("Capacity: %d\n", TUN_RING_CAPACITY(sizeof(TUN_RING)));
    if (!IS_POW2(TUN_RING_CAPACITY(sizeof(TUN_RING))))
    {
        printf("Shit gone wrong...\n");
    }

    return 0;
}

Please make sure to RUN THIS ELEVATED or you will get error 5 ERROR_ACCESS_DENIED.

Answer 1

Okay, after a lot of trial and error I have translated the whole setup routine from the WireGuard Go code (see here: https://github.com/WireGuard/wireguard-go ) to C++, which seems to make it work. It accepts the rings now just as in the first post and the device is shown as connected afterwards...

They are doing some registry tweaks after installing the device (see https://github.com/WireGuard/wireguard-go/blob/4369db522b3fd7adc28a2a82b89315a6f3edbcc4/tun/wintun/wintun_windows.go#L207 ) which I think takes the cake. Thanks for everyone helping in finding this.

Answer 2

I can see a difference in your code when registering rings.

You are doing:

reg_rings.Send.RingSize = sizeof(TUN_RING);
reg_rings.Receive.RingSize = sizeof(TUN_RING);

While the docs says:

Send.RingSize, Receive.RingSize: Sizes of the rings (sizeof(TUN_RING) + capacity + 0x10000, as above)

Your ring is sizeof(TUN_RING) + UCHAR[(1024 * 1024) + 0x10000]

I guess it can't accept a ring that has no data space?

Sorry, I see your TUN_RING includes de data...

May be the events aren't good:

If an event is created from a service or a thread that is impersonating a different user, you can either apply a security descriptor to the event when you create it, or change the default security descriptor for the creating process by changing its default DACL

reg_rings.Send.TailMoved = CreateEvent(0, TRUE, FALSE, 0);
reg_rings.Receive.TailMoved = CreateEvent(0, TRUE, FALSE, 0);

You seem to be using the default DACL.

There may be aligning problems. If malloc isn't returning an aligned address for your buffer (as may be in debug mode, because there are memory management bytes) your Data member for the packet could be not aligned.

You can check the alignment against the address:

template <unsigned int alignment>
struct IsAligned
{
    static_assert((alignment & (alignment - 1)) == 0, "Alignment must be a power of 2");

    static inline bool Value(const void * ptr)
    {
        return (((uintptr_t)ptr) & (alignment - 1)) == 0;
    }
};

std::cout << IsAligned<32>::Value(ptr + i) << std::endl;

Giving the first packet address &(TUN_RING.Data[0]) (I guess.)

As said in your comment, it is the case, it is unaligned. You can try two things.

First reserve memory with aligned_alloc which will give you an aligned buffer for TUN_RING.

Second, if TUN_RING is already aligned and the packet alignment is the problem, then you should give the correct offset to the head and tail:

recv_ring_->Head = 0; // <- aligned byte offset
recv_ring_->Tail = 0;
send_ring_->Head = 0;
send_ring_->Tail = 0;

Remember:

Head: Byte offset of the first packet in the ring. Its value must be a multiple of 4 and less than ring capacity.

Tail: Byte offset of the start of free space in the ring. Its value must be multiple of 4 and less than ring capacity.

The byte offset must be a multiple of 4.

You have to increment those skipped bytes to the buffer size. For that you may need to allocate extra space that won't be used, but I think it won't be too much.

In a second view to events, in the docs it says event has to be auto-reset:

Send.TailMoved: A handle to an auto-reset event created by the client that Wintun signals after it moves the Tail member of the send ring.

Receive.TailMoved: A handle to an auto-reset event created by the client that the client will signal when it changes Receive.Ring->Tail and Receive.Ring->Alertable is non-zero.

In your example, the event is auto-reset:

HANDLE send_event = ::CreateEvent(0, FALSE, FALSE, 0);
HANDLE recv_event = ::CreateEvent(0, FALSE, FALSE, 0);

but in the code you show (at top of question) isn't:

reg_rings.Send.TailMoved = CreateEvent(0, TRUE, FALSE, 0);
reg_rings.Receive.TailMoved = CreateEvent(0, TRUE, FALSE, 0);

I don't know if the parameter checking goes so far as to verify event auto-reset setting (not even if that's possible.) Moreover, the openvpn code creates them non auto-reset (although may be there is some code around to signal them before registering.)

Answer 3

For me the fix to get rid off ERROR_INVALID_PARAMETER (87) was to switch from x86 to x64 architecture in Visual Studio

Answer 4

The issue here is the alignment of the structs. You align your structs to 1 byte [#pragma pack(push, 1)] while the wintun driver does 8(/ZP8 in the solution). This will result in differing struct sizes and thus the size checks will fall through. Furthermore I would like to recommend that you use VirtualAlloc or the Mapping instead of malloc.