Low-level keyboard hook doesn't work with DirectInput when the window is in focus

210 Views Asked by At

I am trying to intercept and prohibit the Alt+Tab combination (it is important for me not only to prohibit this combination) so that the application window does not switch by pressing these keys (for example, this is relevant in some cases with fullscreen mode in games) in the Windows operating system (Windows 10 22H2, apparently this is important as with the DInput version, more on this below). To do this, I use SetWindowsHookEx with WH_KEYBOARD_LL. However, this hook does not work if the application uses the API for processing DirectInput (DInput8) inputs. It is worth noting that the hook works if the application is minimized (even with DirectInput in the DISCL_BACKGROUND mode, which contradicts the behavior described above).

I saw on that setting a flag DISCL_FOREGROUND helped some people - I originally had this flag, tried various combinations of CooperativeLevel - it didn't help.

Also, the combination of hook and DirectInput probably worked for many before, but now it seems that this is not the case (there is a difference between the processing logic in DInput7 and DInput8, as well as between different Windows operating systems). Here I can only assume that DInput7 was used earlier, which internally used the same hooks (SetWindowsHookEx), and not Raw Input.

Judging by the documentation DirectInput processes Raw Input (i.e. WM_INPUT messages). I tried to intercept WM_INPUT using the SetWindowsHookEx hook with WH_GETMESSAGE - it didn't help (this hook in my case did not work at all if DirectInput was enabled).

I tried to change the order of initializations just in case (as for the WH_KEYBOARD_LL, so for the WH_GETMESSAGE) - it didn’t help.

Here I found a number of the same people who faced the same problem and I did not find a solution in these topics: example1 (Windows XP), example2 (the problem is similar with focus, but Raw Input is used here, not DirectInput), example3.

The questions are:

  1. Why does the hook WH_KEYBOARD_LL only work when the focus of the window is lost? Is it possible to solve this problem so that it works with DirectInput and with the window focus active?
  2. Based on point 1: why does the hook continue to work even when DirectInput is active when the window focus is lost (DISCL_BACKGROUND)?
  3. Under the hood of DirectInput (talking about DInput8, see the difference above) is Raw Input processing. If so, why does the SetWindowsHookEx hook with WH_GETMESSAGE to intercept WM_INPUT not work?
  4. What other solutions to this problem are there? At least I know there is a RegisterHotKey.

A minimal example to reproduce the problem:

#include <Windows.h>
#define DIRECTINPUT_VERSION 0x0800
#include <dinput.h>

#pragma comment (lib, "dinput8.lib")
#pragma comment (lib, "dxguid.lib")

HWND g_hWnd = {};
HHOOK g_LLKeyboardHook = {};

LRESULT CALLBACK WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
{
    switch (uMsg)
    {
        case WM_DESTROY:
        {
            PostQuitMessage(EXIT_SUCCESS);
            return ERROR_SUCCESS;
        }
        case WM_PAINT:
        {
            PAINTSTRUCT ps;
            HDC hdc = BeginPaint(hwnd, &ps);

            FillRect(hdc, &ps.rcPaint, (HBRUSH)(COLOR_WINDOW + 1));

            EndPaint(hwnd, &ps);
            return ERROR_SUCCESS;
        }
    }

    return DefWindowProc(hwnd, uMsg, wParam, lParam);
}

LRESULT CALLBACK LowLevelKeyboardProc(int nCode, WPARAM wParam, LPARAM lParam)
{
    const auto hookStruct = reinterpret_cast<KBDLLHOOKSTRUCT*>(lParam);

    switch (nCode)
    {
        case HC_ACTION:
        {
            // For tests ('A' key)
            if (hookStruct->vkCode == 0x41)
            {
                // MessageBox(g_hWnd, TEXT("Pressed 'A' from Hook"), TEXT("Hook"), 0u);
                // Block it
                return TRUE;
            }
            // Alt + Tab
            else if (hookStruct->flags & LLKHF_ALTDOWN && hookStruct->vkCode == VK_TAB)
            {
                // MessageBox(g_hWnd, TEXT("Pressed 'Alt + Tab' from Hook"), TEXT("Hook"), 0u);
                // Block it
                return TRUE;
            }
        }
    }

    return CallNextHookEx(g_LLKeyboardHook, nCode, wParam, lParam);
}

bool InitializeDirectInput(LPDIRECTINPUT8& di, LPDIRECTINPUTDEVICE8& diKeyboard, HINSTANCE hInstance)
{
    if (!hInstance || !g_hWnd)
    {
        return false;
    }

    auto res = DirectInput8Create(
        hInstance,
        DIRECTINPUT_VERSION,
        IID_IDirectInput8,
        reinterpret_cast<void**>(&di),
        NULL);
    if (FAILED(res))
    {
        MessageBox(g_hWnd, TEXT("DirectInput8Create failed"), TEXT("Error"), 0u);
        return false;
    }

    res = di->CreateDevice(
        GUID_SysKeyboard,
        &diKeyboard,
        NULL);
    if (FAILED(res))
    {
        MessageBox(g_hWnd, TEXT("CreateDevice failed"), TEXT("Error"), 0u);
        return false;
    }

    res = diKeyboard->SetDataFormat(&c_dfDIKeyboard);
    if (FAILED(res))
    {
        MessageBox(g_hWnd, TEXT("SetDataFormat failed"), TEXT("Error"), 0u);
        return false;
    }

    res = diKeyboard->SetCooperativeLevel(g_hWnd, DISCL_BACKGROUND | DISCL_NONEXCLUSIVE);
    if (FAILED(res))
    {
        MessageBox(g_hWnd, TEXT("SetCooperativeLevel failed"), TEXT("Error"), 0u);
        return false;
    }

    return true;
}

void CloseDirectInput(LPDIRECTINPUT8& di, LPDIRECTINPUTDEVICE8& diKeyboard)
{
    if (diKeyboard)
    {
        diKeyboard->Unacquire();
    }
    if (di)
    {
        di->Release();
    }
}

void DetectKeys(LPDIRECTINPUTDEVICE8& diKeyboard)
{
    BYTE keyState[256] = {};
    HRESULT res = {};

    res = diKeyboard->Acquire();
    if (FAILED(res))
    {
        // When the application loses focus, capture cannot be performed
        return;
    }

    res = diKeyboard->GetDeviceState(sizeof(keyState), static_cast<LPVOID>(keyState));
    if (FAILED(res))
    {
        return;
    }

    if (keyState[DIK_A] & 0x80)
    {
        MessageBox(g_hWnd, TEXT("Pressed 'A' from DirectInput"), TEXT("DirectInput"), 0u);
    }
}

int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR pCmdLine, int nCmdShow)
{
    const wchar_t windowClassName[] = L"Sample Window Class";

    WNDCLASS wc = {};
    wc.lpfnWndProc = WindowProc;
    wc.hInstance = hInstance;
    wc.lpszClassName = windowClassName;

    RegisterClass(&wc);

    g_hWnd = CreateWindowEx(
        0u,                                             // Optional window styles.
        windowClassName,                                // Window class
        L"Low-level keyboard hook with DirectInput",    // Window text
        WS_OVERLAPPEDWINDOW,                            // Window style

        // Size and position
        CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT,

        NULL,       // Parent window    
        NULL,       // Menu
        hInstance,  // Instance handle
        NULL        // Additional application data
    );
    if (!g_hWnd)
    {
        MessageBox(NULL, TEXT("CreateWindowEx failed"), TEXT("Error"), 0u);
        return EXIT_FAILURE;
    }

    LPDIRECTINPUT8 di = nullptr;
    LPDIRECTINPUTDEVICE8 diKeyboard = nullptr;

    if (!InitializeDirectInput(di, diKeyboard, hInstance))
    {
        return EXIT_FAILURE;
    }

    g_LLKeyboardHook = SetWindowsHookEx(WH_KEYBOARD_LL, LowLevelKeyboardProc, NULL, 0u);
    if (!g_LLKeyboardHook)
    {
        MessageBox(g_hWnd, TEXT("SetWindowsHookEx failed"), TEXT("Error"), 0u);
        CloseDirectInput(di, diKeyboard);
        return EXIT_FAILURE;
    }

    ShowWindow(g_hWnd, nCmdShow);

    MSG msg = {};
    while (true)
    {
        if (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE | PM_NOYIELD))
        {
            if (msg.message == WM_QUIT)
                break;

            TranslateMessage(&msg);
            DispatchMessage(&msg);
        }

        DetectKeys(diKeyboard);
    }

    UnhookWindowsHookEx(g_LLKeyboardHook);
    
    CloseDirectInput(di, diKeyboard);

    return EXIT_SUCCESS;
}
0

There are 0 best solutions below