How to prevent color key flickering visibly when enabling LWA_COLORKEY

Question

I have a window that should sometimes have a transparent hole in it, and sometimes not. Ideally, we would use SetWindowRgn, but that disables visual styles, which not only looks ugly but doesn't draw correctly with per-monitor DPI-awareness, so I am trying to use a layered window with a color key.

When enabling the color key, I first call SetLayeredWindowAttributes(hWnd, colorkey, 0, LWA_COLORKEY), then invalidate the window so that it is redrawn. At this point the window should not contain the key color. Then the window receives WM_PAINT at some point later, and the key color is painted, but at this point the window should have LWA_COLORKEY set, so again, I expect the key color not to be visible.

When disabling the color key, I first repaint the window (synchronously) so that it does not contain the key color, and then disable WS_EX_LAYERED, so again, I never expect to see the key color.

However, a window with the following window procedure constantly flickers between green, transparent and the background color as the mouse moves a across it.

It seems that perhaps SetLayeredWindowAttributes does not take effect immediately (and not even before the next WM_PAINT). How can I make sure that this attribute has taken effect before repainting, or otherwise prevent the key color being visible?

LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    static auto const colorkey = RGB(0,255,0);
    static auto const hbrush = CreateSolidBrush(colorkey);
    static auto transparent = false;
    switch (message)
    {
    case WM_PAINT:
        {
            PAINTSTRUCT ps;
            HDC hdc = BeginPaint(hWnd, &ps);
            if (transparent) {
                RECT rect{30,30,500,500};
                FillRect(hdc, &rect, hbrush);
            }
            EndPaint(hWnd, &ps);
        }
        break;
    case WM_MOUSEMOVE:
        if (transparent) {
            transparent = false;
            RedrawWindow(hWnd, nullptr /* lprcUpdate */, nullptr /* hrgnUpdate */, RDW_INVALIDATE | RDW_ERASE | RDW_UPDATENOW | RDW_ALLCHILDREN);
            SetWindowLongPtr(hWnd, GWL_EXSTYLE, GetWindowLongPtr(hWnd, GWL_EXSTYLE) & ~WS_EX_LAYERED);
        } else {
            SetWindowLongPtr(hWnd, GWL_EXSTYLE, GetWindowLongPtr(hWnd, GWL_EXSTYLE) | WS_EX_LAYERED);
            SetLayeredWindowAttributes(hWnd, colorkey, 0, LWA_COLORKEY);
            transparent = true;
            InvalidateRect(hWnd, nullptr, TRUE);
        }
        break;
    case WM_DESTROY:
        PostQuitMessage(0);
        break;
    default:
        return DefWindowProc(hWnd, message, wParam, lParam);
    }
    return 0;
}

Answer 1

I don't think layered windows are designed to be turned on and off multiple times per second (Windows will allocate/destroy a 32 BPP image etc. each time you toggle).

SWP_FRAMECHANGED and an extra erase does make it much better for me at least:

case WM_MOUSEMOVE:
    if (transparent) {
        transparent = false;
        RedrawWindow(hWnd, nullptr /* lprcUpdate */, nullptr /* hrgnUpdate */, RDW_INTERNALPAINT|RDW_INVALIDATE|RDW_ERASE|RDW_ERASENOW|RDW_UPDATENOW|RDW_ALLCHILDREN);
        SetWindowLongPtr(hWnd, GWL_EXSTYLE, GetWindowLongPtr(hWnd, GWL_EXSTYLE) & ~WS_EX_LAYERED);
        SetWindowPos(hWnd, 0, 0, 0, 0, 0, SWP_NOMOVE|SWP_NOSIZE|SWP_NOZORDER|SWP_FRAMECHANGED|SWP_NOACTIVATE);
        InvalidateRect(hWnd, nullptr, true);
    } else {
        SetWindowLongPtr(hWnd, GWL_EXSTYLE, GetWindowLongPtr(hWnd, GWL_EXSTYLE) | WS_EX_LAYERED);
        SetLayeredWindowAttributes(hWnd, colorkey, 0, LWA_COLORKEY);
        SetWindowPos(hWnd, 0, 0, 0, 0, 0, SWP_NOMOVE|SWP_NOSIZE|SWP_NOZORDER|SWP_FRAMECHANGED|SWP_NOACTIVATE);
        transparent = true;
        InvalidateRect(hWnd, nullptr, true);
    }
    TCHAR b[99];wsprintf(b,TEXT("%d tick=%d"), transparent, GetTickCount()), SetWindowText(hWnd, b);
    break;

Answer 2

It seems those layered window changes take some time to be reflected in the rendering of the window. Add a sleep can make the green doesn't show.

case WM_MOUSEMOVE:
    if (transparent) {
        transparent = false;
        RedrawWindow(hWnd, nullptr /* lprcUpdate */, nullptr /* hrgnUpdate */, RDW_INVALIDATE | RDW_ERASE | RDW_UPDATENOW | RDW_ALLCHILDREN);
        SetWindowLongPtr(hWnd, GWL_EXSTYLE, GetWindowLongPtr(hWnd, GWL_EXSTYLE) & ~WS_EX_LAYERED);
    }
    else {
        SetWindowLongPtr(hWnd, GWL_EXSTYLE, GetWindowLongPtr(hWnd, GWL_EXSTYLE) | WS_EX_LAYERED);
        SetLayeredWindowAttributes(hWnd, colorkey, 0, LWA_COLORKEY);
        Sleep(1); // Add sleep
        transparent = true;
        InvalidateRect(hWnd, nullptr, TRUE);
    }
    break;