CryptoAPI: How to verify a DSA signature from OpenSSL or Java using CryptVerifySignature

Question

I would like to be able to verify an OpenSSL-generated DSA signature using the Microsoft CryptoAPI.

Consider that you have the following inputs:

• an existing DSA public key:
• the data to be verified
• a binary signature

The signature has already been converted from Base64 into a series of 48 bytes.

Answer 1

Without good knowledge of the CryptoAPI, this is much more difficult that it should be.

The major stumbling blocks were:

• Decode the X509 DSA public key using CryptStringToBinaryA and CryptDecodeObjectEx
• Convert the DSA signature format
  • OpenSSL's DSA_sign produces a DSA signature in the ASN.1 DER format
  • CryptoAPI's CryptVerifySignature expects the DSA signature in the P1363 format

Here's a rough sample of how I finally solved the problem:

const char* pubKey =  "MIIBtjCCASsGByqGSM44BAEwggEeAoGBANW/k8nYREKtRMvIShnJTSAwxF33haU4"
                      .....
                      "/FEGAibbOp31rjq9UfaJ2t06eN0t0B+DP1hjz/MfpGtPOxHqF3dQnDRa3ot1FSTP";

bool verify(const unsigned char* msgData, unsigned int msgLength, const unsigned char* signature, unsigned int signatureLength)
{
    HCRYPTPROV hCryptProv;
    if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_DSS, CRYPT_VERIFYCONTEXT))
    {
        return false;
    }

    bool result = false;

    unsigned char derPubKey[2048];
    DWORD  derPubKeyLen = 2048;
    CERT_PUBLIC_KEY_INFO *publicKeyInfo = NULL;
    DWORD  publicKeyInfoLen = 0;

    if ( CryptStringToBinaryA( pubKey, strlen(pubKey), CRYPT_STRING_BASE64, derPubKey, &derPubKeyLen, NULL, NULL ) &&
         CryptDecodeObjectEx( X509_ASN_ENCODING, X509_PUBLIC_KEY_INFO, derPubKey, derPubKeyLen, 
                              CRYPT_ENCODE_ALLOC_FLAG, NULL, &publicKeyInfo, &publicKeyInfoLen ) )
    {
        HCRYPTKEY hPubKey;
        if (CryptImportPublicKeyInfo(hCryptProv, X509_ASN_ENCODING, publicKeyInfo, &hPubKey))
        {
            HCRYPTHASH hHash;
            if (CryptCreateHash(hCryptProv, CALG_SHA1, 0, 0, &hHash))
            {
                CryptHashData(hHash, msgData, msgLength, 0);

                BYTE* dsaSignature = NULL;
                DWORD dsaSignatureLen = 0;
                if (CryptDecodeObjectEx( X509_ASN_ENCODING, X509_DSS_SIGNATURE, signature, signatureLength,
                                         CRYPT_ENCODE_ALLOC_FLAG, NULL, &dsaSignature, &dsaSignatureLen ) )
                {
                    if (CryptVerifySignature(hHash, dsaSignature, dsaSignatureLen, hPubKey, NULL, 0))
                    {
                        result = true;
                    }
                    LocalFree(dsaSignature);
                }

                CryptDestroyHash(hHash);
            }
            CryptDestroyKey(hPubKey);
        }
        LocalFree(publicKeyInfo);
    }
    CryptReleaseContext(hCryptProv, 0);
    return result;
}

Answer 2

Just a note, Microsoft's crypto API doesn't support DSA key lengths greater than 1024. CryptImportPublicKeyInfo() will fail with NTE_BAD_DATA.