RSASSA-PSS signature verification fails dues to padding with mbedtls but succeeds in Python

Question

I am trying to verify signatures like this using mbedtls in C-Ansi but it fails even though that the same signature is verified in Python. The code is being ran in the i.MX RT1170 CM7 using the crypto acceleration module (CAAM). I am getting MBEDTLS_ERR_RSA_INVALID_PADDING after calling mbedtls_rsa_rsassa_pss_verify.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <mbedtls/pk.h>
#include <mbedtls/rsa.h>
#include <mbedtls/error.h>

void hex_string_to_byte_array(const char *hex_string, unsigned char *byte_array) {
    size_t length = strlen(hex_string);
    size_t byte_count = 0;

    for (size_t i = 0; i < length; i += 2) {
        // Convert first hexadecimal character to a nibble
        unsigned char high_nibble = hex_string[i];
        if (high_nibble >= '0' && high_nibble <= '9') {
            high_nibble -= '0';
        } else if (high_nibble >= 'a' && high_nibble <= 'f') {
            high_nibble -= 'a' - 10;
        } else if (high_nibble >= 'A' && high_nibble <= 'F') {
            high_nibble -= 'A' - 10;
        } else {
            // Invalid hexadecimal character
            return;
        }

        // Convert second hexadecimal character to a nibble
        unsigned char low_nibble = hex_string[i + 1];
        if (low_nibble >= '0' && low_nibble <= '9') {
            low_nibble -= '0';
        } else if (low_nibble >= 'a' && low_nibble <= 'f') {
            low_nibble -= 'a' - 10;
        } else if (low_nibble >= 'A' && low_nibble <= 'F') {
            low_nibble -= 'A' - 10;
        } else {
            // Invalid hexadecimal character
            return;
        }

        // Combine the two nibbles into one byte
        byte_array[byte_count++] = (high_nibble << 4) | low_nibble;
    }
}

int main() {
    mbedtls_pk_context pk;
    mbedtls_pk_init(&pk);
    const char *pub_key = "-----BEGIN PUBLIC KEY-----\n\
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtCYUST0WEzptnISpi2oQ\
IGHvSASkSyTzADKsIuAwJ3AY5VXIuAU0A7D4pZbSSFjvcLAJ2+NYYu+ZYwGyicSz\
9p5iv03CitDJTUOj2OUd7GJjCOIgpfx40D50yKQbNq179QauTVGbQM4wT2zq+el0\
6gbunOQUaCC5PecRFIslo/9MivNT7ms+7zTNaj9iaMD/eEyww+aWYfwfGPF6guKP\
NagrhhakRvusfkHbAgWRbd/B3hOVnPmeXnK6pyWT+9zoq4ZFiEct7e7ul57OXZsE\
BEB8y3w9LHSrpMxko1yVPdSi3JKyyBjL+QA5gY+PQMLfmSmsisI72KTyra90wBHH\
mQIDAQAB\n\
-----END PUBLIC KEY-----\
";
    // Load the public key
    if (mbedtls_pk_parse_public_key(&pk, (const unsigned char *)pub_key, strlen(pub_key) + 1) != 0) {
        printf("Failed to parse public key.\n");
        return -1;
    }
    // Input data
    const unsigned char signatureString[] = "89073B346EC6995DBC1ED4D8B3DBE4979229517D9EFB80EC13BBDCCFC3C90B942EDF1396199C4D9F5F0C2DBF0C8F878FC89205A319AF959C467E781241EB0DB37BF21D65DC99F764592086A55F8EADC7BA9EAF03B49A473C659C29AE64FD778B4B844A919C70C0FF3260B280EF91EEF059E7B56881879B915C5DA0642D15D658247D3C8A851CE17FD78D48EA2197998DF1BDA6DDF5A564AC76B5CC4C3FD4B07BCB292B5C415C48B8726A90918130520786E3138386DDD9A5C25D535AA81707298010D9FB36BEF4BE057A6931E79DD491CD1CE18178E55B8334563E0B45E75DC16468DA2E105FBB3BCA79B467BF9BA336FD8931DBD6474D7FC652903E048175C8";
    unsigned char signature[256];
    const unsigned char payloadToHash[] = "3DB8F3960000000020000000060000000000000001000000000000000000000044616E69656C";
    const unsigned char hashString[]= "6440167c0bbec2ba23b794866894442c32264e8cc5adf90502e3571423d9d8b0";
    unsigned char hash[32];

    hex_string_to_byte_array(hashString, hash);
    hex_string_to_byte_array(signatureString, signature);

    // Verify the signature
    mbedtls_rsa_context *rsa_ctx = mbedtls_pk_rsa(pk);
    if (!rsa_ctx) {
        printf("Failed to extract RSA context.\r\n");
        mbedtls_pk_free(&pk);
        return -1;
    }
    
    if (mbedtls_rsa_rsassa_pss_verify(rsa_ctx, NULL, NULL, MBEDTLS_RSA_PUBLIC, MBEDTLS_MD_SHA256,
                                      32, hash, signature) != 0) {
        printf("The signature is not authentic.\r\n");        
    }
    else
    {
        printf("The signature is authentic.\r\n");
    }
    mbedtls_pk_free(&pk);
}

This is the Python version:

!pip install pycryptodome
from Crypto.PublicKey import RSA

from base64 import b64decode
from Crypto.Signature import pss
from Crypto.Hash import SHA256
from Crypto.PublicKey import RSA
import binascii
from cryptography.hazmat.primitives.asymmetric import padding
from cryptography.hazmat.primitives import hashes, serialization

pub_key = b'-----BEGIN PUBLIC KEY-----\n\
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtCYUST0WEzptnISpi2oQ\
IGHvSASkSyTzADKsIuAwJ3AY5VXIuAU0A7D4pZbSSFjvcLAJ2+NYYu+ZYwGyicSz\
9p5iv03CitDJTUOj2OUd7GJjCOIgpfx40D50yKQbNq179QauTVGbQM4wT2zq+el0\
6gbunOQUaCC5PecRFIslo/9MivNT7ms+7zTNaj9iaMD/eEyww+aWYfwfGPF6guKP\
NagrhhakRvusfkHbAgWRbd/B3hOVnPmeXnK6pyWT+9zoq4ZFiEct7e7ul57OXZsE\
BEB8y3w9LHSrpMxko1yVPdSi3JKyyBjL+QA5gY+PQMLfmSmsisI72KTyra90wBHH\
mQIDAQAB\n\
-----END PUBLIC KEY-----'
rsa_ctx = RSA.import_key(pub_key)

signatureString = '89073B346EC6995DBC1ED4D8B3DBE4979229517D9EFB80EC13BBDCCFC3C90B942EDF1396199C4D9F5F0C2DBF0C8F878FC89205A319AF959C467E781241EB0DB37BF21D65DC99F764592086A55F8EADC7BA9EAF03B49A473C659C29AE64FD778B4B844A919C70C0FF3260B280EF91EEF059E7B56881879B915C5DA0642D15D658247D3C8A851CE17FD78D48EA2197998DF1BDA6DDF5A564AC76B5CC4C3FD4B07BCB292B5C415C48B8726A90918130520786E3138386DDD9A5C25D535AA81707298010D9FB36BEF4BE057A6931E79DD491CD1CE18178E55B8334563E0B45E75DC16468DA2E105FBB3BCA79B467BF9BA336FD8931DBD6474D7FC652903E048175C8'
signature = bytes.fromhex(signatureString) 

payloadToHash = '3DB8F3960000000020000000060000000000000001000000000000000000000044616E69656C'
hash = bytes.fromhex(payloadToHash)  
hash = SHA256.new(hash)

verifier = pss.new(rsa_ctx)
try:
    verifier.verify(hash, signature)
    print("The signature is authentic.")
except (ValueError):
    print("The signature is not authentic.")

So far I know:

• Hash matches
• RSA key numbers are the same
• rsa_ctx.n == rsa_ctx->N.p (little endian aligned in the microcontroller)
• rsa_ctx.e == rsa_ctx->E (0x10001)
• even if I apply RSAVP1 to produce em = sig^E mod N using the public key, em would be the same

I think the problem might be in the alignment but mbedtls has __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ defined and I hope is taking care of it.

Any suggestions would be appreciated very much.