oaep decryption with HSM private key

Question

I have a key pair storage in HSM. SP for HSM is not support "RSA/ECB/OAEPPadding" decryption. I can decrypt without padding with existing private key.

Cipher cipher = Cipher.getInstance("RSA/ECB/NoPadding", "SunPKCS11-cknfast0");
cipher.init(Cipher.DECRYPT_MODE, privatePK11Key); 
decrypted = cipher.doFinal(data);

now how I can transform decrypted value to get original text?

Answer 1

OAEP decryption is defined in the PKCS#1 RSA standard. The decoding of the padding starts at step 3.

You could take a look at existing software implementations of RSA OAEP, e.g. for the open source Bouncy Castle library if you don't want to fully program it yourself.

Answer 2

So, after some research I got the OAEP decryption. I use Bouncy Castle 1.7 and adjust code from org.bouncycastle.crypto.encodings.OAEPEncoding class.

Cipher cipher = Cipher.getInstance("RSA/ECB/NoPadding", "SunPKCS11-cknfast0");
cipher.init(Cipher.DECRYPT_MODE, privatePK11Key); 
decryptedWithOaep = cipher.doFinal(data);
// this is for with OAEP SHA-256
OAEPEncoding eng =  new OAEPEncoding(new SHA256Digest(), new SHA256Digest());
// if you need with OAEP SHA-1
//new OAEPEncoding();
byte[] decrypted = eng.decodeBlock(decryptedWithOaep,0,decryptedWithOaep.length, getOutputBlockSize(publicPK11Key.getModulus().bitLength());

public static int getOutputBlockSize(bitSize)
{
  return (bitSize + 7) / 8 - 1;
}

import org.bouncycastle.crypto.Digest;
import org.bouncycastle.crypto.InvalidCipherTextException;
import org.bouncycastle.crypto.util.DigestFactory;
import org.bouncycastle.util.Arrays;
import org.bouncycastle.util.Pack;

public class OAEPEncoding
{
// this part is adjusting to my case

    private byte[]                  defHash ;
    private Digest                  mgf1Hash ;

    public OAEPEncoding()
    {
        Digest hash = DigestFactory.createSHA1();
        mgf1Hash = DigestFactory.createSHA1();
        defHash = new byte[mgf1Hash.getDigestSize()];
        hash.reset();
        hash.doFinal(defHash, 0);
    }

    public OAEPEncoding(Digest sha256Digest, Digest sha256Digest1) {
        mgf1Hash = sha256Digest1;
        defHash = new byte[mgf1Hash.getDigestSize()];
        sha256Digest.reset();
        sha256Digest.doFinal(defHash, 0);
    }

    public byte[] decodeBlock(
        byte[]  in,
        int     inOff,
        int     inLen,
        int outputBlockSize)
        throws InvalidCipherTextException
    {
        byte[]  data = in;
        byte[]  block = new byte[outputBlockSize];

// from here starting original org.bouncycastle.crypto.encodings.OAEPEncoding code
        //
        // as we may have zeros in our leading bytes for the block we produced
        // on encryption, we need to make sure our decrypted block comes back
        // the same size.
        //

        // i.e. wrong when block.length < (2 * defHash.length) + 1
        int wrongMask = (block.length - ((2 * defHash.length) + 1)) >> 31;

        if (data.length <= block.length)
        {
            System.arraycopy(data, 0, block, block.length - data.length, data.length);
        }
        else
        {
            System.arraycopy(data, 0, block, 0, block.length);
            wrongMask |= 1;
        }

        //
        // unmask the seed.
        //
        byte[] mask = maskGeneratorFunction1(
                    block, defHash.length, block.length - defHash.length, defHash.length);

        for (int i = 0; i != defHash.length; i++)
        {
            block[i] ^= mask[i];
        }

        //
        // unmask the message block.
        //
        mask = maskGeneratorFunction1(block, 0, defHash.length, block.length - defHash.length);

        for (int i = defHash.length; i != block.length; i++)
        {
            block[i] ^= mask[i - defHash.length];
        }

        //
        // check the hash of the encoding params.
        // long check to try to avoid this been a source of a timing attack.
        //
        for (int i = 0; i != defHash.length; i++)
        {
            wrongMask |= defHash[i] ^ block[defHash.length + i];
        }

        //
        // find the data block
        //
        int start = -1;

        for (int index = 2 * defHash.length; index != block.length; index++)
        {
            int octet = block[index] & 0xFF;

            // i.e. mask will be 0xFFFFFFFF if octet is non-zero and start is (still) negative, else 0.
            int shouldSetMask = (-octet & start) >> 31;

            start += index & shouldSetMask;
        }

        wrongMask |= start >> 31;
        ++start;
        wrongMask |= block[start] ^ 1;

        if (wrongMask != 0)
        {
            Arrays.fill(block, (byte)0);
            throw new InvalidCipherTextException("data wrong");
        }

        ++start;

        //
        // extract the data block
        //
        byte[]  output = new byte[block.length - start];

        System.arraycopy(block, start, output, 0, output.length);
        Arrays.fill(block, (byte)0);

        return output;
    }

    /**
     * mask generator function, as described in PKCS1v2.
     */
    private byte[] maskGeneratorFunction1(
        byte[]  Z,
        int     zOff,
        int     zLen,
        int     length)
    {
        byte[]  mask = new byte[length];
        byte[]  hashBuf = new byte[mgf1Hash.getDigestSize()];
        byte[]  C = new byte[4];
        int     counter = 0;

        mgf1Hash.reset();

        while (counter < (length / hashBuf.length))
        {
            Pack.intToBigEndian(counter, C, 0);

            mgf1Hash.update(Z, zOff, zLen);
            mgf1Hash.update(C, 0, C.length);
            mgf1Hash.doFinal(hashBuf, 0);

            System.arraycopy(hashBuf, 0, mask, counter * hashBuf.length, hashBuf.length);

            counter++;
        }

        if ((counter * hashBuf.length) < length)
        {
            Pack.intToBigEndian(counter, C, 0);

            mgf1Hash.update(Z, zOff, zLen);
            mgf1Hash.update(C, 0, C.length);
            mgf1Hash.doFinal(hashBuf, 0);

            System.arraycopy(hashBuf, 0, mask, counter * hashBuf.length, mask.length - (counter * hashBuf.length));
        }

        return mask;
    }
}

Also, I have issue with decryptedWithOaep length, it was longer than getOutputBlockSize, for this I delete additional 0 from decrypted with SunPKCS11 data. But if I try it for random RSA keys generated with bouncy castle it was not happening.