TweetNaCl.js encryption and decryption not working properly

Question

I need to encrypt a message with a user's public key. The user will receive the message and will have to decrypt it having the private key. I'm trying in every way but I can't figure out what I'm doing wrong. I made a node script that works standalone by installing only the two necessary dependencies (tweetnacl and tweetnacl-utils). Apparently the message is encrypted correctly, but when I try to decrypt it I get "null" to the decrypted_message constant.

I already use both libraries to sign messages and the signing part works fine, so I'm not going to switch libraries. Can you tell me what I'm doing wrong? This is the code you can run with node js

const nacl = require('tweetnacl');
const naclUtil = require('tweetnacl-util');

// Getting keypair from a key
const x25519_from_key = (key) => {
    console.log(key.length)
    const boxKeyPair = nacl.box.keyPair.fromSecretKey(key);
    const secretKey = boxKeyPair.secretKey;
    const publicKey = boxKeyPair.publicKey;

    const result = {
        publicKey:
        {
            uint: publicKey,
            base64: naclUtil.encodeBase64(publicKey)
        },
        secretKey: {
            uint: secretKey,
            base64: naclUtil.encodeBase64(secretKey)
        }
    }

    return result;
};

// Encrypt a message using the public key
const encryptMessage = (message, x25519_public_uint) => {

    const nonce = nacl.randomBytes(nacl.box.nonceLength);
    const message_decoded = naclUtil.decodeUTF8(message);

    const encrypted_message = nacl.box.after(
        message_decoded,
        nonce,
        x25519_public_uint
    );

    return {
        nonce_base64: naclUtil.encodeBase64(nonce),
        encrypted_message_encoded: naclUtil.encodeBase64(encrypted_message),
    };
};

// Decrypt message using the secret key
const decryptMessage = (encrypted_message, nonce_base64, x25519_secret_key) => {

  
    const nonce = naclUtil.decodeBase64(nonce_base64);
    const encrypted_message_decoded = naclUtil.decodeBase64(encrypted_message);
  
    const decrypted_message = nacl.box.open.after(
      encrypted_message_decoded,
      nonce,
      x25519_secret_key
    );


    return naclUtil.encodeUTF8(decrypted_message);
  };
  

// Generating keypair x25519
const keyPair = x25519_from_key(nacl.randomBytes(nacl.box.secretKeyLength));

// Encrypt with publicKey
const message = 'This is a plain message';
const encrypted = encryptMessage(message, keyPair.publicKey.uint);


console.log('Original message:', message);

// Show encrypted message
console.log('Encrypted message:', encrypted.encrypted_message_encoded);
console.log('Nonce:', encrypted.nonce_base64);


// Show decrypted message
const decrypted = decryptMessage(encrypted.encrypted_message_encoded, encrypted.nonce_base64, keyPair.secretKey.uint);
console.log('Decrypted message:', decrypted);

Answer 1

When encrypting or decrypting, the own secret key and the public key of the other side have to be applied. This is shown in the following working code, which is essentially based on your code:

var naclUtil = nacl.util

// Getting keypair from a key
const x25519_from_key = (key) => {
    console.log(key.length)
    const boxKeyPair = nacl.box.keyPair.fromSecretKey(key);
    const secretKey = boxKeyPair.secretKey;
    const publicKey = boxKeyPair.publicKey;

    const result = {
        publicKey:
        {
            uint: publicKey,
            base64: naclUtil.encodeBase64(publicKey)
        },
        secretKey: {
            uint: secretKey,
            base64: naclUtil.encodeBase64(secretKey)
        }
    }

    return result;
};

// Encrypt message using the own secret key and the public key of the other side
const encryptMessage = (message, x25519_public_uint, x25519_secret_key) => {

    const nonce = nacl.randomBytes(nacl.box.nonceLength);
    const message_decoded = naclUtil.decodeUTF8(message);

    const encrypted_message = nacl.box(
        message_decoded,
        nonce,
        x25519_public_uint,
        x25519_secret_key
    );

    return {
        nonce_base64: naclUtil.encodeBase64(nonce),
        encrypted_message_encoded: naclUtil.encodeBase64(encrypted_message),
    };
};

// Decrypt message using the own secret key and the public key of the other side
const decryptMessage = (encrypted_message, nonce_base64, x25519_public_uint, x25519_secret_key) => {
  
    const nonce = naclUtil.decodeBase64(nonce_base64);
    const encrypted_message_decoded = naclUtil.decodeBase64(encrypted_message);
    const decrypted_message = nacl.box.open(
      encrypted_message_decoded,
      nonce,
      x25519_public_uint,
      x25519_secret_key
    );

    return naclUtil.encodeUTF8(decrypted_message);
  };
  

// Generating keypair x25519 for A and B side
const keyPairA = x25519_from_key(nacl.randomBytes(nacl.box.secretKeyLength));
const keyPairB = x25519_from_key(nacl.randomBytes(nacl.box.secretKeyLength));

// A side: Encrypt message using the own secret key (A side) and the public key of the other side (B side)
const message = 'This is a plain message';
const encrypted = encryptMessage(message, keyPairB.publicKey.uint, keyPairA.secretKey.uint);

console.log('Original message:', message);

// Show encrypted message
console.log('Encrypted message:', encrypted.encrypted_message_encoded);
console.log('Nonce:', encrypted.nonce_base64);


// B side: Decrypt message using the own secret key (B side) and the public key of the other side (A side)
const decrypted = decryptMessage(encrypted.encrypted_message_encoded, encrypted.nonce_base64, keyPairA.publicKey.uint, keyPairB.secretKey.uint);
console.log('Decrypted message:', decrypted);

<script src="https://cdn.jsdelivr.net/npm/[email protected]/nacl-util.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/nacl-fast.min.js"></script>

The main difference with your code is that the code posted here uses nacl.box() (instead of nacl.box.after()) for encryption and nacl.box.open() (instead of nacl.box.open.after()) for decryption.

NaCl/Libsodium allows the determination of the shared secret (before() methods) and the encryption/decryption (after() methods) to be performed separately, see here in the TweetNaCl documentation or more detailed here in the Libsodium documentation.

In your code, the after() methods are applied incorrectly (after() requires the shared secret, which in turn has to be determined by the before() methods).