Troubleshooting TLS Handshake Failure after OpenSSL and OpenLDAP Upgrade

129 Views Asked by At

I have encountered an issue while upgrading OpenSSL and OpenLDAP on our Windows 2019 server.

The upgrade involved transitioning from OpenSSL version 1.1.1 to version 3.0.10 and OpenLDAP version 2.4.47 to version 2.5.16. However, I have encountered a problem with the TLS Handshake when attempting to connect the client with the LDAP server. It appears that the client certificates are not being picked up by the LDAP server.

To diagnose the issue, I created a console application and conducted the following scenarios:

Using the command prompt, I successfully connected the upgraded OpenSSL with the upgraded LDAP server by setting the client certificate as an environment variable. The client was able to establish a connection with the server and receive responses without encountering any errors.

Similarly, using the command prompt, I attempted to connect the upgraded LDAP server with the client "ldapsearch.exe" by setting the same client certificate as an environment variable. Once again, the client was able to establish a connection with the server and receive responses without any errors.

The TLS Handshake fails when the client attempts to connect with the LDAP server through our application.

Given Below is the code snippet that produces the TLS Handshake failure.

**Test1.cpp : Defines the entry point for the console application.**

#include "stdafx.h"
#include "windows.h"
#include "winldap.h"
#include "stdio.h"
#include "wincrypt.h"
#include <openssl/ssl.h>
#include <openssl/err.h>
//#include "ldap.h"

#pragma comment(lib, "wldap32.lib")
#pragma comment(lib, "crypt32.lib")
#pragma comment(lib, "D:/Test1/libssl.lib") //lib built using openssl3.0.10
#pragma comment(lib, "D:/Test1/libcrypto.lib") //lib built using openssl3.0.10

//The following LDAP TLS options are mentioned in ldap.h

#define LDAP_OPT_X_TLS_REQUIRE_CERT           0x6006
#define LDAP_OPT_X_TLS_HARD                              1
#define LDAP_OPT_X_TLS_ALLOW           3
#define LDAP_OPT_X_TLS_CACERTFILE  0x6002
#define LDAP_OPT_X_TLS_CACERTDIR   0x6003
#define LDAP_OPT_X_TLS_CERTFILE                       0x6004
#define LDAP_OPT_X_TLS_KEYFILE                          0x6005
#define LDAP_OPT_X_TLS_CIPHER                           0x6014
#define LDAP_OPT_X_TLS_PROTOCOL_TLS1_3                   ((3 << 8) + 4)

const size_t newsize = 100;
//  Entry point for application

int main(int argc, char* argv[])

{
    PWCHAR hostName = NULL;
    LDAP* pLdapConnection = NULL;
    ULONG version = LDAP_VERSION3;
    ULONG getOptSuccess = 0;
    ULONG connectSuccess = 0;
    INT returnCode = 0;

                PCCERT_CONTEXT cert_ctx = NULL;
                ULONG lv;
                const char *cert_path = "D:/Test1/c_usr";
                const char *cert_file = "c_usr.crt"; // Path to the client certificate file
                const char *key_file = "c_usr.key.pem"; // Path to the private key file associated with the client certificate
                const char *CA_file = "root_ca.crt";
                const char *cipher_tls = "TLS_AES_256_GCM_SHA384";

    //  Pass the hostname.

    if (argc > 1)
    {
        //  Convert argv[] to a wchar_t*
        size_t origsize = strlen(argv[1]) + 1;
        size_t convertedChars = 0;
        wchar_t wcstring[newsize];
        mbstowcs_s(&convertedChars, wcstring, origsize, argv[1], _TRUNCATE);
        wcscat_s(wcstring, L" (wchar_t *)");
        hostName = wcstring;
    }
    else
    {
        hostName = NULL;
    }
    //  Initialize a session. LDAP_PORT is the default port, 389.
               pLdapConnection=ldap_sslinit(hostName, 636, 1);
    if (pLdapConnection == NULL)
    {
        //  Set the HRESULT based on the Windows error code.
        char hr = HRESULT_FROM_WIN32(GetLastError());
        printf( "ldap_init failed with 0x%x.\n",hr);
        goto error_exit;
    }
   else
        printf("ldap_init succeeded \n");
    //  Set the version to 3.0 .
    returnCode = ldap_set_option(pLdapConnection,
                                 LDAP_OPT_PROTOCOL_VERSION,
                                 (void*)&version);
   if(returnCode == LDAP_SUCCESS)
        printf("ldap_set_option succeeded - version set to 3\n");
    else
    {
        printf("SetOption Error:%0X\n", returnCode);
        goto error_exit;
    }
                returnCode = ldap_set_option(pLdapConnection,
                                 LDAP_OPT_X_TLS_CACERTDIR,
                                 (void*)cert_path);
   if(returnCode == LDAP_SUCCESS)
        printf("ldap_set_option succeeded - certificate path\n");
    else
    {
        printf("SetOption Error:%0X\n", returnCode);
        goto error_exit;
    }

                 
    // Set TLS/SSL options
     returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_CIPHER, cipher_tls);
                if(returnCode == LDAP_SUCCESS)
        printf("ldap_set_option succeeded - cipher set to TLS_AES_256_GCM_SHA384\n");
     else
     {
        printf("SetOption Error:%0X\n", returnCode);
        goto error_exit;
    }

    returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_REQUIRE_CERT, (void *)LDAP_OPT_X_TLS_ALLOW); // Require valid server certificate
                if(returnCode == LDAP_SUCCESS)
        printf("ldap_set_option succeeded - LDAP_OPT_X_TLS_ALLOW \n");
     else
     {
        printf("SetOption Error:%0X\n", returnCode);
        goto error_exit;
    }

    returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_CERTFILE, cert_file); // Set client certificate file
                if(returnCode == LDAP_SUCCESS)
        printf("ldap_set_option succeeded - client certificate\n");
     else
     {
        printf("SetOption Error:%0X\n", returnCode);
        goto error_exit;
    }

    returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_KEYFILE, key_file); // Set private key file
                if(returnCode == LDAP_SUCCESS)
        printf("ldap_set_option succeeded - certificate key file\n");
     else
     {
        printf("SetOption Error:%0X\n", returnCode);
        goto error_exit;
    }

                returnCode = ldap_set_option(pLdapConnection, LDAP_OPT_X_TLS_CACERTFILE, CA_file); // Set client certificate file
                if(returnCode == LDAP_SUCCESS)
        printf("ldap_set_option succeeded - CA file \n");
     else
     {
        printf("SetOption Error:%0X\n", returnCode);
        goto error_exit;
    }

     connectSuccess = ldap_connect(pLdapConnection, NULL);
    if(connectSuccess == LDAP_SUCCESS)
        printf("ldap_connect succeeded \n");
    else
    {
        printf("ldap_connect failed with 0x%x.\n",connectSuccess);
        goto error_exit;
    }

 
    //  Bind with current credentials (login credentials). Be
    //  aware that the password itself is never sent over the
   //  network, and encryption is not used.

    printf("Binding ...\n");
    returnCode = ldap_bind_s(pLdapConnection, NULL, NULL,
                             LDAP_AUTH_NEGOTIATE);
    if (returnCode == LDAP_SUCCESS)
        printf("The bind was successful");
    else
        goto error_exit;
    //  Normal cleanup and exit.

    ldap_unbind(pLdapConnection);
    return 0;
    //  On error cleanup and exit.
    error_exit:
        ldap_unbind(pLdapConnection);
        return -1;
}

Using the command prompt, I tried the following

openssl s_client -debug -connect server:636 -starttls ldap -tls1_3 -cert c_usr.crt -key c_usr.key.pem -CAfile c_ca.crt :Unable to connect the server when the option "-starttls ldap" is used
Whereas the openssl client gets connected to the ldapserver when the option "-starttls ldap" is not provided.

Error Log#
connection_read(3): checking for input on id=1000
TLS trace: SSL_accept:before SSL initialization
tls_read: want=5, got=5
0000:  30 1d 02 01 01                                     0....
TLS trace: SSL_accept:error in error
TLS: can't accept: error:0A00010B:SSL routines::wrong version number.
connection_read(3): TLS accept failure error=-1 id=1000, closing
connection_closing: readying conn=1000 sd=3 for close
daemon: activity on 1 descriptor
daemon: waked
daemon: WSselect: listen=2 active_threads=0 tvp=NULL
connection_close: conn=1000 sd=3
daemon: removing 3
conn=1000 fd=3 closed (TLS negotiation failure)

openssl s_client -debug -connect server:636 -tls1_3 -cert c_usr.crt -key c_usr.key.pem -CAfile c_ca.crt -  
Able to connect the LDAP server

LDAP SERVER LOG for failure scenario:

conn=1000 fd=3 ACCEPT from IP=XX.XXX.XXX.XX:XXXXX (IP=0.0.0.0:636)
connection_get(3)
connection_get(3): got connid=1000
connection_read(3): checking for input on id=1000
TLS trace: SSL_accept:before SSL initialization
tls_read: want=5, got=5
tls_read: want=142, got=142
TLS trace: SSL_accept:SSLv3/TLS write server done
tls_read: want=5 error=Unknown error
TLS trace: SSL_accept:error in SSLv3/TLS write server done
daemon: activity on 1 descriptor
daemon: waked
daemon: WSselect: listen=2 active_threads=0 tvp=NULL
daemon: activity on 4 descriptors
daemon: activity on:
daemon: read activity on
connection_get(3)
daemon: WSselec: listen=2 active_threads=0 tvp=NULL
connection_get(3): got connid=1000
daemon: activity on 3 descriptors
connection_read(3): checking for input on id=1000
slap_listener_activate(2):
tls_read: want=5, got=0
daemon: activity on:65d4802a.10506dd7 00003A10 >>> slap_listener(ldaps://)
daemon: accept() = 51328
TLS trace: SSL_accept:error in SSLv3/TLS write server done
daemon: WSselect: listen=2 busy
daemon: listen=2, new connection on 4
daemon: added 4r (active) listener=00000000
daemon: activity on 1 descriptor
TLS: can't accept: (unknown).
conn=1001 fd=4 ACCEPT from IP=XX.XXX.XXX.XX:XXXXX (IP=0.0.0.0:636)
daemon: waked
connection_read(3): TLS accept failure error=-1 id=1000, closing
daemon: WSselect: listen=2 active_threads=0 tvp=NULL
connection_closing: readying conn=1000 sd=3 for close
daemon: activity on 5 descriptors
connection_close: conn=1000 sd=3
daemon: waked
daemon: removing
daemon: WSselect: listen=2 active_threads=0 tvp=NULL
conn=1000 fd=3 closed (TLS negotiation failure)
daemon: activity on 5 descriptors
daemon: waked
daemon: WSselect: listen=2 active_threads=0 tvp=NULL
daemon: activity on 5 descriptors
daemon: activity on:
daemon: read activity on
connection_get(4)
daemon: WSselect: listen=2 active_threads=0 tvp=NULL
connection_get(4): got connid=1001
connection_read(4): checking for input on id=1001
TLS trace: SSL_accept:before SSL initialization
tls_read: want=5, got=5
tls_read: want=142, got=142
TLS trace: SSL_accept:before SSL initialization
TLS trace: SSL_accept:SSLv3/TLS read client hello
TLS trace: SSL_accept:SSLv3/TLS write server hello
TLS trace: SSL_accept:SSLv3/TLS write certificate
TLS trace: SSL_accept:SSLv3/TLS write key exchange
TLS trace: SSL_accept:SSLv3/TLS write certificate request
tls_write: want=1807, written=1807
TLS trace: SSL_accept:SSLv3/TLS write server done
tls_read: want=5 error=Unknown error
TLS trace: SSL_accept:error in SSLv3/TLS write server done
daemon: activity on 1 descriptor
daemon: waked
daemon: WSselect: listen=2 active_threads=0 tvp=NULL
daemon: shutdown requested and initiated.
daemon: removing
daemon: closing
connection_closing: readying conn=1001 sd=4 for close
connection_close: conn=1001 sd=4
daemon: removing 4
conn=1001 fd=4 closed (slapd shutdown)
slapd shutdown: waiting for 0 operations/tasks to finish
slapd shutdown: initiated
slapd destroy: freeing system resources.
slapd stopped.
0

There are 0 best solutions below