How to implement ServerCertificateCustomValidationCallback from HttpClientHandler

Question

I'm trying to implement a method for ServerCertificateCustomValidationCallback for an instance of HttpClientHandler class. I'm at a loss here because I don't know what to validate. I need to send a certificate to the server (which I'm already doing), as well as validate the certificates the server sends me. I have seen this and this implementations, which tell me to compare the received certificates with a known value. I'm not sure how to get this known value, I know that I have a trusted certificate installed in the LocalMachine's Trusted root store, should I expect a certificate from the chain to contain the same hash of my installed certificate?

Here's what I have so far:

private HttpClientHandler CreateHandler()
{
    var handler = new HttpClientHandler
    {
        SslProtocols = SslProtocols.Tls12
    };
    var certPath = System.Configuration.ConfigurationManager.AppSettings["MY_CERT_PATH"];
    handler.ClientCertificates.Add(new X509Certificate2(Path.Combine(certPath, "tls.pfx")));
    handler.CheckCertificateRevocationList = true;
    handler.ServerCertificateCustomValidationCallback = ValidateServerCertificate;
    return handler;
}

private bool ValidateServerCertificate(HttpRequestMessage message, X509Certificate2 certificate, X509Chain chain, SslPolicyErrors policy)
{
    if (policy == SslPolicyErrors.None)
        return true;

    using (var store = new X509Store(StoreName.Root, StoreLocation.LocalMachine))
    {
        store.Open(OpenFlags.ReadOnly);
        foreach (var element in chain.ChainElements)
        {
            var hashes = store.Certificates.Find(X509FindType.FindBySubjectDistinguishedName, element.Certificate.Subject, true)
                .Cast<X509Certificate2>()
                .Select(cert => cert.GetCertHashString(HashAlgorithmName.SHA512));
            var hash = element.Certificate.GetCertHashString(HashAlgorithmName.SHA512);
            if (hashes.Any(h => h == hash))
                return true;
        }

        store.Close();
    }

    return false;
}