I have this registration class as you can see :
public static class InternalSymbolServiceRegister
{
    public const string DefaultConfigSectionName = "myService";
    public static IServiceCollection AddSymbols(
        this IServiceCollection services, 
        IConfiguration configuration,
        HttpMessageHandler customHttpMessageHandler = null,
        string configSectionName = DefaultConfigSectionName)
    {
        services.Configure<SymbolOptions>(configuration.GetSection(configSectionName));
        services.AddHttpClient<ISymbolService, InternalSymbolService>((sp, client) =>
        {
            var options = sp.GetRequiredService<IOptions<SymbolOptions>>().Value;
            client.BaseAddress = new Uri(options.ServiceUrl);
            client.Timeout = TimeSpan.FromMilliseconds(options.TimeoutMs);
          
        }).AddPolicyHandler(GetCircuitBreakerPolicy());
        if (customHttpMessageHandler != null)
        {
            services.Configure<HttpClientFactoryOptions>(nameof(ISymbolService), options =>
            {
                options.HttpMessageHandlerBuilderActions.Add(builder =>
                {
                    builder.PrimaryHandler = customHttpMessageHandler;
                });
            });
        }
        return services;
    }
    private static IAsyncPolicy<HttpResponseMessage> GetCircuitBreakerPolicy()
        => HttpPolicyExtensions
            .HandleTransientHttpError()
            .CircuitBreakerAsync(handledEventsAllowedBeforeBreaking: 5, durationOfBreak: TimeSpan.FromSeconds(7));
}
As you can see I want to inject my httpMessageHandler to my registration here to add custom header :
if (customHttpMessageHandler != null)
{
    services.Configure<HttpClientFactoryOptions>(nameof(ISymbolService), options =>
    {
        options.HttpMessageHandlerBuilderActions.Add(builder =>
        {
            builder.PrimaryHandler = customHttpMessageHandler;
        });
    });
}
Here I add my symbol in startup
services.AddSymbols(Configuration, customHttpMessageHandler: new InternalSymbolHeaderHandler());
Here is my header :
public sealed class InternalSymbolHeaderHandler : DelegatingHandler
{
    protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
    {
        request.Headers.Add("admin_header", "_11111");
        return await base.SendAsync(request, cancellationToken);
    }
}
But when I call my api the header of client is Empty :

                        
After checking the sample code, I found you are missing
services.AddTransient<InternalSymbolHeaderHandler>();.