How to mock authentication in .Net core using TestServer / Specflow

Question

I have a .net 7 api that i tried to test cover using Specflow/xUnit but i cannot seem to mock the authentication for my api.
I always receive Unauthorized on my authenticated endpoints.

The .net 7 api is using the following program.cs :

    public partial class Program
    {
        public static void Main(string[] args)
        {
            var builder = WebApplication.CreateBuilder(args);

            // Configuration
            OidcConfigurationSection oidcConfig = new OidcConfigurationSection();
            builder.Configuration.GetSection(OidcConfigurationSection.SectionName).Bind(oidcConfig);

            // Authentication
            builder.Services.AddAuthentication(options =>
            {
                options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
                options.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme;
            }).AddJwtBearer(options =>
            {
                //options.Authority = oidcConfig.Authority;
                //options.Audience = oidcConfig.Identifier;
                // Received token should have a user identified, the audience it asked for initially and the issuer should be the one that we asked for.
                options.TokenValidationParameters = new TokenValidationParameters()
                {
                    NameClaimType = ClaimTypes.NameIdentifier,
                    ValidAudience = oidcConfig.Identifier,
                    ValidIssuer = oidcConfig.Authority
                };
            });

            ...

            var app = builder.Build();

            ...

            app.Run();
        }
    }

Here is a sample simplified endpoint :

    [ApiController]
    [Route("user/settings")]
    [Authorize]
    public class UserSettingsController : ControllerBase
    {

        ...

        [HttpGet]
        public async Task<IActionResult> Get()
        {
            return Ok(_dataRepo.Get())
        }
    }

And in my specflow project, i have a static FakeJwtManager that can issue token for a random issuer/audience :

public static class FakeJwtManager
    {
        public static string Issuer { get; } = Guid.NewGuid().ToString();
        public static string Audience { get; } = Guid.NewGuid().ToString();
        public static SecurityKey SecurityKey { get; }
        public static SigningCredentials SigningCredentials { get; }

        private static readonly JwtSecurityTokenHandler tokenHandler = new JwtSecurityTokenHandler();
        private static readonly RandomNumberGenerator generator = RandomNumberGenerator.Create();
        private static readonly byte[] key = new byte[32];

        static FakeJwtManager()
        {
            generator.GetBytes(key);
            SecurityKey = new SymmetricSecurityKey(key) { KeyId = Guid.NewGuid().ToString() };
            SigningCredentials = new SigningCredentials(SecurityKey, SecurityAlgorithms.HmacSha256);
        }

        public static string GenerateJwtToken()
        {
            return tokenHandler.WriteToken(new JwtSecurityToken(Issuer, Audience, null, null, DateTime.UtcNow.AddMinutes(10), SigningCredentials));
        }
    }

The FakeJwtManager is used in a FakeJwtWebApplicationFactory<Program> that is supposed to override the previous authentication registered on my Program :

public class FakeJwtWebApplicationFactory<TStartup> : WebApplicationFactory<Program>
    {
        protected override void ConfigureWebHost(IWebHostBuilder builder)
        {
            builder.ConfigureTestServices(services =>
            {
                services.PostConfigure<JwtBearerOptions>(JwtBearerDefaults.AuthenticationScheme, options =>
                {
                    //options.Authority = FakeJwtManager.Issuer;
                    //options.Audience = FakeJwtManager.Audience;
                    options.TokenValidationParameters = new TokenValidationParameters()
                    {
                        IssuerSigningKey = FakeJwtManager.SecurityKey,
                        ValidIssuer = FakeJwtManager.Issuer,
                        ValidAudience = FakeJwtManager.Audience
                    };
                });
            });
        }
    }

However, in my test cases, using the FakeJwtWebApplicationFactory and providing a token emited by the FakeJwtManager, i always get Unauthorized from my endpoints :

[Binding]
    public class UserCreationStepDefinitions
    {
        private readonly FakeJwtWebApplicationFactory<Program> _webApplicationFactory;
        public HttpClient _httpClient { get; set; } = null!;
        private HttpResponseMessage _response { get; set; } = null!;

        public UserCreationStepDefinitions(FakeJwtWebApplicationFactory<Program> webApplicationFactory)
        {
            _webApplicationFactory = webApplicationFactory;
        }

        [Given(@"I am a client")]
        public void GivenIAmAClient()
        {
            _httpClient = _webApplicationFactory.CreateDefaultClient();
        }

        [Given(@"I provide valid authentified oidc user")]
        public void GivenIProvideValidAuthentifiedOidcUser()
        {
            var accessToken = FakeJwtManager.GenerateJwtToken();
            _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken);
        }

        [When(@"I make a GET request to '([^']*)'")]
        public async Task WhenIMakeAGETRequestTo(string endpoint)
        {
            _response = await _httpClient.GetAsync($"{endpoint}");
        }

        [Then(@"The response status code is '([^']*)'")]
        public void ThenTheResponseStatusCodeIs(int statusCode)
        {
            var expected = (HttpStatusCode)statusCode;
            Assert.Equal(expected, _response.StatusCode);   <-- **Always false, expected is 'Unauthorized' instead of 'Ok'
        }
    }

Does anyone has an idea about what might be wrong ?

Regards, Nicolas