Not getting roles using a SSO identity framework 6 server

Question

I have been trying to make a login server to use as single sign on server for multiple applications but I can't seem to get roles on my client side of the application. When I check the roles in User.Claims nothing is there and [Authorize(roles = "Admin")] doesn't seem to work either

My current code is:

        JwtSecurityTokenHandler.DefaultInboundClaimTypeMap.Clear();

        builder.Services.AddAuthentication(options =>
        {
            options.DefaultScheme = "Cookies";
            options.DefaultChallengeScheme = "oidc";
        })
            .AddCookie(options =>
            {
                options.Cookie.HttpOnly = true;
                options.ExpireTimeSpan = TimeSpan.FromMinutes(30);
                options.SlidingExpiration = true;
            })
            .AddOpenIdConnect("oidc", options =>
            {
                options.SignInScheme = "Cookies";
                options.Authority = "https://localhost:5001";
                options.ClientId = "CRMERPClient";
                options.ClientSecret = "secret";
                options.MapInboundClaims = false;
                options.SaveTokens = true;
                options.RequireHttpsMetadata = true;
                options.GetClaimsFromUserInfoEndpoint = true;


                // code flow + PKCE (PKCE is turned on by default)
                options.ResponseType = "code";
                options.UsePkce = true;
        
                options.Scope.Add("crm");
                options.Scope.Add("erp");

                options.TokenValidationParameters.NameClaimType = "name";
                options.TokenValidationParameters.RoleClaimType = "role";

                options.ClaimActions.MapJsonKey("role", "role");
            });

        builder.Services.AddSession(options =>
        {
            options.IdleTimeout = TimeSpan.FromMinutes(120);
            options.Cookie.HttpOnly = true;
            options.Cookie.IsEssential = true;
        });

^ My client side program.cs

            builder.Services.AddIdentityServer(options =>
            {
                options.Events.RaiseErrorEvents = true;
                options.Events.RaiseInformationEvents = true;
                options.Events.RaiseFailureEvents = true;
                options.Events.RaiseSuccessEvents = true;
                options.EmitStaticAudienceClaim = true;
            })
                .AddInMemoryIdentityResources(Config.IdentityResources)
                .AddInMemoryApiResources(Config.ApiResources)
                .AddInMemoryApiScopes(Config.ApiScopes)
                .AddInMemoryClients(Config.Clients)
                .AddTestUsers(TestUsers.Users)
                .AddServerSideSessions()
                .AddServerSideSessionStore<CustomServerSessionStore>()
                .AddDeveloperSigningCredential().AddProfileService<ProfileService>();

            builder.Services.AddTransient<IProfileService, ProfileService>();

^ My Server side program.cs

    public static IEnumerable<IdentityResource> IdentityResources =>
        new List<IdentityResource>
        {
            new IdentityResources.OpenId(),
            new IdentityResources.Profile(),
        };

    public static IEnumerable<ApiScope> ApiScopes =>
        new List<ApiScope>
        {
            new ApiScope("crm", "CRM"),
            new ApiScope("erp", "ERP"),
            new ApiScope("dms", "DMS")
        };

    public static IEnumerable<ApiResource> ApiResources =>
        new List<ApiResource>
        {
            new ApiResource("crm")
            {
                Scopes = { "crm", "roles"}
            },

            new ApiResource("erp")
            {
                Scopes = { "erp", "roles"}
            },

            new ApiResource("dms")
            {
                Scopes = { "dms", "roles"}
            }
        };

    public static IEnumerable<Client> Clients =>
        new List<Client>
        {
            new Client
            {
                ClientId = "CRMERPClient",

                AllowedGrantTypes = GrantTypes.Code,

                // where to redirect after login
                RedirectUris = { "https://localhost:7218/signin-oidc" },

                // where to redirect after logout
                PostLogoutRedirectUris = { "https://localhost:7218/signout-callback-oidc" },

                ClientSecrets =
                {
                    new Secret("secret".Sha256())
                },

                //https://docs.duendesoftware.com/identityserver/v6/quickstarts/1_client_credentials/
                  AllowedScopes = { "crm", "erp",
                        IdentityServerConstants.StandardScopes.OpenId,
                        IdentityServerConstants.StandardScopes.Profile,
                        IdentityServerConstants.StandardScopes.Email,
                        JwtClaimTypes.Role
                    },
            },
            new Client
            {
                ClientId = "DocumentManagementSystem",
                ClientSecrets = { new Secret("secret".Sha256()) },

                AllowedGrantTypes = GrantTypes.Code,

                RedirectUris = { "https://localhost:7218/signin-oidc" },
                PostLogoutRedirectUris = { "https://localhost:7218/signout-callback-oidc" },

                AllowOfflineAccess = true,
                AllowedScopes = { "openid", "profile", "dms", "roles" }
            }
        };

^ My config of the Identity Server

    public class ProfileService : IProfileService
    {
        public ProfileService() { }

        public Task GetProfileDataAsync(ProfileDataRequestContext context)
        {
            var roleClaims = context.Subject.FindAll(JwtClaimTypes.Role);
            context.IssuedClaims.AddRange(roleClaims);
            return Task.CompletedTask;
        }

        public Task IsActiveAsync(IsActiveContext context)
        {
            return Task.CompletedTask;
        }
    }

^And my profile Service

Am I missing something here? I tried to resolve the issue by following numerous tutorials and looking over stackoverflow but it was hard to find identity server 6 answers instead of identity server 4.

Thank you for the help!