Access multiple user's Google Fit data from server side periodically

Question

I am trying to access steps count of all users who are resisted in my website. I have setup a cronjob every day at a fixed time. I have created a service account credintials and downloaded its json

public async Task<IActionResult> GoogleFit(string email)
    {
        var scopes = new[] { FitnessService.Scope.FitnessActivityRead };
        var credentials = GoogleCredential.FromJson(System.IO.File.ReadAllText(jsonPath))
            .CreateScoped(scopes);

        var fitnessService = new FitnessService(new BaseClientService.Initializer()
        {
            HttpClientInitializer = credentials,
            ApplicationName = "App name"
        });

        var dataSources = await fitnessService.Users.DataSources.List(email).ExecuteAsync();
        var foundData = dataSources.DataSource.FirstOrDefault(x => x.Type == "raw")?.DataStreamId;
        if (foundData == null)
        {
            return NotFound("No data source");
        }

        var start = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 0, 0, 0, DateTimeKind.Utc);
        var end = start.AddDays(1);
        DateTime unixStart = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
        long unixTimeStart = (long)(start - unixStart).TotalSeconds;
        long unixTimeEnd = (long)(end - unixStart).TotalSeconds;
        var result = await fitnessService.Users.Dataset.Aggregate(new AggregateRequest()
        {
            AggregateBy = new List<AggregateBy>
                        {
                             new AggregateBy
                                    {
                                         DataSourceId = foundData,
                                         DataTypeName = "com.google.step_count.delta"
                                    }
                         },
            BucketByTime = new BucketByTime
            {
                DurationMillis = 86400000
            },
            StartTimeMillis = unixTimeStart,
            EndTimeMillis = unixTimeEnd

        }, email).ExecuteAsync();
        return Ok(result);
    }

This is the api i use to test, when i give a email i get the

    The service fitness has thrown an exception.
HttpStatusCode is Forbidden.
Google.Apis.Requests.RequestError
The caller does not have permission [403]
Errors [
    Message[The caller does not have permission] Location[ - ] Reason[forbidden] Domain[global]
]

Also if i pass "me" instead of email in var dataSources = await fitnessService.Users.DataSources.List(email).ExecuteAsync();

i get no error but dataSources count is 0