Make Bulk rest calls to an API

Question

There's a rest API.. which I need to call. Post Call - it has a payload

{ "name" : <<value_in_excel>>,
  "something" : "all rest values are fixed"
}

There are 100,000 values for name attributes in an Excel/CSV. One API call takes 800ms-1sec.

How can I make this bulk call?

Answer 1

Reading excel file by excel.js and call to create user by POST call using Axios library.

Overview steps

This is demo code.

File Structure

Save as 'create_user.js'

const axios = require('axios');
const qs = require('qs');
const ExcelJS = require('exceljs');

// Keycloak details
const client_id = 'admin-cli';
const user_name = 'admin';
const pass_word = 'admin';
const grant_type = 'password';
const tokenURL = 'http://localhost:8080/auth/realms/master/protocol/openid-connect/token';
const usersEndpoint = 'http://localhost:8080/auth/admin/realms/my-realm/users';

// Read from Excel file
async function readExcelFile() {
    const workbook = new ExcelJS.Workbook();
    await workbook.xlsx.readFile('Users.xlsx');
    const worksheet = workbook.getWorksheet('Users');
    let users = [];
    worksheet.eachRow({ includeEmpty: false }, (row, rowNumber) => {
        if (rowNumber !== 1) { // Skipping header row
            users.push({
                username: row.getCell(1).value,
                firstName: row.getCell(2).value,
                lastName: row.getCell(3).value,
                email: row.getCell(4).value
            });
        }
    });
    return users;
}

// Function to get token
async function getToken() {
    const data = qs.stringify({
        client_id: client_id,
        username: user_name,
        password: pass_word,
        grant_type: grant_type
    });

    const config = {
        headers: {
            'Content-Type': 'application/x-www-form-urlencoded'
        }
    };

    try {
        const response = await axios.post(tokenURL, data, config);
        return response.data.access_token;
    } catch (error) {
        console.error('Error fetching token:', error);
        throw error;
    }
}

// Function to create user in Keycloak
async function createUserInKeycloak(user, token) {
    try {
        await axios.post(usersEndpoint, {
            username: user.username,
            email: user.email,
            firstName: user.firstName,
            lastName: user.lastName,
            enabled: true,
            emailVerified: true
        }, {
            headers: {
                'Authorization': `Bearer ${token}`,
                'Content-Type': 'application/json'
            }
        });
        console.log(`User created: ${user.username}`);
    } catch (error) {
        console.error(`Error creating user ${user.username}:`, error);
    }
}

// Main function to handle the process
async function main() {
    try {
        const users = await readExcelFile();
        const token = await getToken();
        for (const user of users) {
            await createUserInKeycloak(user, token);
        }
    } catch (error) {
        console.error('An error occurred:', error);
    }
}

main();

Run it

node create_user.js

This is result.

Install Dependencies

Save as 'package.json'

{
  "dependencies": {
    "@faker-js/faker": "^8.3.1",
    "axios": "^1.6.3",
    "exceljs": "^4.4.0",
    "qs": "^6.11.2"
  }
}

Install dependencies

npm install

This is create 100K users by fake.js

Save as 'save_excel.js'

const { faker } = require('@faker-js/faker');
const ExcelJS = require('exceljs');

const USER_COUNT = 100000;
const FILE_NAME = 'Users.xlsx';

// Sets to store used usernames and emails
const usedUsernames = new Set();
const usedEmails = new Set();

// Function to create a random user with a unique username and email
const createRandomUser = () => {
    let username, email;

    do {
        username = faker.internet.userName();
        if (usedUsernames.has(username)) {
            username += Math.floor(Math.random() * 10000);
        }
    } while (usedUsernames.has(username));
    usedUsernames.add(username);

    do {
        email = faker.internet.email();
        if (usedEmails.has(email)) {
            email = email.split('@')[0] + Math.floor(Math.random() * 10000) + '@' + email.split('@')[1];
        }
    } while (usedEmails.has(email));
    usedEmails.add(email);

    return {
        username: username,
        firstName: faker.person.firstName(),
        lastName: faker.person.lastName(),
        email: email
    };
}

// Function to write users to Excel file
const saveUsersToExcel = async (users) => {
    const workbook = new ExcelJS.Workbook();
    const worksheet = workbook.addWorksheet('Users');

    // Define columns
    worksheet.columns = [
        { header: 'Username', key: 'username', width: 20 },
        { header: 'First Name', key: 'firstName', width: 15 },
        { header: 'Last Name', key: 'lastName', width: 15 },
        { header: 'Email', key: 'email', width: 50 }
    ];

    // Add users to the worksheet
    worksheet.addRows(users);

    // Save workbook
    await workbook.xlsx.writeFile(FILE_NAME);
    console.log(`Saved ${users.length} users to ${FILE_NAME}`);
}

// Main function to create users and save to Excel
const main = async () => {
    let users = [];

    for (let i = 0; i < USER_COUNT; i++) {
        users.push(createRandomUser());
    }

    await saveUsersToExcel(users);
}

main().catch(err => console.error(err));

Create excel file

node save_excel.js

Excel result

Terminal

I created 100K by fake.js. To avoid username and email code too.

This is launch Keycloak by docker compose

version: '3.7'

services:
  postgres:
      image: postgres
      volumes:
        - postgres_data:/var/lib/postgresql/data
      environment:
        POSTGRES_DB: keycloak
        POSTGRES_USER: keycloak
        POSTGRES_PASSWORD: password
  keycloak:
      image: quay.io/keycloak/keycloak:legacydocke
      environment:
        DB_VENDOR: POSTGRES
        DB_ADDR: postgres
        DB_DATABASE: keycloak
        DB_USER: keycloak
        DB_SCHEMA: public
        DB_PASSWORD: password
        KEYCLOAK_USER: admin
        KEYCLOAK_PASSWORD: admin
        # KC_FEATURES:
        #   scripts
      ports:
        - 8080:8080 # Expose to user with localhost:8080
      restart: always
      depends_on:
        - postgres

volumes:
  postgres_data:  # Keycloack volume
      driver: local

Launching Keycloak

docker compose up

Create 'my-realm' by manual

Increase Master Token Lifespan

More detail information

Create 100K users by Fake user name https://fakerjs.dev/api/person.html

Reading Excel by excel.js https://www.npmjs.com/package/exceljs

Get master token for Keycloak Keycloak v.18: How to manipulate with users using Keycloak API

Create user by POST call for Keycloak Is it possible to create new user with jboss/keycloak:16.1.1using REST APIs?