how to use cross-validation with ktrain?

Question

I am using the ktrain package to perform multiclass text classification. The example on the official ktrain website works great (https://github.com/amaiya/ktrain)

categories = ['alt.atheism', 'soc.religion.christian','comp.graphics', 'sci.med']
from sklearn.datasets import fetch_20newsgroups
train_b = fetch_20newsgroups(subset='train', categories=categories, shuffle=True)
test_b = fetch_20newsgroups(subset='test',categories=categories, shuffle=True)
(x_train, y_train) = (train_b.data, train_b.target)
(x_test, y_test) = (test_b.data, test_b.target)

# build, train, and validate model (Transformer is wrapper around transformers library)
import ktrain
from ktrain import text
MODEL_NAME = 'distilbert-base-uncased'
t = text.Transformer(MODEL_NAME, maxlen=500, class_names=train_b.target_names)
trn = t.preprocess_train(x_train, y_train)
val = t.preprocess_test(x_test, y_test)
model = t.get_classifier()
learner = ktrain.get_learner(model, train_data=trn, val_data=val, batch_size=6)
learner.fit_onecycle(5e-5, 4)
learner.validate(class_names=t.get_classes())

Accuracy is pretty high.

However, I am comparing this model with other models trained with scikit-learn and, in particular, the other models' accuracy is assessed using cross validation

cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring="accuracy")

How can I adapt the code above to make sure the transformer model used with ktrain is also evaluated with the same cross validation methodology?

Answer 1

You can try something like this:

from ktrain import text
import ktrain
import pandas as pd
from sklearn.model_selection import train_test_split,KFold
from sklearn.metrics import accuracy_score
from sklearn.datasets import fetch_20newsgroups

# load text data
categories = ['alt.atheism', 'soc.religion.christian','comp.graphics', 'sci.med']
train_b = fetch_20newsgroups(subset='train', categories=categories, shuffle=True)
test_b = fetch_20newsgroups(subset='test',categories=categories, shuffle=True)
(x_train, y_train) = (train_b.data, train_b.target)
(x_test, y_test) = (test_b.data, test_b.target)
df = pd.DataFrame({'text':x_train, 'target': [train_b.target_names[y] for y in y_train]})

# CV with transformers
N_FOLDS = 2
EPOCHS = 3
LR = 5e-5
def transformer_cv(MODEL_NAME):
    predictions,accs=[],[]
    data = df[['text', 'target']]
    for train_index, val_index in KFold(N_FOLDS).split(data):
        preproc  = text.Transformer(MODEL_NAME, maxlen=500)
        train,val=data.iloc[train_index],data.iloc[val_index]
        x_train=train.text.values
        x_val=val.text.values

        y_train=train.target.values
        y_val=val.target.values

        trn = preproc.preprocess_train(x_train, y_train)
        model = preproc.get_classifier()
        learner = ktrain.get_learner(model, train_data=trn, batch_size=16)
        learner.fit_onecycle(LR, EPOCHS)
        predictor = ktrain.get_predictor(learner.model, preproc)
        pred=predictor.predict(x_val)
        acc=accuracy_score(y_val,pred)
        print('acc',acc)
        accs.append(acc)
    return accs
print( transformer_cv('distilbert-base-uncased') )

# output:
# [0.9627989371124889, 0.9689716312056738]

REFERENCE: See this Kaggle notebook for a regression example.