How to use resnet instead of VGG to improve Cifar 100 performance

Question

In order to improve the performance of cifar 100, we wanted to improve the code by using resnet. However, the accuracy continues to hover around 50%. There seems to be a problem with my code, but since I only know how to use tensorflow, I can't seem to improve the code much. Is there a way to improve this bunch of code?

from tensorflow.keras.datasets import cifar100
from tensorflow.keras.models import Sequential
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, Dropout, BatchNormalization, Dense, Activation, Input, concatenate
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.applications import ResNet50
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf

# Load CIFAR-100 dataset
(X_train, y_train), (X_test, y_test) = cifar100.load_data()

# Data preprocessing
X_train = X_train.astype('float32') / 255.0
X_test = X_test.astype('float32') / 255.0

# Data Number
nb_classes = len(np.unique(y_train))

# Size
img_width, img_height, img_num_channels = 32, 32, 3
input_shape = (img_width, img_height, img_num_channels)

# one hot incode
y_train = to_categorical(y_train, nb_classes)
y_test = to_categorical(y_test, nb_classes)

# Create data augmentation object
datagen = ImageDataGenerator(
    rotation_range=15,
    width_shift_range=0.1,
    height_shift_range=0.1,
    horizontal_flip=True,
    vertical_flip=True,
    fill_mode='nearest'
)
datagen.fit(X_train)

# Model
model = ResNet50(include_top=True, weights=None, input_shape=(32, 32, 3), classes=10)
model = Sequential()

model.add(Conv2D(128, (3, 3), padding='same', input_shape=X_train.shape[1:])) # 입력의 영상의 크기를 유지하기 위해 padding = 'same'으로 사용
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Conv2D(128, (3, 3)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(256, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Conv2D(256, (3, 3)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(512, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Conv2D(512, (3, 3)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(GlobalAveragePooling2D())
model.add(Dense(1024))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.3))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))



model.compile(loss=categorical_crossentropy, optimizer=Adam(learning_rate=0.0001), metrics=['accuracy'])
model.summary()

early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15, restore_best_weights=True)

# Fit data to model with data augmentation and learning rate scheduling
history = model.fit(datagen.flow(X_train, y_train, batch_size=128),
                    steps_per_epoch=len(X_train) / 128,
                    epochs=100,
                    validation_data=datagen.flow(X_test, y_test, batch_size=128),
                    callbacks=[early_stop],
                    verbose=1)

# Plot accuracy
plt.plot(history.history['accuracy'], label='train')
plt.plot(history.history['val_accuracy'], label='test')
plt.legend()
plt.show()

The original code was about 40-45% accurate. We improved it as much as possible and implemented it up to 55%. What else should I do with this?