Low Precision and Recall in LSTM Anomaly Detection Model

Question

My LSTM anomaly detection model for telemetry data has high accuracy (90%) but struggles with low precision and recall (around 10%) for identifying actual anomalies. I suspect the issue might be related to the data preprocessing (imputation with normal distribution) or the use of StandardScaler.

I've been experimenting with various hyperparameter configurations (model complexity, anomaly thresholds) to optimize my LSTM-based anomaly detection model for telemetry data. While the model achieves a high overall accuracy of 90% on the test set, it yields disappointing results in terms of precision and recall for actual anomaly detection, hovering around 10% for both metrics. This suggests the model might be struggling to differentiate between normal and anomalous data points.

This is the code:

import numpy as np
import pandas as pd
import json
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler,RobustScaler
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import LSTM, Dropout, RepeatVector, TimeDistributed, Dense
from keras.callbacks import EarlyStopping
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

# Load the Excel file
df = pd.read_excel('C:/DL Tutorials/anomaly3.xlsx')

# Select telemetry data columns
telemetry_columns = ['EnclosureTemp', 'Noise', 'Solar_Battery_Voltage', 'Dust_2_5', 'Dust_10']

# Function to safely load JSON data
def safe_json_loads(json_str):
    try:
        # Remove characters after the closing curly brace ('}')
        clean_json_str = json_str.split('}')[0] + '}'
        return json.loads(clean_json_str)
    except (json.JSONDecodeError, TypeError):
        return {}

# Parse telemetry column
telemetry_dicts = df['Telemetry'].apply(safe_json_loads)
telemetry_df = pd.json_normalize(telemetry_dicts)

# Combine telemetry DataFrame with original DataFrame
result_df = pd.concat([df, telemetry_df], axis=1)

# Drop original telemetry column
result_df = result_df.drop('Telemetry', axis=1)

# Select the desired columns from the DataFrame
df_new = result_df[['Timestamp', 'DeviceId', 'EnclosureTemp', 'Noise', 'Solar_Battery_Voltage', 'Dust_2_5', 'Dust_10']]

df = pd.DataFrame(df_new)
df = df.loc[:, ~df.columns.duplicated()]
mc5n_details = df[df['DeviceId'] == 'MC-5N']
selected_features = ['EnclosureTemp', 'Noise', 'Solar_Battery_Voltage', 'Dust_2_5', 'Dust_10']
data_selected = mc5n_details[selected_features]
def increase_normal_and_anomalous_data_points(existing_data):
    existing_data = existing_data.copy()  
    mean_val = existing_data.mean() 
    std_val = existing_data.std() 

    num_anomalous_points = int(existing_data.isnull().sum() * 0.4)

    anomalous_indices = existing_data.isnull().sample(num_anomalous_points).index
    
    large_number = 1000
  
    existing_data.loc[anomalous_indices] = large_number  
    
    ratio_replaced_anomalous = len(anomalous_indices) / existing_data.isnull().sum() 
    
    new_data_points = np.random.normal(mean_val, std_val, size=existing_data.isnull().sum())
    
    existing_data.loc[existing_data.isnull()] = new_data_points


    print(ratio_replaced_anomalous)
    return existing_data
    

selected_features = ['EnclosureTemp', 'Noise', 'Solar_Battery_Voltage', 'Dust_2_5', 'Dust_10']
for feature in selected_features:
    data_selected[feature] = increase_normal_and_anomalous_data_points(data_selected[feature])

# Function to convert data to sequences
def to_sequences(x_values, seq_size=1):
    X = []
    y = []

    for i in range(len(x_values) - seq_size):
        X.append(x_values[i:(i + seq_size)])
        y.append(x_values[i + seq_size])

    return np.array(X), np.array(y)

def anomaly_detection_model(input_shape):
    
    model = Sequential([
    LSTM(units=256, input_shape=input_shape, return_sequences=True),
    Dropout(rate=0.1),
    LSTM(units=128, return_sequences=True),
    LSTM(units=64, return_sequences=False),
    RepeatVector(n=input_shape[0]), 
    LSTM(units=64, return_sequences=True),
    Dropout(rate=0.1),
    LSTM(units=128, return_sequences=True),
    LSTM(units=256, return_sequences=True),

    TimeDistributed(Dense(units=input_shape[1])) 
])
    model.compile(loss='mae', optimizer='adam')
    model.summary()
    return model

# Train anomaly detection models for each feature
anomaly_detection_models = {}
scalers = {}
thresholds = {}

for feature in telemetry_columns:
    # Prepare data for the current feature
    feature_data = data_selected[feature].values.reshape(-1, 1)
    scaler = StandardScaler()
    feature_data_normalized = scaler.fit_transform(feature_data)
    X, y = to_sequences(feature_data_normalized,seq_size=30)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    
    # Train the model
    model = anomaly_detection_model(input_shape=(X_train.shape[1], X_train.shape[2]))
    early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)
    history = model.fit(X_train, y_train, epochs=100, batch_size=32, validation_split=0.1, verbose=1, callbacks=[early_stopping])
    
    # Determine threshold for anomalies (e.g., 90th percentile of reconstruction errors)
    reconstruction_errors = np.mean(np.abs(model.predict(X) - y[:, :, np.newaxis]), axis=(1, 2))
    threshold = np.percentile(reconstruction_errors, 60)
    
    
    # Save model, scaler, and threshold
    anomaly_detection_models[feature] = model
    scalers[feature] = scaler
    thresholds[feature] = threshold

# Detect anomalies and calculate metrics
results = {}
for feature in telemetry_columns:
    feature_data = data_selected[feature].values.reshape(-1, 1)
    feature_data_normalized = scalers[feature].transform(feature_data)
    X = to_sequences(feature_data_normalized ,seq_size=30)
    reconstruction_errors = np.mean(np.abs(anomaly_detection_models[feature].predict(X) - y[:, :, np.newaxis]), axis=(1, 2))

    anomalies = reconstruction_errors > thresholds[feature]
    
    # Calculate metrics
    
    true_anomalies = data_selected[feature].iloc[:-30].values == 1000
    accuracy = accuracy_score(true_anomalies, anomalies)
    precision = precision_score(true_anomalies, anomalies)
    recall = recall_score(true_anomalies, anomalies)
    f1 = f1_score(true_anomalies, anomalies)
    
    results[feature] = {
        'Accuracy': accuracy,
         'Precision': precision,
        'Recall': recall,
        'F1 Score': f1
    }


for feature, metrics in results.items():
    print(f"Metrics for {feature}:")
    for metric, value in metrics.items():
        print(f"{metric}: {value:.4f}")
    print()

# Overall metrics
overall_accuracy = np.mean([metrics['Accuracy'] for metrics in results.values()])
overall_precision = np.mean([metrics['Precision'] for metrics in results.values()])
overall_recall = np.mean([metrics['Recall'] for metrics in results.values()])
overall_f1 = np.mean([metrics['F1 Score'] for metrics in results.values()])

print("Overall Metrics:")
print(f"Overall Accuracy: {overall_accuracy:.4f}")
print(f"Overall Precision: {overall_precision:.4f}")
print(f"Overall Recall: {overall_recall:.4f}")
print(f"Overall F1 Score: {overall_f1:.4f}")

This is the last result: Metrics for EnclosureTemp: Accuracy: 0.9039 Precision: 0.0374 Recall: 0.0374 F1 Score: 0.0374

Metrics for Noise: Accuracy: 0.8456 Precision: 0.0623 Recall: 0.1509 F1 Score: 0.0882

Metrics for Solar_Battery_Voltage: Accuracy: 0.8993 Precision: 0.0000 Recall: 0.0000 F1 Score: 0.0000

Metrics for Dust_2_5: Accuracy: 0.9035 Precision: 0.0374 Recall: 0.0370 F1 Score: 0.0372

Metrics for Dust_10: Accuracy: 0.6726 Precision: 0.0373 Recall: 0.2243 F1 Score: 0.0640

Overall Metrics: Overall Accuracy: 0.8450 Overall Precision: 0.0349 Overall Recall: 0.0899 Overall F1 Score: 0.0453