Step_Data_Project_India/old/final-32-automated-code-new(1).ipynb at f82c609a0df8f92f698cde61d38b552551175886

bs/Step_Data_Project_India

Fork 0

Files

T

bs 0f123471f2 Restructuring

2026-07-02 14:17:22 +02:00

37 KiB

Raw Blame History

In [4]:

# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session

In [6]:

import numpy as np
import pandas as pd
import shutil
import os
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout,GRU,Bidirectional
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
import keras_tuner as kt
from keras_tuner import RandomSearch
from sklearn.metrics import accuracy_score

# === Clean previous tuning directory ===
shutil.rmtree("/kaggle/working/my_dir", ignore_errors=True)

# === Load dataset ===
file_path = '/kaggle/input/32usrs/ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx' 

df = pd.read_excel(file_path)

[0;31m---------------------------------------------------------------------------[0m
[0;31mModuleNotFoundError[0m                       Traceback (most recent call last)
Cell [0;32mIn[6], line 11[0m
[1;32m      9[0m [38;5;28;01mimport[39;00m[38;5;250m [39m[38;5;21;01mkeras_tuner[39;00m[38;5;250m [39m[38;5;28;01mas[39;00m[38;5;250m [39m[38;5;21;01mkt[39;00m
[1;32m     10[0m [38;5;28;01mfrom[39;00m[38;5;250m [39m[38;5;21;01mkeras_tuner[39;00m[38;5;250m [39m[38;5;28;01mimport[39;00m RandomSearch
[0;32m---> 11[0m [38;5;28;01mfrom[39;00m[38;5;250m [39m[38;5;21;01msklearn[39;00m[38;5;21;01m.[39;00m[38;5;21;01mmetrics[39;00m[38;5;250m [39m[38;5;28;01mimport[39;00m accuracy_score
[1;32m     13[0m [38;5;66;03m# === Clean previous tuning directory ===[39;00m
[1;32m     14[0m shutil[38;5;241m.[39mrmtree([38;5;124m"[39m[38;5;124m/kaggle/working/my_dir[39m[38;5;124m"[39m, ignore_errors[38;5;241m=[39m[38;5;28;01mTrue[39;00m)

[0;31mModuleNotFoundError[0m: No module named 'sklearn'

In [ ]:

# === Helper functions for scenario selection ===
def get_user_input_for_scenario(scenario_type):
    print(f"\nPlease define your custom {scenario_type} scenario:")
    years_input = input(f"Enter {scenario_type} years (comma-separated, e.g., 2017,2018): ").strip()
    years = list(map(int, years_input.split(',')))
    years_months = []
    for year in years:
        months_input = input(f"Enter months for year {year} (comma-separated, e.g., 1,2,3): ").strip()
        months = list(map(int, months_input.split(',')))
        years_months.append((year, months))
    return years_months

def display_warning_about_2020_data():
    print("\n⚠️ Warning: 2020 data after February is excluded due to COVID-19.")
    print("✅ Only Jan and Feb 2020 are used for testing. Do not use them in training/validation.")

def display_warnings_for_scenarios(scenario_type):
    if scenario_type == "training":
        print("\n⚠️ Predefined Training Scenarios (for reference only):")
        for name, scenario in predefined_training_scenarios.items():
            parts = [f"{year}-{months}" for year, months in scenario['years_months']]
            print(f"  {name}: {', '.join(parts)}")
    elif scenario_type == "validation":
        print("\n⚠️ Predefined Validation Scenario:")
        for name, scenario in predefined_validation_scenarios.items():
            parts = [f"{year}-{months}" for year, months in scenario['years_months']]
            print(f"  {name}: {', '.join(parts)}")
        print("  - This uses Oct, Nov, Dec of 2019")

predefined_training_scenarios = {
    "Scenario 1": {"years_months": [(2018, list(range(1, 13))), (2019, list(range(1, 10)))]},
    "Scenario 2": {"years_months": [(2017, list(range(1, 13))), (2018, list(range(1, 13))), (2019, list(range(1, 10)))]}
}
predefined_validation_scenarios = {
    "Scenario A": {"years_months": [(2019, [10, 11, 12])]}
}

In [ ]:

# === Get user-defined training and validation scenarios ===
print("=== Training Scenario Setup ===")
display_warning_about_2020_data()
display_warnings_for_scenarios("training")
training_scenario = get_user_input_for_scenario("training")

print("\n=== Validation Scenario Setup ===")
display_warning_about_2020_data()
display_warnings_for_scenarios("validation")
validation_scenario = get_user_input_for_scenario("validation")

# === Filter and preprocess data ===
def filter_data(df, scenario):
    filtered = pd.DataFrame()
    for year, months in scenario:
        filtered = pd.concat([filtered, df[(df['Year'] == year) & (df['Month'].isin(months))]])
    return filtered.drop(columns=['Month', 'Year', 'date', 'DayOfWeek']) 

data = filter_data(df, training_scenario)
data_val = filter_data(df, validation_scenario)

In [ ]:



# === Organize by user ===
df_sorted = data.sort_values(by='user').reset_index(drop=True)
df_sorted_val = data_val.sort_values(by='user').reset_index(drop=True)
users = df_sorted['user'].unique()
users_val = df_sorted_val['user'].unique()

user_data = {user: df_sorted[df_sorted['user'] == user] for user in users}
user_data_val = {user: df_sorted_val[df_sorted_val['user'] == user] for user in users_val}

# === Callbacks ===
early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)
lr_scheduler = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, verbose=1)

In [ ]:

# === Model tuning and training loop ===
best_models = {}

for sequence_length in range(20, 30, 5):
    print(f"\n=== Training for Sequence Length: {sequence_length} ===")

    # Training data
    X, y = [], []
    for user, data in user_data.items():
        features = data.drop('user', axis=1).values
        labels = data['user'].values
        for i in range(len(features) - sequence_length):
            X.append(features[i:i + sequence_length])
            y.append(labels[i + sequence_length])
    X = np.array(X)
    y = np.array(y)

    # Validation data
    X_val, y_val = [], []
    for user, data in user_data_val.items():
        features = data.drop('user', axis=1).values
        labels = data['user'].values
        for i in range(len(features) - sequence_length):
            X_val.append(features[i:i + sequence_length])
            y_val.append(labels[i + sequence_length])
    X_val = np.array(X_val)
    y_val = np.array(y_val)

    if X.shape[0] == 0 or X_val.shape[0] == 0:
        print(f"⚠️ Skipped sequence length {sequence_length} due to insufficient data.")
        continue

    n_features = X.shape[2]

    def build_model(hp):
        model = Sequential()
        model.add(Bidirectional(LSTM(units=hp.Int('units', 32, 256, step=2),
                                     input_shape=(sequence_length, n_features))))
        model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.5, step=0.1)))
        model.add(Dense(len(users), activation='softmax'))
        model.compile(
            optimizer=Adam(learning_rate=hp.Choice('learning_rate', [1e-2, 1e-3, 1e-4])),
            loss='sparse_categorical_crossentropy',
            metrics=['accuracy']
        )
        return model

    tuner = RandomSearch(
        build_model,
        objective='val_loss',
        max_trials=30,
        executions_per_trial=2,
        directory='/kaggle/working/my_dir',
        project_name=f'lstm_seq_{sequence_length}'
    )

    tuner.search(X, y, epochs=30, validation_data=(X_val, y_val),
                 callbacks=[early_stopping, lr_scheduler], verbose=1)

    best_hps = tuner.get_best_hyperparameters(1)[0]
    best_model = tuner.hypermodel.build(best_hps)
    best_model.fit(X, y, epochs=30, validation_data=(X_val, y_val),
                   callbacks=[early_stopping, lr_scheduler], verbose=0)

    best_models[sequence_length] = {
        'model': best_model,
        'best_hyperparameters': {
            'units': best_hps.get('units'),
            'dropout_rate': best_hps.get('dropout_rate'),
            'learning_rate': best_hps.get('learning_rate')
        }
    }

In [ ]:



# === Get test scenario input ===
def get_user_input_for_test():
    print("\n=== Testing Scenario Setup ===")
    print("⚠️ Only January and February of 2020 were used for testing in predefined setup.")
    print("⚠️ Avoid using 2020 data after February due to COVID-19 impact.\n")
    years_input = input("Enter test years (comma-separated, e.g., 2020): ").strip()
    years = list(map(int, years_input.split(',')))
    years_months = []
    for year in years:
        months_input = input(f"Enter months for year {year} (comma-separated, e.g., 1,2): ").strip()
        months = list(map(int, months_input.split(',')))
        years_months.append((year, months))
    return years_months

def filter_test_data(df, scenario):
    data_parts = []
    for year, months in scenario:
        part = df[(df['Year'] == year) & (df['Month'].isin(months))]
        data_parts.append(part)
    return pd.concat(data_parts, ignore_index=True)

In [3]:

import pandas as pd
import os

def evaluate_model_on_test_data(model, test_df, sequence_length, excel_writer):
    print("\n🧪 Evaluating on Test Data...")
    test_df = test_df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'])
    test_df = test_df.sort_values(by='user').reset_index(drop=True)

    users = test_df['user'].unique()
    results = []
    accuracy_above_50 = 0

    for user in users:
        user_df = test_df[test_df['user'] == user]
        X, y_true = [], []
        user_features = user_df.drop(columns=['user']).values
        user_labels = user_df['user'].values

        if len(user_df) <= sequence_length:
            print(f"Skipping User {user} (not enough data for sequence length {sequence_length})")
            continue

        for i in range(len(user_df) - sequence_length):
            seq_x = user_features[i:i + sequence_length]
            seq_y = user_labels[i + sequence_length]
            X.append(seq_x)
            y_true.append(seq_y)

        X = np.array(X)
        y_true = np.array(y_true)

        if len(X) == 0:
            continue

        y_pred = model.predict(X, verbose=0)
        y_pred_classes = np.argmax(y_pred, axis=1)

        unique_pred, counts_pred = np.unique(y_pred_classes, return_counts=True)
        label_counts_pred = dict(zip(unique_pred, counts_pred))

        unique_true, counts_true = np.unique(y_true, return_counts=True)
        label_counts_true = dict(zip(unique_true, counts_true))

        acc = accuracy_score(y_true, y_pred_classes)
        if acc > 0.5:
            accuracy_above_50 += 1

        # Append result to list
        results.append({
            'User': user,
            'Accuracy (%)': acc * 100,
            'Predicted Class Distribution': str(label_counts_pred),
            'Actual Class Distribution': str(label_counts_true)
        })

        print(f"\n=== User {user} ===")
        print(f"✅ Accuracy: {acc * 100:.2f}%")
        print("📊 Predicted Class Distribution:", label_counts_pred)
        print("📌 Actual Class Distribution:   ", label_counts_true)

    final_accuracy_percent = (accuracy_above_50 / 32) * 100
    print(f"\n🟩 Final Evaluation Summary for Sequence Length {sequence_length}:")
    print(f"Users with >50% Accuracy: {accuracy_above_50} / 32")
    print(f"✅ Final Success Rate: {final_accuracy_percent:.2f}%")

    # Append overall stats as a new row
    results.append({
        'User': 'TOTAL',
        'Accuracy (%)': '',
        'Predicted Class Distribution': f'Users >50% Acc: {accuracy_above_50}/32',
        'Actual Class Distribution': f'Success Rate: {final_accuracy_percent:.2f}%'
    })

    # Save results to Excel sheet
    df_results = pd.DataFrame(results)
    df_results.to_excel(excel_writer, sheet_name=f"SeqLen_{sequence_length}", index=False)

In [6]:

from pandas import ExcelWriter

# === Run evaluation for each trained sequence length ===
test_scenario = get_user_input_for_test()
test_data = filter_test_data(df, test_scenario)

output_excel_path = "/kaggle/working/evaluation_results.xlsx"

with ExcelWriter(output_excel_path) as writer:
    for sequence_length, result in best_models.items():
        print(f"\n🔍 Testing Model for Sequence Length: {sequence_length}")
        evaluate_model_on_test_data(
            result['model'],
            test_data.copy(),
            sequence_length,
            writer  # 👈 pass the writer
        )

print(f"\n✅ All evaluations completed. Results saved to: {output_excel_path}")

=== Testing Scenario Setup ===
⚠️ Only January and February of 2020 were used for testing in predefined setup.
⚠️ Avoid using 2020 data after February due to COVID-19 impact.

Enter test years (comma-separated, e.g., 2020):  2020
Enter months for year 2020 (comma-separated, e.g., 1,2):  1,2

🔍 Testing Model for Sequence Length: 20

🧪 Evaluating on Test Data...

=== User 0 ===
✅ Accuracy: 47.50%
📊 Predicted Class Distribution: {0: 19, 18: 9, 24: 7, 26: 1, 30: 3, 31: 1}
📌 Actual Class Distribution:    {0: 40}

=== User 1 ===
✅ Accuracy: 82.50%
📊 Predicted Class Distribution: {1: 33, 31: 7}
📌 Actual Class Distribution:    {1: 40}

=== User 2 ===
✅ Accuracy: 0.00%
📊 Predicted Class Distribution: {6: 2, 12: 12, 17: 13, 30: 12, 31: 1}
📌 Actual Class Distribution:    {2: 40}

=== User 3 ===
✅ Accuracy: 41.03%
📊 Predicted Class Distribution: {3: 16, 6: 1, 12: 8, 29: 13, 30: 1}
📌 Actual Class Distribution:    {3: 39}

=== User 4 ===
✅ Accuracy: 2.50%
📊 Predicted Class Distribution: {2: 1, 4: 1, 8: 2, 9: 3, 18: 11, 23: 3, 26: 16, 29: 1, 30: 1, 31: 1}
📌 Actual Class Distribution:    {4: 40}

=== User 5 ===
✅ Accuracy: 57.50%
📊 Predicted Class Distribution: {2: 5, 5: 23, 23: 2, 29: 6, 30: 3, 31: 1}
📌 Actual Class Distribution:    {5: 40}

=== User 6 ===
✅ Accuracy: 25.00%
📊 Predicted Class Distribution: {6: 10, 17: 1, 30: 5, 31: 24}
📌 Actual Class Distribution:    {6: 40}

=== User 7 ===
✅ Accuracy: 52.50%
📊 Predicted Class Distribution: {7: 21, 10: 3, 11: 14, 18: 2}
📌 Actual Class Distribution:    {7: 40}

=== User 8 ===
✅ Accuracy: 62.50%
📊 Predicted Class Distribution: {8: 25, 23: 1, 29: 8, 30: 6}
📌 Actual Class Distribution:    {8: 40}

=== User 9 ===
✅ Accuracy: 100.00%
📊 Predicted Class Distribution: {9: 40}
📌 Actual Class Distribution:    {9: 40}

=== User 10 ===
✅ Accuracy: 57.50%
📊 Predicted Class Distribution: {10: 23, 11: 15, 30: 2}
📌 Actual Class Distribution:    {10: 40}

=== User 11 ===
✅ Accuracy: 35.00%
📊 Predicted Class Distribution: {1: 1, 10: 15, 11: 14, 12: 1, 14: 4, 15: 2, 16: 2, 25: 1}
📌 Actual Class Distribution:    {11: 40}

=== User 12 ===
✅ Accuracy: 62.50%
📊 Predicted Class Distribution: {3: 1, 12: 25, 26: 14}
📌 Actual Class Distribution:    {12: 40}

=== User 13 ===
✅ Accuracy: 55.00%
📊 Predicted Class Distribution: {10: 3, 11: 3, 12: 2, 13: 22, 16: 1, 21: 9}
📌 Actual Class Distribution:    {13: 40}

=== User 14 ===
✅ Accuracy: 70.00%
📊 Predicted Class Distribution: {0: 1, 14: 28, 16: 2, 18: 7, 25: 2}
📌 Actual Class Distribution:    {14: 40}

=== User 15 ===
✅ Accuracy: 100.00%
📊 Predicted Class Distribution: {15: 40}
📌 Actual Class Distribution:    {15: 40}

=== User 16 ===
✅ Accuracy: 17.50%
📊 Predicted Class Distribution: {15: 20, 16: 7, 18: 13}
📌 Actual Class Distribution:    {16: 40}

=== User 17 ===
✅ Accuracy: 40.00%
📊 Predicted Class Distribution: {0: 2, 16: 6, 17: 16, 18: 1, 28: 1, 31: 14}
📌 Actual Class Distribution:    {17: 40}

=== User 18 ===
✅ Accuracy: 97.50%
📊 Predicted Class Distribution: {0: 1, 18: 39}
📌 Actual Class Distribution:    {18: 40}

=== User 19 ===
✅ Accuracy: 72.50%
📊 Predicted Class Distribution: {1: 3, 6: 7, 19: 29, 22: 1}
📌 Actual Class Distribution:    {19: 40}

=== User 20 ===
✅ Accuracy: 77.50%
📊 Predicted Class Distribution: {2: 8, 20: 31, 26: 1}
📌 Actual Class Distribution:    {20: 40}

=== User 21 ===
✅ Accuracy: 92.50%
📊 Predicted Class Distribution: {21: 37, 24: 3}
📌 Actual Class Distribution:    {21: 40}

=== User 22 ===
✅ Accuracy: 0.00%
📊 Predicted Class Distribution: {8: 4, 9: 2, 23: 1, 29: 27, 30: 1}
📌 Actual Class Distribution:    {22: 35}

=== User 23 ===
✅ Accuracy: 77.50%
📊 Predicted Class Distribution: {3: 9, 23: 31}
📌 Actual Class Distribution:    {23: 40}

=== User 24 ===
✅ Accuracy: 92.50%
📊 Predicted Class Distribution: {21: 3, 24: 37}
📌 Actual Class Distribution:    {24: 40}

=== User 25 ===
✅ Accuracy: 2.50%
📊 Predicted Class Distribution: {2: 14, 12: 11, 23: 1, 25: 1, 29: 4, 30: 9}
📌 Actual Class Distribution:    {25: 40}

=== User 26 ===
✅ Accuracy: 0.00%
📊 Predicted Class Distribution: {12: 18, 18: 3, 21: 13, 24: 6}
📌 Actual Class Distribution:    {26: 40}

=== User 27 ===
✅ Accuracy: 0.00%
📊 Predicted Class Distribution: {12: 38, 21: 1, 24: 1}
📌 Actual Class Distribution:    {27: 40}

=== User 28 ===
✅ Accuracy: 100.00%
📊 Predicted Class Distribution: {28: 40}
📌 Actual Class Distribution:    {28: 40}

=== User 29 ===
✅ Accuracy: 40.00%
📊 Predicted Class Distribution: {12: 12, 26: 1, 29: 16, 30: 11}
📌 Actual Class Distribution:    {29: 40}

=== User 30 ===
✅ Accuracy: 35.00%
📊 Predicted Class Distribution: {12: 1, 18: 9, 23: 5, 25: 3, 26: 3, 29: 2, 30: 14, 31: 3}
📌 Actual Class Distribution:    {30: 40}

=== User 31 ===
✅ Accuracy: 50.00%
📊 Predicted Class Distribution: {12: 2, 18: 18, 31: 20}
📌 Actual Class Distribution:    {31: 40}

🟩 Final Evaluation Summary for Sequence Length 20:
Users with >50% Accuracy: 17 / 32
✅ Final Success Rate: 53.12%

🔍 Testing Model for Sequence Length: 25

🧪 Evaluating on Test Data...

=== User 0 ===
✅ Accuracy: 17.14%
📊 Predicted Class Distribution: {0: 6, 18: 2, 24: 3, 25: 2, 26: 14, 30: 7, 31: 1}
📌 Actual Class Distribution:    {0: 35}

=== User 1 ===
✅ Accuracy: 8.57%
📊 Predicted Class Distribution: {1: 3, 31: 32}
📌 Actual Class Distribution:    {1: 35}

=== User 2 ===
✅ Accuracy: 5.71%
📊 Predicted Class Distribution: {2: 2, 12: 5, 17: 11, 21: 1, 30: 3, 31: 13}
📌 Actual Class Distribution:    {2: 35}

=== User 3 ===
✅ Accuracy: 14.71%
📊 Predicted Class Distribution: {3: 5, 12: 1, 29: 5, 30: 16, 31: 7}
📌 Actual Class Distribution:    {3: 34}

=== User 4 ===
✅ Accuracy: 0.00%
📊 Predicted Class Distribution: {2: 4, 9: 4, 10: 1, 25: 7, 26: 5, 27: 1, 30: 12, 31: 1}
📌 Actual Class Distribution:    {4: 35}

=== User 5 ===
✅ Accuracy: 100.00%
📊 Predicted Class Distribution: {5: 35}
📌 Actual Class Distribution:    {5: 35}

=== User 6 ===
✅ Accuracy: 31.43%
📊 Predicted Class Distribution: {6: 11, 31: 24}
📌 Actual Class Distribution:    {6: 35}

=== User 7 ===
✅ Accuracy: 65.71%
📊 Predicted Class Distribution: {7: 23, 10: 3, 13: 9}
📌 Actual Class Distribution:    {7: 35}

=== User 8 ===
✅ Accuracy: 82.86%
📊 Predicted Class Distribution: {4: 2, 8: 29, 22: 2, 30: 2}
📌 Actual Class Distribution:    {8: 35}

=== User 9 ===
✅ Accuracy: 97.14%
📊 Predicted Class Distribution: {4: 1, 9: 34}
📌 Actual Class Distribution:    {9: 35}

=== User 10 ===
✅ Accuracy: 40.00%
📊 Predicted Class Distribution: {10: 14, 13: 6, 23: 3, 25: 2, 30: 10}
📌 Actual Class Distribution:    {10: 35}

=== User 11 ===
✅ Accuracy: 31.43%
📊 Predicted Class Distribution: {10: 22, 11: 11, 12: 1, 19: 1}
📌 Actual Class Distribution:    {11: 35}

=== User 12 ===
✅ Accuracy: 57.14%
📊 Predicted Class Distribution: {12: 20, 29: 15}
📌 Actual Class Distribution:    {12: 35}

=== User 13 ===
✅ Accuracy: 57.14%
📊 Predicted Class Distribution: {12: 1, 13: 20, 21: 14}
📌 Actual Class Distribution:    {13: 35}

=== User 14 ===
✅ Accuracy: 62.86%
📊 Predicted Class Distribution: {0: 4, 14: 22, 15: 2, 18: 7}
📌 Actual Class Distribution:    {14: 35}

=== User 15 ===
✅ Accuracy: 100.00%
📊 Predicted Class Distribution: {15: 35}
📌 Actual Class Distribution:    {15: 35}

=== User 16 ===
✅ Accuracy: 40.00%
📊 Predicted Class Distribution: {7: 2, 15: 13, 16: 14, 18: 6}
📌 Actual Class Distribution:    {16: 35}

=== User 17 ===
✅ Accuracy: 65.71%
📊 Predicted Class Distribution: {0: 1, 16: 11, 17: 23}
📌 Actual Class Distribution:    {17: 35}

=== User 18 ===
✅ Accuracy: 82.86%
📊 Predicted Class Distribution: {0: 6, 18: 29}
📌 Actual Class Distribution:    {18: 35}

=== User 19 ===
✅ Accuracy: 60.00%
📊 Predicted Class Distribution: {6: 13, 19: 21, 22: 1}
📌 Actual Class Distribution:    {19: 35}

=== User 20 ===
✅ Accuracy: 5.71%
📊 Predicted Class Distribution: {2: 33, 20: 2}
📌 Actual Class Distribution:    {20: 35}

=== User 21 ===
✅ Accuracy: 100.00%
📊 Predicted Class Distribution: {21: 35}
📌 Actual Class Distribution:    {21: 35}

=== User 22 ===
✅ Accuracy: 0.00%
📊 Predicted Class Distribution: {8: 2, 9: 2, 29: 26}
📌 Actual Class Distribution:    {22: 30}

=== User 23 ===
✅ Accuracy: 65.71%
📊 Predicted Class Distribution: {3: 4, 23: 23, 30: 8}
📌 Actual Class Distribution:    {23: 35}

=== User 24 ===
✅ Accuracy: 100.00%
📊 Predicted Class Distribution: {24: 35}
📌 Actual Class Distribution:    {24: 35}

=== User 25 ===
✅ Accuracy: 0.00%
📊 Predicted Class Distribution: {2: 33, 12: 1, 30: 1}
📌 Actual Class Distribution:    {25: 35}

=== User 26 ===
✅ Accuracy: 0.00%
📊 Predicted Class Distribution: {12: 29, 21: 6}
📌 Actual Class Distribution:    {26: 35}

=== User 27 ===
✅ Accuracy: 0.00%
📊 Predicted Class Distribution: {12: 35}
📌 Actual Class Distribution:    {27: 35}

=== User 28 ===
✅ Accuracy: 100.00%
📊 Predicted Class Distribution: {28: 35}
📌 Actual Class Distribution:    {28: 35}

=== User 29 ===
✅ Accuracy: 28.57%
📊 Predicted Class Distribution: {2: 1, 12: 2, 26: 8, 29: 10, 30: 14}
📌 Actual Class Distribution:    {29: 35}

=== User 30 ===
✅ Accuracy: 34.29%
📊 Predicted Class Distribution: {2: 4, 26: 2, 27: 4, 29: 13, 30: 12}
📌 Actual Class Distribution:    {30: 35}

=== User 31 ===
✅ Accuracy: 60.00%
📊 Predicted Class Distribution: {12: 1, 16: 1, 18: 12, 31: 21}
📌 Actual Class Distribution:    {31: 35}

🟩 Final Evaluation Summary for Sequence Length 25:
Users with >50% Accuracy: 16 / 32
✅ Final Success Rate: 50.00%

✅ All evaluations completed. Results saved to: /kaggle/working/evaluation_results.xlsx

In [ ]:


# # === Evaluation function (your version) ===
# def evaluate_model_on_test_data(model, test_df, sequence_length):
#     print("\n🧪 Evaluating on Test Data...")
#     test_df = test_df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'])
#     test_df = test_df.sort_values(by='user').reset_index(drop=True)

#     users = test_df['user'].unique()
#     results = {}
#     accuracy_above_50 = 0

#     for user in users:
#         user_df = test_df[test_df['user'] == user]
#         X, y_true = [], []
#         user_features = user_df.drop(columns=['user']).values
#         user_labels = user_df['user'].values

#         if len(user_df) <= sequence_length:
#             print(f"Skipping User {user} (not enough data for sequence length {sequence_length})")
#             continue

#         for i in range(len(user_df) - sequence_length):
#             seq_x = user_features[i:i + sequence_length]
#             seq_y = user_labels[i + sequence_length]
#             X.append(seq_x)
#             y_true.append(seq_y)

#         X = np.array(X)
#         y_true = np.array(y_true)

#         if len(X) == 0:
#             continue

#         y_pred = model.predict(X, verbose=0)
#         y_pred_classes = np.argmax(y_pred, axis=1)

#         unique_pred, counts_pred = np.unique(y_pred_classes, return_counts=True)
#         label_counts_pred = dict(zip(unique_pred, counts_pred))

#         unique_true, counts_true = np.unique(y_true, return_counts=True)
#         label_counts_true = dict(zip(unique_true, counts_true))

#         acc = accuracy_score(y_true, y_pred_classes)
#         if acc > 0.5:
#             accuracy_above_50 += 1

#         results[user] = {
#             'accuracy': acc,
#             'predicted_counts': label_counts_pred,
#             'actual_counts': label_counts_true
#         }

#         print(f"\n=== User {user} ===")
#         print(f"✅ Accuracy: {acc * 100:.2f}%")
#         print("📊 Predicted Class Distribution:", label_counts_pred)
#         print("📌 Actual Class Distribution:   ", label_counts_true)

#     final_accuracy_percent = (accuracy_above_50 / 32) * 100
#     print(f"\n🟩 Final Evaluation Summary for Sequence Length {sequence_length}:")
#     print(f"Users with >50% Accuracy: {accuracy_above_50} / 32")
#     print(f"✅ Final Success Rate: {final_accuracy_percent:.2f}%")

# # === Run evaluation for each trained sequence length ===
# test_scenario = get_user_input_for_test()
# test_data = filter_test_data(df, test_scenario)

# for sequence_length, result in best_models.items():
#     print(f"\n🔍 Testing Model for Sequence Length: {sequence_length}")
#     evaluate_model_on_test_data(result['model'], test_data.copy(), sequence_length)

# print("\n✅ All evaluations completed.")

37 KiB Raw Blame History

37 KiB

Raw Blame History