Added Baselines + extended manual evaluation

main.py

@@ -4,7 +4,10 @@ import os
 import numpy as np
 import pandas as pd
 import sklearn
+from keras.src.regularizers import L1L2
 from matplotlib import pyplot as plt
+from pandas import DataFrame
+from sklearn.dummy import DummyClassifier
 
 from pipeline import (
     load_dataset,
@@ -13,7 +16,8 @@ from pipeline import (
     prepare_user_data,
     train_models,
     evaluate_models,
-    prepare_data_for_model, model_type_gru, model_type_lstm, model_type_bilstm, train_models_v2, train_one_model
+    prepare_data_for_model, model_type_gru, model_type_lstm, model_type_bilstm, train_models_v2, train_one_model,
+    eval_metrics
 )
 
 year_str = 'Year'
@@ -29,6 +33,7 @@ timespan_str = 'time used'
 hour_timespan_str = '1HR'
 min_timespan_str = '15MIN'
 sequence_length_str = 'sequence length'
+accuracy_str = 'accuracy'
 precision_str = 'precision'
 recall_str = 'recall'
 f1_string = 'f1 score'
@@ -332,7 +337,7 @@ def manual_tuning(model_type):
     df = load_dataset(file_path)
     df = remove_covid_data(df)
 
-    tr, val, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=20)
+    tr, val, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=100)
     tr = reduce_columns(tr, data_filename)
     val = reduce_columns(val, data_filename)
     te = reduce_columns(te, data_filename)
@@ -342,15 +347,18 @@ def manual_tuning(model_type):
 
     # fit and evaluate model
     # config
-    repeats = 5
-    n_batch = 4
+    repeats = 3
+    n_batch = 1024
     n_epochs = 500
-    n_neurons = 1
+    n_neurons = 16
+    l_rate = 1e-4
+    reg = L1L2(l1=0.0, l2=0.0)
 
     history_list = list()
     # run diagnostic tests
     for i in range(repeats):
-        history = train_one_model(user_data_train, user_data_val, n_batch, n_epochs, n_neurons,
+        history = train_one_model(user_data_train, user_data_val, n_batch, n_epochs,
+                                  n_neurons, l_rate, reg,
                                      sequence_length=sequence_length,
                                      model_type=model_type)
         history_list.append(history)
@@ -358,11 +366,45 @@ def manual_tuning(model_type):
         for history in history_list:
             plt.plot(history['train_'+metric], color='blue')
             plt.plot(history['test_'+metric], color='orange')
-        plt.savefig(figure_path+metric+'_epochs_diagnostic.png')
+        plt.savefig(figure_path+metric+'_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+
+                    str(n_batch)+'_l'+str(l_rate)+'_diagnostic.png')
         plt.clf()
     print('Done')
 
 
+
+def calculate_baselines():
+    file_combinations = [(hour_timespan_str, with_threshold_str,'ALL32USERS1HR_WITHTHRESHOLD.xlsx'),
+                         (min_timespan_str, with_threshold_str, 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx'),
+                         (min_timespan_str, without_threshold_str, 'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx'),
+                         (hour_timespan_str, without_threshold_str, 'ALLUSERS_32_1HR_WITHOUT_THRESHOLD.xlsx'),
+                         ]
+    baseline_res = pd.DataFrame()
+    for timespan_id, threshold_id, filename in file_combinations:
+        file_path = os.path.join(dataset_path, filename)
+        df = load_dataset(file_path)
+        df = remove_covid_data(df)
+
+        _, _, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=20)
+        te = reduce_columns(te, filename)
+        user_data_te = prepare_user_data(te)
+        for sequence_length in range(5,30, 5):
+            x, y = prepare_data_for_model(user_data=user_data_te, sequence_length=sequence_length)
+
+            for strategy in ['most_frequent', 'stratified', 'uniform']:
+                cls = DummyClassifier(strategy=strategy)
+                cls.fit(x,y)
+                y_pred = cls.predict(x)
+                acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred)
+                baseline_res = pd.concat([baseline_res,
+                                          DataFrame({ 'strategy':[strategy], threshold_str:[threshold_id],
+                                                      timespan_str:[timespan_id], sequence_length_str:[sequence_length],
+                                              accuracy_str:[acc],precision_str:[p],recall_str:[r],
+                                                     f1_string:f1})], ignore_index=True)
+    baseline_res.to_json('baseline_results.json')
+    print('Done')
+
+
 if __name__ == "__main__":
     # main_two_v1()
     # visualise_results_v1()
@@ -370,4 +412,5 @@ if __name__ == "__main__":
    # main_two_v2(model_type=model_type_gru)
     #visualise_results_v2()
     manual_tuning(model_type=model_type_lstm)
+    #calculate_baselines()
     print('Done')