Restructuring
This commit is contained in:
@@ -1,29 +1,23 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
import math
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import sklearn
|
||||
from keras.src.regularizers import L1L2
|
||||
from matplotlib import pyplot as plt
|
||||
from pandas import DataFrame
|
||||
from sklearn.dummy import DummyClassifier
|
||||
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier
|
||||
from sklearn.model_selection import GridSearchCV
|
||||
from sklearn.naive_bayes import BernoulliNB
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
|
||||
from pipeline import (
|
||||
load_dataset,
|
||||
filter_data,
|
||||
filter_test_data,
|
||||
prepare_user_data,
|
||||
train_models,
|
||||
evaluate_models,
|
||||
prepare_data_for_model, model_type_gru, model_type_lstm, model_type_bilstm, train_models_v2, train_one_model,
|
||||
eval_metrics, get_save_id
|
||||
prepare_data_for_neural_model, model_type_gru, model_type_lstm, model_type_bilstm,
|
||||
eval_metrics, prepare_data_for_basic_algorithm, train_one_model,
|
||||
)
|
||||
|
||||
year_str = 'Year'
|
||||
month_str = 'Month'
|
||||
day_str = 'Day'
|
||||
date_str = 'Date'
|
||||
time_str = 'Time'
|
||||
day_of_week_str = 'DayOfWeek'
|
||||
@@ -31,9 +25,6 @@ user_str = 'user'
|
||||
split_str = 'split type'
|
||||
data_split_str = 'data percentages'
|
||||
month_split_str = 'month percentages'
|
||||
threshold_str = 'threshold used'
|
||||
with_threshold_str = 'WITH'
|
||||
without_threshold_str = 'WITHOUT'
|
||||
timespan_str = 'time used'
|
||||
hour_timespan_str = '1HR'
|
||||
min_timespan_str = '15MIN'
|
||||
@@ -46,30 +37,11 @@ model_type_str = 'model type'
|
||||
week_column_names = ['DayOfWeek_' + day for day in
|
||||
['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday' ]]
|
||||
figure_path = 'figures/'
|
||||
predicitons_path = 'preds/'
|
||||
|
||||
# === Configurable Parameters ===
|
||||
dataset_path = './Datasets/'
|
||||
dataset_hrs_path = './Datasets/hours.json'
|
||||
dataset_min_path = './Datasets/minutes.json'
|
||||
DATA_PATH = dataset_path +'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx'
|
||||
OUTPUT_EXCEL_PATH = './working/evaluation_results.xlsx'
|
||||
result_filename_v1 = './working/evaluation_results.json'
|
||||
result_filename_v2 = './working/evaluation_results_v2.json'
|
||||
SEQUENCE_LENGTHS = [30, 25, 20, 15, 10, 5] # You can add more: [20, 25, 30]
|
||||
|
||||
TRAINING_SCENARIO = [(2018, list(range(1, 13))), (2019, list(range(1, 10)))]
|
||||
VALIDATION_SCENARIO = [(2019, [10, 11, 12])]
|
||||
TEST_SCENARIO = [(2020, [1, 2])] # Jan–Feb 2020 only
|
||||
|
||||
# === Optional display only ===
|
||||
predefined_training_scenarios = {
|
||||
"Scenario 1": {"years_months": [(2018, list(range(1, 13))), (2019, list(range(1, 10)))]},
|
||||
"Scenario 2": {"years_months": [(2017, list(range(1, 13))), (2018, list(range(1, 13))), (2019, list(range(1, 10)))]}
|
||||
}
|
||||
predefined_validation_scenarios = {
|
||||
"Scenario A": {"years_months": [(2019, [10, 11, 12])]}
|
||||
}
|
||||
|
||||
def create_dir(path):
|
||||
"""
|
||||
@@ -81,424 +53,74 @@ def create_dir(path):
|
||||
os.makedirs(path)
|
||||
|
||||
def remove_covid_data(df):
|
||||
"""
|
||||
Removes covid data from dataframe because the steps from these times will most likely differ from before
|
||||
:param df: Dataframe with the data
|
||||
:return: the data without covid time data
|
||||
"""
|
||||
df = df[~(df[year_str]>=2020)]
|
||||
return df
|
||||
|
||||
def split_data_by_month_percentage(df, percentages):
|
||||
train_p, valid_p, test_p = percentages
|
||||
ids = df[[year_str, month_str]].drop_duplicates().sort_values([year_str, month_str])
|
||||
tr, va, te = np.split(ids, [int((train_p/100) * len(ids)), int(((train_p + valid_p)/100) * len(ids))])
|
||||
return df.merge(tr, on=[year_str, month_str], how='inner'), df.merge(va, on=[year_str, month_str], how='inner'), df.merge(te, on=[year_str, month_str], how='inner')
|
||||
|
||||
def split_data_by_userdata_percentage(df, percentages, sample=100):
|
||||
"""
|
||||
Splits data by userdata percentages. Each users data will be split according to the given percentages along the time axis
|
||||
|
||||
:param df: Data with all users data
|
||||
:param percentages: triple with percentages
|
||||
:param sample: overall percentage if less data should be used. Use only for testing. Has an error!!
|
||||
:return: the split data
|
||||
"""
|
||||
train_p, valid_p, test_p = percentages
|
||||
tr, va, te = pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
|
||||
for user_id in df[user_str].unique():
|
||||
user_data = df[df[user_str]==user_id].sample(frac=sample/ 100).sort_values([year_str, month_str])
|
||||
# !! following sample creates gaps in data if sample smaller 100
|
||||
user_data = df[df[user_str]==user_id].sample(frac=sample/ 100).sort_values([date_str]) # have to sort for time shift
|
||||
u_tr, u_va, u_te = np.split(user_data, [int((train_p/100)*len(user_data)), int(((train_p+valid_p)/100)*len(user_data))])
|
||||
tr = pd.concat([tr, u_tr], ignore_index=True)
|
||||
va = pd.concat([va, u_va], ignore_index=True)
|
||||
te = pd.concat([te, u_te], ignore_index=True)
|
||||
return tr, va, te
|
||||
|
||||
|
||||
def main():
|
||||
# print("=== Training Scenario Setup ===")
|
||||
# display_warning_about_2020_data()
|
||||
# display_warnings_for_scenarios("training", predefined_training_scenarios, predefined_validation_scenarios)
|
||||
|
||||
# print("\n=== Validation Scenario Setup ===")
|
||||
# display_warning_about_2020_data()
|
||||
# display_warnings_for_scenarios("validation", predefined_training_scenarios, predefined_validation_scenarios)
|
||||
|
||||
# === Load and preprocess ===
|
||||
df = load_dataset(DATA_PATH)
|
||||
|
||||
ALLUSERS32_15MIN_WITHOUTTHREHOLD = False
|
||||
if('ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx' in DATA_PATH):
|
||||
ALLUSERS32_15MIN_WITHOUTTHREHOLD = True
|
||||
|
||||
training_data = filter_data(df, TRAINING_SCENARIO, ALLUSERS32_15MIN_WITHOUTTHREHOLD)
|
||||
validation_data = filter_data(df, VALIDATION_SCENARIO, ALLUSERS32_15MIN_WITHOUTTHREHOLD)
|
||||
|
||||
user_data_train = prepare_user_data(training_data)
|
||||
user_data_val = prepare_user_data(validation_data)
|
||||
|
||||
# === Train models ===
|
||||
best_models = train_models(user_data_train, user_data_val, sequence_lengths=SEQUENCE_LENGTHS)
|
||||
|
||||
# === Load and evaluate test ===
|
||||
test_df = filter_test_data(df, TEST_SCENARIO)
|
||||
evaluate_models(best_models, test_df, SEQUENCE_LENGTHS, OUTPUT_EXCEL_PATH, ALLUSERS32_15MIN_WITHOUTTHREHOLD)
|
||||
|
||||
print(f"\n✅ All evaluations completed. Results saved to: {OUTPUT_EXCEL_PATH}")
|
||||
|
||||
|
||||
def reduce_columns(df, filename):
|
||||
if min_timespan_str in filename:
|
||||
return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'] + week_column_names, errors='ignore')
|
||||
else:
|
||||
return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'], errors='ignore')
|
||||
|
||||
|
||||
def reduce_columns_v3(df):
|
||||
def reduce_columns(df):
|
||||
"""
|
||||
Removes unnecessary columns from dataframe.
|
||||
:param df: Dataframe with the data
|
||||
:return: dataframe without unnecessary columns
|
||||
"""
|
||||
return df.drop(columns=[month_str, year_str, date_str])
|
||||
|
||||
def load_previous_results(filename):
|
||||
results = pd.DataFrame()
|
||||
if os.path.exists(filename):
|
||||
results = pd.DataFrame(json.load(open(filename)))
|
||||
return results
|
||||
def filter_and_preprocess_data(filename, sample=100, print_unique=False):
|
||||
"""
|
||||
Preprocesses data. Removes users with too little data or which subsume another, bins the step data and creates train, valid, test splits
|
||||
|
||||
def main_two_v2(model_type):
|
||||
seq_length = range(10,31, 5)
|
||||
for sequence_length in seq_length:
|
||||
for data_filename in os.listdir(dataset_path):
|
||||
timespan_id = hour_timespan_str
|
||||
threshold_id = with_threshold_str
|
||||
if min_timespan_str in data_filename:
|
||||
timespan_id = min_timespan_str
|
||||
if without_threshold_str in data_filename:
|
||||
threshold_id = without_threshold_str
|
||||
|
||||
results = load_previous_results(result_filename_v2)
|
||||
if len(results) > 0:
|
||||
if len(results[(results[timespan_str]==timespan_id) &
|
||||
(results[threshold_str]==threshold_id) &
|
||||
(results[sequence_length_str]==sequence_length) &
|
||||
(results[model_type_str]==model_type)]) > 0:
|
||||
continue
|
||||
|
||||
file_path = os.path.join(dataset_path, data_filename)
|
||||
df = load_dataset(file_path)
|
||||
df = remove_covid_data(df)
|
||||
|
||||
tr,val,te = split_data_by_userdata_percentage(df, percentages=(80,10,10))
|
||||
tr = reduce_columns(tr, data_filename)
|
||||
val = reduce_columns(val, data_filename)
|
||||
te = reduce_columns(te, data_filename)
|
||||
|
||||
user_data_train = prepare_user_data(tr)
|
||||
user_data_val = prepare_user_data(val)
|
||||
|
||||
best_model = train_models_v2(user_data_train, user_data_val,
|
||||
sequence_length=sequence_length,
|
||||
model_type=model_type)
|
||||
|
||||
results = load_previous_results(result_filename_v2)
|
||||
results = pd.concat([results,
|
||||
evaluate_model_on_test_data(model=best_model,
|
||||
test_df=te,
|
||||
sequence_length=sequence_length,
|
||||
time_span_id=timespan_id,
|
||||
threshold_id=threshold_id,
|
||||
model_type=model_type,
|
||||
split_id=data_split_str)],
|
||||
ignore_index=True)
|
||||
results.to_json(result_filename_v2)
|
||||
|
||||
def main_two_v1():
|
||||
seq_length = [30, 25, 20, 15, 10, 5] # You can add more: [20, 25, 30]
|
||||
results = pd.DataFrame()
|
||||
if os.path.exists(result_filename_v1):
|
||||
results = pd.DataFrame(json.load(open(result_filename_v1)))
|
||||
for sequence_length in seq_length:
|
||||
for data_filename in os.listdir(dataset_path):
|
||||
for split_id, split_method in [(data_split_str, split_data_by_userdata_percentage),(month_split_str, split_data_by_month_percentage)]:
|
||||
for model_type in [model_type_lstm, model_type_bilstm, model_type_gru]:
|
||||
timespan_id = hour_timespan_str
|
||||
threshold_id = with_threshold_str
|
||||
if min_timespan_str in data_filename:
|
||||
timespan_id = min_timespan_str
|
||||
if without_threshold_str in data_filename:
|
||||
threshold_id = without_threshold_str
|
||||
if len(results) > 0:
|
||||
if len(results[(results[split_str]==split_id) &
|
||||
(results[timespan_str]==timespan_id) &
|
||||
(results[threshold_str]==threshold_id) &
|
||||
(results[sequence_length_str]==sequence_length) &
|
||||
(results[model_type_str]==model_type)]) > 0:
|
||||
continue
|
||||
|
||||
file_path = os.path.join(dataset_path, data_filename)
|
||||
df = load_dataset(file_path)
|
||||
df = remove_covid_data(df)
|
||||
tr,val,te = split_method(df, percentages=(80,10,10))
|
||||
tr = reduce_columns(tr, data_filename)
|
||||
val = reduce_columns(val, data_filename)
|
||||
te = reduce_columns(te, data_filename)
|
||||
|
||||
user_data_train = prepare_user_data(tr)
|
||||
user_data_val = prepare_user_data(val)
|
||||
|
||||
best_models = train_models(user_data_train, user_data_val, sequence_lengths=[sequence_length], model_type=model_type)
|
||||
|
||||
results = pd.concat([results,
|
||||
evaluate_model_on_test_data(model=best_models[sequence_length]['model'],
|
||||
test_df=te, split_id=split_id,
|
||||
sequence_length=sequence_length,
|
||||
time_span_id=timespan_id,
|
||||
threshold_id=threshold_id,
|
||||
model_type=model_type)], ignore_index=True)
|
||||
results.to_json(result_filename_v1)
|
||||
|
||||
# === Evaluation ===
|
||||
def evaluate_model_on_test_data(model, test_df,sequence_length, split_id, threshold_id, time_span_id, model_type):
|
||||
user_data = prepare_user_data(test_df)
|
||||
x, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length)
|
||||
|
||||
y_pred = model.predict(x, verbose=0)
|
||||
y_pred_classes = np.argmax(y_pred, axis=1)
|
||||
|
||||
recall = sklearn.metrics.recall_score(y, y_pred_classes, average='weighted')
|
||||
precision = sklearn.metrics.precision_score(y, y_pred_classes, average='weighted')
|
||||
f1_score = sklearn.metrics.f1_score(y, y_pred_classes, average='weighted')
|
||||
return pd.DataFrame({split_str:[split_id], threshold_str:[threshold_id], timespan_str:[time_span_id],
|
||||
sequence_length_str:[sequence_length],
|
||||
model_type_str:[model_type], recall_str:[recall],
|
||||
precision_str:[precision], f1_string:[f1_score]})
|
||||
|
||||
def visualise_results_v1():
|
||||
results = pd.DataFrame(json.load(open(result_filename_v1)))
|
||||
# Month split ist immer schlechter
|
||||
results = results[results[split_str] == data_split_str]
|
||||
with_threshold = results[results[threshold_str] == with_threshold_str]
|
||||
without_threshold = results[results[threshold_str] == without_threshold_str]
|
||||
fig, axes = plt.subplots(2, 3)
|
||||
ax_col_id = 0
|
||||
ax_row_id = -1
|
||||
for timespan in [hour_timespan_str,min_timespan_str]:
|
||||
ax_row_id +=1
|
||||
for model in [model_type_lstm, model_type_bilstm, model_type_gru]:
|
||||
with_sub = with_threshold[(with_threshold[timespan_str] == timespan) & (with_threshold[model_type_str] == model)]
|
||||
without_sub = without_threshold[(without_threshold[timespan_str] == timespan) & (without_threshold[model_type_str] == model)]
|
||||
ax = axes[ax_row_id, ax_col_id]
|
||||
ax.set_title(model+' '+timespan)
|
||||
ax.plot(with_sub[sequence_length_str], with_sub[f1_string], label=with_threshold_str)
|
||||
ax.plot(without_sub[sequence_length_str], without_sub[f1_string], label=without_threshold_str)
|
||||
ax.legend()
|
||||
ax_col_id +=1
|
||||
ax_col_id %= 3
|
||||
fig.tight_layout()
|
||||
fig.savefig(figure_path+'v1_results.svg')
|
||||
# Fazit: keine eindeutig besseren Versionen erkennbar
|
||||
|
||||
|
||||
def visualise_results_v2():
|
||||
results = pd.DataFrame(json.load(open(result_filename_v2)))
|
||||
with_threshold = results[results[threshold_str] == with_threshold_str]
|
||||
without_threshold = results[results[threshold_str] == without_threshold_str]
|
||||
fig, axes = plt.subplots(2, 3)
|
||||
ax_col_id = 0
|
||||
ax_row_id = -1
|
||||
for timespan in [hour_timespan_str,min_timespan_str]:
|
||||
ax_row_id +=1
|
||||
for model in [model_type_lstm, model_type_bilstm, model_type_gru]:
|
||||
with_sub = with_threshold[(with_threshold[timespan_str] == timespan) & (with_threshold[model_type_str] == model)]
|
||||
without_sub = without_threshold[(without_threshold[timespan_str] == timespan) & (without_threshold[model_type_str] == model)]
|
||||
with_sub = with_sub.sort_values(sequence_length_str)
|
||||
without_sub = without_sub.sort_values(sequence_length_str)
|
||||
ax = axes[ax_row_id, ax_col_id]
|
||||
ax.set_title(model+' '+timespan)
|
||||
ax.plot(with_sub[sequence_length_str], with_sub[f1_string], label=with_threshold_str)
|
||||
ax.plot(without_sub[sequence_length_str], without_sub[f1_string], label=without_threshold_str)
|
||||
ax.legend()
|
||||
ax_col_id +=1
|
||||
ax_col_id %= 3
|
||||
fig.tight_layout()
|
||||
fig.savefig(figure_path+'v2_results.svg')
|
||||
# Fazit: keine eindeutig besseren Versionen erkennbar
|
||||
|
||||
|
||||
def test(model_type):
|
||||
sequence_length = 20
|
||||
data_filename = os.listdir(dataset_path)[0]
|
||||
timespan_id = hour_timespan_str
|
||||
threshold_id = with_threshold_str
|
||||
|
||||
file_path = os.path.join(dataset_path, data_filename)
|
||||
df = load_dataset(file_path)
|
||||
df = remove_covid_data(df)
|
||||
results = pd.DataFrame()
|
||||
|
||||
for percentage in [33,66,100]:
|
||||
print('Percentage:', percentage)
|
||||
tr,val,te = split_data_by_userdata_percentage(df, percentages=(80,10,10),sample=percentage)
|
||||
tr = reduce_columns(tr, data_filename)
|
||||
val = reduce_columns(val, data_filename)
|
||||
te = reduce_columns(te, data_filename)
|
||||
|
||||
user_data_train = prepare_user_data(tr)
|
||||
user_data_val = prepare_user_data(val)
|
||||
|
||||
best_model = train_models_v2(user_data_train, user_data_val,
|
||||
sequence_length=sequence_length,
|
||||
model_type=model_type)
|
||||
|
||||
results = pd.concat([results,
|
||||
evaluate_model_on_test_data(model=best_model,
|
||||
test_df=te,
|
||||
sequence_length=sequence_length,
|
||||
time_span_id=timespan_id,
|
||||
threshold_id=threshold_id,
|
||||
model_type=model_type,
|
||||
split_id=data_split_str)],
|
||||
ignore_index=True)
|
||||
print(results)
|
||||
|
||||
def manual_tuning(model_type):
|
||||
# load dataset
|
||||
sequence_length = 20
|
||||
data_filename = 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx'
|
||||
timespan_id = min_timespan_str
|
||||
threshold_id = with_threshold_str
|
||||
|
||||
file_path = os.path.join(dataset_path, data_filename)
|
||||
df = load_dataset(file_path)
|
||||
df = remove_covid_data(df)
|
||||
|
||||
tr, val, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=100)
|
||||
tr = reduce_columns(tr, data_filename)
|
||||
val = reduce_columns(val, data_filename)
|
||||
te = reduce_columns(te, data_filename)
|
||||
|
||||
user_data_train = prepare_user_data(tr)
|
||||
user_data_val = prepare_user_data(val)
|
||||
|
||||
# fit and evaluate model
|
||||
# config
|
||||
repeats = 3
|
||||
n_batch = 1024
|
||||
n_epochs = 500
|
||||
n_neurons = 16
|
||||
l_rate = 1e-4
|
||||
reg = L1L2(l1=0.0, l2=0.0)
|
||||
|
||||
history_list = list()
|
||||
# run diagnostic tests
|
||||
for i in range(repeats):
|
||||
history = train_one_model(user_data_train, user_data_val, n_batch, n_epochs,
|
||||
n_neurons, l_rate, reg,
|
||||
sequence_length=sequence_length,
|
||||
model_type=model_type)
|
||||
history_list.append(history)
|
||||
for metric in ['p', 'r', 'f1']:
|
||||
for history in history_list:
|
||||
plt.plot(history['train_'+metric], color='blue')
|
||||
plt.plot(history['test_'+metric], color='orange')
|
||||
plt.savefig(figure_path+metric+'_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+
|
||||
str(n_batch)+'_l'+str(l_rate)+'_diagnostic.png')
|
||||
plt.clf()
|
||||
print('Done')
|
||||
|
||||
|
||||
def upsampling(df):
|
||||
max_user_data = df[user_str].value_counts().max()
|
||||
for user in df[user_str].unique():
|
||||
user_data = df[df[user_str]==user]
|
||||
user_count = user_data.shape[0]
|
||||
times = max_user_data / user_count
|
||||
before_comma = math.floor(times)
|
||||
after_comma = times % 1
|
||||
after_comma_data = user_data.sample(frac=after_comma)
|
||||
for i in range(1, before_comma):
|
||||
df = pd.concat([df, user_data], ignore_index=True)
|
||||
df = pd.concat([df, after_comma_data], ignore_index=True)
|
||||
return df
|
||||
|
||||
|
||||
def manual_tuning_v3(model_type):
|
||||
# TODO: hrs/min
|
||||
sequence_length = 1
|
||||
|
||||
tr, val, te = get_prepared_data_v3(dataset_hrs_path)
|
||||
|
||||
# fit and evaluate model
|
||||
# config
|
||||
repeats = 3
|
||||
n_batch = 1024
|
||||
n_epochs = 10
|
||||
n_neurons = 256
|
||||
n_neurons2 = 512
|
||||
n_neurons3 = 512
|
||||
n_neurons4 = 128
|
||||
l_rate = 1e-2
|
||||
d1 = 256
|
||||
reg1 = L1L2(l1=0.0, l2=0.001)
|
||||
r1 = '0001'
|
||||
reg2 = L1L2(l1=0.0, l2=0.1)
|
||||
r2 = '01'
|
||||
|
||||
history_list = list()
|
||||
# run diagnostic tests
|
||||
for i in range(repeats):
|
||||
history = train_one_model(tr, val, n_batch, n_epochs,
|
||||
n_neurons,n_neurons2, n_neurons3, n_neurons4, l_rate, d1, r1, reg1, r2, reg2,
|
||||
sequence_length=sequence_length,
|
||||
model_type=model_type)
|
||||
history_list.append(history)
|
||||
for metric in ['acc', 'p', 'r', 'f1']:
|
||||
for history in history_list:
|
||||
plt.plot(history['train_'+metric], color='blue')
|
||||
plt.plot(history['test_'+metric], color='orange')
|
||||
plt.savefig(figure_path+'v3/'+metric+get_save_id(n_epochs, n_neurons, n_neurons2, n_neurons3,n_neurons4, n_batch, l_rate, d1, r1, r2)
|
||||
+'.png')
|
||||
plt.clf()
|
||||
print('Done')
|
||||
|
||||
|
||||
|
||||
def calculate_baselines():
|
||||
file_combinations = [(hour_timespan_str, with_threshold_str,'ALL32USERS1HR_WITHTHRESHOLD.xlsx'),
|
||||
(min_timespan_str, with_threshold_str, 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx'),
|
||||
(min_timespan_str, without_threshold_str, 'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx'),
|
||||
(hour_timespan_str, without_threshold_str, 'ALLUSERS_32_1HR_WITHOUT_THRESHOLD.xlsx'),
|
||||
]
|
||||
baseline_res = pd.DataFrame()
|
||||
for timespan_id, threshold_id, filename in file_combinations:
|
||||
file_path = os.path.join(dataset_path, filename)
|
||||
df = load_dataset(file_path)
|
||||
df = remove_covid_data(df)
|
||||
|
||||
_, _, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=20)
|
||||
te = reduce_columns(te, filename)
|
||||
user_data_te = prepare_user_data(te)
|
||||
for sequence_length in range(5,30, 5):
|
||||
x, y = prepare_data_for_model(user_data=user_data_te, sequence_length=sequence_length)
|
||||
|
||||
for strategy in ['most_frequent', 'stratified', 'uniform']:
|
||||
cls = DummyClassifier(strategy=strategy)
|
||||
cls.fit(x,y)
|
||||
y_pred = cls.predict(x)
|
||||
acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred)
|
||||
baseline_res = pd.concat([baseline_res,
|
||||
DataFrame({ 'strategy':[strategy], threshold_str:[threshold_id],
|
||||
timespan_str:[timespan_id], sequence_length_str:[sequence_length],
|
||||
accuracy_str:[acc],precision_str:[p],recall_str:[r],
|
||||
f1_string:f1})], ignore_index=True)
|
||||
baseline_res.to_json('baseline_results.json')
|
||||
print('Done')
|
||||
|
||||
def get_prepared_data_v3(filename, sample=100):
|
||||
:param filename: Name of the file for loading
|
||||
:param sample: percentage of the sample in case less data is wanted (e.g., for testing)
|
||||
:param print_unique: To print the number of unique users
|
||||
:return: train, valid, test splits as dicts from user_id to dataframe
|
||||
"""
|
||||
df = pd.read_json(filename)
|
||||
df = remove_covid_data(df)
|
||||
|
||||
# remove users with too little data (optional)
|
||||
value_counts = df[user_str].value_counts()
|
||||
# df = df[df[user_str].isin(value_counts[value_counts>1000].index)]
|
||||
# remove users which are a complete subset of another user (but keep one)
|
||||
users_to_remove = []
|
||||
for user_a in df[user_str].unique():
|
||||
for user_b in df[user_str].unique():
|
||||
if user_a != user_b:
|
||||
data = pd.concat([df[df[user_str]==user_a], df[df[user_str]==user_b]])
|
||||
columns = data.columns.tolist()
|
||||
columns.remove(user_str)
|
||||
|
||||
adjusted_df = pd.DataFrame()
|
||||
# adjust labels
|
||||
new_id = 0
|
||||
for user_id in df[user_str].unique():
|
||||
user_data = df[df[user_str]==user_id]
|
||||
user_data[user_str] = new_id
|
||||
adjusted_df = pd.concat([adjusted_df, user_data], ignore_index=True)
|
||||
new_id += 1
|
||||
no_dup = data.drop_duplicates(columns, keep=False)
|
||||
if len(no_dup[no_dup[user_str]==user_a]) == 0:
|
||||
if print_unique:
|
||||
print(user_a, 'is subset of',user_b)
|
||||
if user_b not in users_to_remove:
|
||||
users_to_remove.append(user_a)
|
||||
df = df[~df[user_str].isin(users_to_remove)]
|
||||
|
||||
# bin steps per hour TODO: adjust for minutes
|
||||
for hour in ['Hour_'+str(i) for i in range(24)]:
|
||||
hour_data = adjusted_df[hour]
|
||||
hour_data = df[hour]
|
||||
# smaller 1000 - round to 10
|
||||
a = ((hour_data[hour_data<1000]/10).round()*10)
|
||||
# between 1000 and 10000 - round to next 100
|
||||
@@ -507,66 +129,306 @@ def get_prepared_data_v3(filename, sample=100):
|
||||
c = hour_data[hour_data > 10000]
|
||||
c = pd.Series(data={ind:10000 for ind in c.index}, index=c.index)
|
||||
new = pd.concat([a, b, c]).sort_index().astype(int)
|
||||
adjusted_df[hour] = new
|
||||
df[hour] = new
|
||||
|
||||
tr, val, te = split_data_by_userdata_percentage(adjusted_df, percentages=(70, 15, 15), sample=sample)
|
||||
tr = reduce_columns_v3(tr)
|
||||
val = reduce_columns_v3(val)
|
||||
te = reduce_columns_v3(te)
|
||||
# remove users with too little data
|
||||
min_datapoints = 500 # 500 leads to at least 75 datapoints in the valid set
|
||||
users_to_remove = set()
|
||||
cols = df.columns.tolist()
|
||||
cols.remove(user_str)
|
||||
reduced = df.drop_duplicates(subset=cols, keep=False)
|
||||
for user_id in df[user_str].unique():
|
||||
subset = df[df[user_str] == user_id]
|
||||
reduced_subset = reduced[reduced[user_str] == user_id]
|
||||
if print_unique:
|
||||
print(user_id, len(subset), len(reduced_subset))
|
||||
if len(reduced_subset) < min_datapoints:
|
||||
users_to_remove.add(user_id)
|
||||
if print_unique:
|
||||
print('removing', user_id)
|
||||
df = df[~df[user_str].isin(users_to_remove)]
|
||||
|
||||
tr, val, te = split_data_by_userdata_percentage(df, percentages=(70, 15, 15), sample=sample)
|
||||
tr = reduce_columns(tr)
|
||||
val = reduce_columns(val)
|
||||
te = reduce_columns(te)
|
||||
|
||||
if print_unique:
|
||||
print('Train: Users', len(tr[user_str].unique()), 'mean num datapoins:', tr[user_str].value_counts().mean())
|
||||
print('Valid: Users', len(val[user_str].unique()), 'mean num datapoins:', val[user_str].value_counts().mean())
|
||||
print('Test: Users', len(te[user_str].unique()), 'mean num datapoins:', te[user_str].value_counts().mean())
|
||||
|
||||
tr, val, te = add_features(tr), add_features(val), add_features(te)
|
||||
|
||||
scaler = MinMaxScaler()
|
||||
scaler.fit(tr.drop(columns=[user_str]))
|
||||
|
||||
return scale_dataset(scaler, tr), scale_dataset(scaler, val), scale_dataset(scaler, te)
|
||||
|
||||
|
||||
def scale_dataset(scaler, df):
|
||||
"""
|
||||
Data scaling function
|
||||
:param scaler: The scaler object
|
||||
:param df: data to scale
|
||||
:return: the scaled data
|
||||
"""
|
||||
y = df[user_str]
|
||||
x_scaled = scaler.transform(df.drop(columns=[user_str]))
|
||||
x_scaled = pd.DataFrame(x_scaled)
|
||||
x_scaled.columns = df.drop(columns=[user_str]).columns
|
||||
|
||||
df_scaled = pd.concat([pd.DataFrame(x_scaled), pd.DataFrame(y)], axis=1)
|
||||
df_scaled.columns = df.columns
|
||||
return prepare_user_data(df_scaled)
|
||||
df_scaled = pd.concat([x_scaled, pd.DataFrame(y.reset_index()[user_str])], axis=1)
|
||||
return convert_to_user_dict(df_scaled)
|
||||
|
||||
def convert_to_user_dict(df):
|
||||
"""
|
||||
Converts the dataframe to a dict of dataframes with the key the user id
|
||||
|
||||
def calculate_baselines_v3():
|
||||
:param df: Complete dataframe
|
||||
:return: the dict of dataframes
|
||||
"""
|
||||
users = df[user_str].unique()
|
||||
return {user: df[df[user_str] == user] for user in users}
|
||||
|
||||
def calculate_baselines():
|
||||
"""
|
||||
Calculates very simple baselines for the scenario and saves them
|
||||
:return the calculated baselines
|
||||
"""
|
||||
file_combinations = [(hour_timespan_str, dataset_hrs_path),
|
||||
# (min_timespan_str, dataset_min_path), # TODO: dataset bining not ready for minutes
|
||||
# (min_timespan_str, dataset_min_path), # TODO: dataset binning not ready for minutes yet, rerun this method when that works
|
||||
]
|
||||
baseline_res = pd.DataFrame()
|
||||
for timespan_id, filename in file_combinations:
|
||||
_, _, te = get_prepared_data_v3(filename)
|
||||
for sequence_length in range(1,30,5):
|
||||
x, y = prepare_data_for_model(user_data=te, sequence_length=sequence_length)
|
||||
str_result_filename = 'baseline_results.json'
|
||||
if os.path.exists(str_result_filename):
|
||||
baseline_res = pd.read_json(str_result_filename)
|
||||
else:
|
||||
baseline_res = pd.DataFrame()
|
||||
|
||||
for strategy in ['most_frequent', 'stratified', 'uniform']:
|
||||
cls = DummyClassifier(strategy=strategy)
|
||||
cls.fit(x,y)
|
||||
y_pred = cls.predict(x)
|
||||
acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred)
|
||||
baseline_res = pd.concat([baseline_res,
|
||||
DataFrame({ 'strategy':[strategy],
|
||||
timespan_str:[timespan_id], sequence_length_str:[sequence_length],
|
||||
accuracy_str:[acc],precision_str:[p],recall_str:[r],
|
||||
f1_string:f1})], ignore_index=True)
|
||||
baseline_res.to_json('baseline_results_v3.json')
|
||||
for timespan_id, filename in file_combinations:
|
||||
_, _, te = filter_and_preprocess_data(filename)
|
||||
for sequence_length in range(1,30,5):
|
||||
x, y = prepare_data_for_neural_model(user_data=te, sequence_length=sequence_length)
|
||||
|
||||
for strategy in ['most_frequent', 'stratified', 'uniform']:
|
||||
cls = DummyClassifier(strategy=strategy)
|
||||
cls.fit(x,y)
|
||||
y_pred = cls.predict(x)
|
||||
acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred)
|
||||
baseline_res = pd.concat([baseline_res,
|
||||
DataFrame({ 'strategy':[strategy],
|
||||
timespan_str:[timespan_id], sequence_length_str:[sequence_length],
|
||||
accuracy_str:[acc],precision_str:[p],recall_str:[r],
|
||||
f1_string:f1})], ignore_index=True)
|
||||
baseline_res.to_json(str_result_filename)
|
||||
return baseline_res
|
||||
|
||||
def hypertune_basic_algorithms():
|
||||
"""
|
||||
Function can be used to hypertune basic sklearn algorithms. Takes very long.
|
||||
"""
|
||||
# TODO: mnake it run for minutes, iterate over sequence lengths
|
||||
sequence_length = 7
|
||||
|
||||
tr, val, te = filter_and_preprocess_data(dataset_hrs_path)
|
||||
|
||||
x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length)
|
||||
x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length)
|
||||
|
||||
random_state = 17
|
||||
results = pd.DataFrame()
|
||||
for tag, clf, grid in [
|
||||
('GradientBoosting', GradientBoostingClassifier(random_state=random_state),
|
||||
{'loss': ['log_loss', 'exponential'],
|
||||
'learning_rate': [0.1, 0.5, 1.0,2.0, 5.0],
|
||||
'n_estimators': [10, 50, 100, 150, 200],
|
||||
'subsample': [0.1, 0.5, 1.0],
|
||||
'criterion': ['friedman_mse', 'squared_error'],
|
||||
'min_samples_split': [2, 10, 100],
|
||||
'min_samples_leaf': [1, 5, 10],
|
||||
'min_weight_fraction_leaf': [0.0, 0.1, 0.5],
|
||||
'max_depth': [None, 2, 10, 100],
|
||||
'min_impurity_decrease': [0.0, 0.1, 0.5],
|
||||
'max_features': ['sqrt', 'log2', None, 10, 20],
|
||||
'max_leaf_nodes': [None, 1, 5, 10],
|
||||
}),
|
||||
('Bernoulli', BernoulliNB(), {'fit_prior': [True, False],
|
||||
'binarize': [0.0, 0.1, 0.25, 0.5, 0.75],
|
||||
'force_alpha': [True, False],
|
||||
'alpha':[0.0, 0.25, 0.5, 0.75, 1.0]}),
|
||||
('extra trees', ExtraTreesClassifier(random_state=random_state, n_jobs=1),
|
||||
{'n_estimators': [10, 50, 100, 150, 200],
|
||||
'criterion': ['gini', 'entropy', 'log_loss'],
|
||||
'max_depth': [None, 2, 10, 100],
|
||||
'min_samples_split': [2, 10, 100],
|
||||
'min_samples_leaf': [1, 5, 10],
|
||||
'min_weight_fraction_leaf': [0.0, 0.1, 0.5],
|
||||
'max_features': ['sqrt', 'log2', None, 10, 20],
|
||||
'max_leaf_nodes': [None, 1, 5, 10],
|
||||
'min_impurity_decrease': [0.0, 0.1, 0.5],
|
||||
'bootstrap': [True, False],
|
||||
'class_weight': [None, 'balanced', 'balanced_subsample'],
|
||||
'max_samples': [None, 0.1, 0.2, 0.3]}
|
||||
),
|
||||
('random forest', RandomForestClassifier(random_state=random_state, n_jobs=1),
|
||||
{'n_estimators':[10, 50, 100, 150, 200],
|
||||
'criterion':['gini', 'entropy', 'log_loss'],
|
||||
'max_depth':[None, 2, 10,100],
|
||||
'min_samples_split': [2,10,100],
|
||||
'min_samples_leaf':[1,5,10],
|
||||
'min_weight_fraction_leaf':[0.0,0.1, 0.5],
|
||||
'max_features':['sqrt', 'log2', None, 10, 20],
|
||||
'max_leaf_nodes':[None, 1, 5, 10],
|
||||
'min_impurity_decrease':[0.0, 0.1, 0.5],
|
||||
'bootstrap':[True, False],
|
||||
'class_weight':[None, 'balanced', 'balanced_subsample'],
|
||||
'max_samples':[None,0.1, 0.2, 0.3]})
|
||||
]:
|
||||
grid_search = GridSearchCV(
|
||||
estimator=clf, param_grid=grid, scoring='f1_weighted', cv=5, n_jobs=1)
|
||||
grid_search.fit(x_tr, y_tr)
|
||||
|
||||
best_model = grid_search.best_estimator_
|
||||
y_pred = best_model.predict(x_val)
|
||||
acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred)
|
||||
results = pd.concat([results, DataFrame({ 'params': str(grid_search.best_params_),
|
||||
'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True)
|
||||
results.to_json('basic_ht_results.json')
|
||||
print('Done')
|
||||
|
||||
def test_basic_algorithms():
|
||||
"""
|
||||
Method for testing basic algorithms from the sklearn library. Tested more, but those 4 were the best
|
||||
:return: The calculated values
|
||||
"""
|
||||
# TODO: also check for minutes
|
||||
# TODO: iterate over sequence lengths
|
||||
sequence_length = 21
|
||||
|
||||
tr, val, te = filter_and_preprocess_data(dataset_hrs_path)
|
||||
|
||||
x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length)
|
||||
x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length)
|
||||
|
||||
random_state = 17
|
||||
results = pd.DataFrame()
|
||||
for tag, clf in [
|
||||
('Bernoulli', BernoulliNB()),
|
||||
('GradientBoosting', GradientBoostingClassifier(random_state=random_state)),
|
||||
('extra trees', ExtraTreesClassifier(random_state=random_state)),
|
||||
('random forest', RandomForestClassifier(random_state=random_state))
|
||||
]:
|
||||
clf.fit(x_tr, y_tr)
|
||||
y_pred = clf.predict(x_val)
|
||||
acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred)
|
||||
results = pd.concat([results, DataFrame({ 'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True)
|
||||
return results
|
||||
|
||||
def add_features(df):
|
||||
"""
|
||||
Functiion for adding additional features to the dataframe
|
||||
|
||||
:param df: dataframe
|
||||
:return: dataframe with features added
|
||||
"""
|
||||
# indicator weekend
|
||||
df['weekend'] = df[day_of_week_str + '_Saturday']+df[day_of_week_str + '_Sunday']
|
||||
# sum of steps per day
|
||||
df['day_total'] = sum([df['Hour_'+str(i)] for i in range(23)])
|
||||
# sum of steps morning, afternoon, evening, night
|
||||
df['morning_total'] = sum([df['Hour_' + str(i)] for i in range(6,13)])
|
||||
df['afternoon_total'] = sum([df['Hour_' + str(i)] for i in range(13,19)])
|
||||
df['evening_total'] = sum([df['Hour_' + str(i)] for i in range(19,23)])
|
||||
df['night_total'] = sum([df['Hour_' + str(i)] for i in [23,0,1,2,3,4,5]])
|
||||
return df
|
||||
|
||||
def test_basic_algorithm_on_sequence_lengths(clf = RandomForestClassifier(random_state=17)):
|
||||
"""
|
||||
Runs a basic algorith from scikit learn on different sequence lengths. Plots an image for the results.
|
||||
:param clf: the sklearn classifier
|
||||
"""
|
||||
tr, val, te = filter_and_preprocess_data(dataset_hrs_path)
|
||||
|
||||
results_train = pd.DataFrame()
|
||||
results_valid = pd.DataFrame()
|
||||
# iterate over sequence lengths
|
||||
for sequence_length in range(1, 60, 5):
|
||||
x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length)
|
||||
x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length)
|
||||
clf.fit(x_tr, y_tr)
|
||||
acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=clf.predict(x_val))
|
||||
results_valid = pd.concat([results_valid, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True)
|
||||
acc, p, r, f1 = eval_metrics(y_true=y_tr, y_pred=clf.predict(x_tr))
|
||||
results_train = pd.concat([results_train, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True)
|
||||
|
||||
fig = plt.figure()
|
||||
|
||||
for frame in [results_train, results_valid]:
|
||||
plt.plot(frame[sequence_length_str], frame[f1_string])
|
||||
|
||||
plt.show()
|
||||
print('')
|
||||
|
||||
def tune_neural_network(model_type):
|
||||
"""
|
||||
Tunes a neural model for different sequence lengths. Plots the results
|
||||
:param model_type: either lstm, gru or bilstm, use set strings
|
||||
"""
|
||||
# TODO: Also do this for minute data
|
||||
tr, val, te = filter_and_preprocess_data(dataset_hrs_path)
|
||||
n_epochs= 20
|
||||
n_neurons = 1024
|
||||
n_batch = 1024
|
||||
results_train = pd.DataFrame()
|
||||
results_valid = pd.DataFrame()
|
||||
# iterate over sequence lengths
|
||||
for sequence_length in range(1, 50, 5):
|
||||
train_data = prepare_data_for_neural_model(user_data=tr, sequence_length=sequence_length)
|
||||
val_data = prepare_data_for_neural_model(user_data=val, sequence_length=sequence_length)
|
||||
|
||||
# fit and evaluate model
|
||||
history_list = list()
|
||||
repeats = 3
|
||||
# run diagnostic tests
|
||||
for i in range(repeats):
|
||||
history = train_one_model(train_data, val_data, n_batch, n_epochs,
|
||||
n_neurons, sequence_length=sequence_length,
|
||||
model_type=model_type)
|
||||
history_list.append(history)
|
||||
results = pd.concat([history.tail(1) for history in history_list]).mean()
|
||||
results_train = pd.concat([results_train,
|
||||
DataFrame({sequence_length_str:[sequence_length],
|
||||
accuracy_str:[results['train_acc']],
|
||||
precision_str:[results['train_p']],
|
||||
recall_str:[results['train_r']],
|
||||
f1_string:[results['train_f1']]})], ignore_index=True)
|
||||
results_valid = pd.concat([results_valid,
|
||||
DataFrame({sequence_length_str:[sequence_length],
|
||||
accuracy_str:[results['test_acc']],
|
||||
precision_str:[results['test_p']],
|
||||
recall_str:[results['test_r']],
|
||||
f1_string:[results['test_f1']]})], ignore_index=True)
|
||||
|
||||
fig = plt.figure()
|
||||
for frame in [results_train, results_valid]:
|
||||
plt.plot(frame[sequence_length_str], frame[f1_string])
|
||||
|
||||
plt.show()
|
||||
print('Done')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Ordner erstellen, die benötigt werden
|
||||
# create needed directories
|
||||
create_dir('results/')
|
||||
create_dir(figure_path)
|
||||
pd.options.mode.copy_on_write = True
|
||||
|
||||
# main_two_v1()
|
||||
# visualise_results_v1()
|
||||
#test(model_type=model_type_gru)
|
||||
# main_two_v2(model_type=model_type_gru)
|
||||
#visualise_results_v2()
|
||||
#manual_tuning(model_type=model_type_lstm)
|
||||
#calculate_baselines()
|
||||
baselines = calculate_baselines()
|
||||
|
||||
#### Ab hier aktuell (21.01.2026)
|
||||
#calculate_baselines_v3()
|
||||
manual_tuning_v3(model_type=model_type_lstm)
|
||||
# use basic algorithms from scikit learn
|
||||
test_basic_algorithms()
|
||||
hypertune_basic_algorithms()
|
||||
test_basic_algorithm_on_sequence_lengths()
|
||||
|
||||
# tune a neural network
|
||||
tune_neural_network(model_type=model_type_lstm)
|
||||
print('Done')
|
||||
Reference in New Issue
Block a user