bs/Step_Data_Project_India
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Switched preprocessing to not use boolean values but actual step counts

Browse Source
This commit is contained in:
bs
2025-11-27 16:09:38 +01:00
parent b9fb1128c7
commit 05808cf8f3
7 changed files with 239 additions and 50896 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+91 -5
View File
@@ -8,6 +8,7 @@ from keras.src.regularizers import L1L2
from matplotlib import pyplot as plt
from pandas import DataFrame
from sklearn.dummy import DummyClassifier
from sklearn.preprocessing import MinMaxScaler
from pipeline import (
load_dataset,
@@ -22,6 +23,9 @@ from pipeline import (
year_str = 'Year'
month_str = 'Month'
date_str = 'Date'
time_str = 'Time'
day_of_week_str = 'DayOfWeek'
user_str = 'user'
split_str = 'split type'
data_split_str = 'data percentages'
@@ -38,12 +42,14 @@ precision_str = 'precision'
recall_str = 'recall'
f1_string = 'f1 score'
model_type_str = 'model type'
weak_column_names = ['DayOfWeek_'+day for day in
week_column_names = ['DayOfWeek_' + day for day in
['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday' ]]
figure_path = 'figures/'
# === Configurable Parameters ===
dataset_path = './Datasets/'
dataset_hrs_path = './Datasets/hours.json'
dataset_min_path = './Datasets/minutes.json'
DATA_PATH = dataset_path +'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx'
OUTPUT_EXCEL_PATH = './working/evaluation_results.xlsx'
result_filename_v1 = './working/evaluation_results.json'
@@ -73,7 +79,7 @@ def split_data_by_month_percentage(df, percentages):
tr, va, te = np.split(ids, [int((train_p/100) * len(ids)), int(((train_p + valid_p)/100) * len(ids))])
return df.merge(tr, on=[year_str, month_str], how='inner'), df.merge(va, on=[year_str, month_str], how='inner'), df.merge(te, on=[year_str, month_str], how='inner')
def split_data_by_userdata_percentage(df, percentages, sample):
def split_data_by_userdata_percentage(df, percentages, sample=100):
train_p, valid_p, test_p = percentages
tr, va, te = pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
for user_id in df[user_str].unique():
@@ -119,10 +125,13 @@ def main():
def reduce_columns(df, filename):
if min_timespan_str in filename:
return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek']+weak_column_names, errors='ignore')
return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'] + week_column_names, errors='ignore')
else:
return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'], errors='ignore')
def reduce_columns_v3(df):
return df.drop(columns=[month_str, year_str, date_str])
def load_previous_results(filename):
results = pd.DataFrame()
@@ -372,6 +381,37 @@ def manual_tuning(model_type):
print('Done')
def manual_tuning_v3(model_type):
# TODO: hrs/min + different sequence lengths
sequence_length = 20
tr, val, te = get_prepared_data_v3(dataset_hrs_path)
# fit and evaluate model
# config
repeats = 3
n_batch = 1024
n_epochs = 500
n_neurons = 16
l_rate = 1e-4
history_list = list()
# run diagnostic tests
for i in range(repeats):
history = train_one_model(tr, val, n_batch, n_epochs,
n_neurons, l_rate,
sequence_length=sequence_length,
model_type=model_type)
history_list.append(history)
for metric in ['p', 'r', 'f1']:
for history in history_list:
plt.plot(history['train_'+metric], color='blue')
plt.plot(history['test_'+metric], color='orange')
plt.savefig(figure_path+'v3/'+metric+'_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+
str(n_batch)+'_l'+str(l_rate)+'_diagnostic.png')
plt.clf()
print('Done')
def calculate_baselines():
file_combinations = [(hour_timespan_str, with_threshold_str,'ALL32USERS1HR_WITHTHRESHOLD.xlsx'),
@@ -404,6 +444,51 @@ def calculate_baselines():
baseline_res.to_json('baseline_results.json')
print('Done')
def get_prepared_data_v3(filename, sample=100):
df = pd.read_json(filename)
df = remove_covid_data(df)
tr, val, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=sample)
tr = reduce_columns_v3(tr)
val = reduce_columns_v3(val)
te = reduce_columns_v3(te)
scaler = MinMaxScaler()
scaler.fit(tr.drop(columns=[user_str]))
return scale_dataset(scaler, tr), scale_dataset(scaler, val), scale_dataset(scaler, te)
def scale_dataset(scaler, df):
y = df[user_str]
x_scaled = scaler.transform(df.drop(columns=[user_str]))
df_scaled = pd.concat([pd.DataFrame(x_scaled), pd.DataFrame(y)], axis=1)
df_scaled.columns = df.columns
return prepare_user_data(df)
def calculate_baselines_v3():
file_combinations = [(hour_timespan_str, dataset_hrs_path),
(min_timespan_str, dataset_min_path),
]
baseline_res = pd.DataFrame()
for timespan_id, filename in file_combinations:
_, _, te = get_prepared_data_v3(filename)
for sequence_length in range(5,30, 5):
x, y = prepare_data_for_model(user_data=te, sequence_length=sequence_length)
for strategy in ['most_frequent', 'stratified', 'uniform']:
cls = DummyClassifier(strategy=strategy)
cls.fit(x,y)
y_pred = cls.predict(x)
acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred)
baseline_res = pd.concat([baseline_res,
DataFrame({ 'strategy':[strategy],
timespan_str:[timespan_id], sequence_length_str:[sequence_length],
accuracy_str:[acc],precision_str:[p],recall_str:[r],
f1_string:f1})], ignore_index=True)
baseline_res.to_json('baseline_results_v3.json')
print('Done')
if __name__ == "__main__":
# main_two_v1()
@@ -411,6 +496,7 @@ if __name__ == "__main__":
#test(model_type=model_type_gru)
# main_two_v2(model_type=model_type_gru)
#visualise_results_v2()
manual_tuning(model_type=model_type_lstm)
#manual_tuning(model_type=model_type_lstm)
#calculate_baselines()
calculate_baselines_v3()
print('Done')