bs
/
Step_Data_Project_India
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Added Baselines + extended manual evaluation

master
Bianca Steffes 3 weeks ago
parent
ac57fef0e5
commit
40b32c30d3
3 changed files with 73 additions and 17 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      .gitignore
  2. 57
      main.py
  3. 32
      pipeline.py

1
.gitignore
View File

@ -142,3 +142,4 @@ cython_debug/
working/tuner working/tuner
working working
figures figures
baseline_results.json

57
main.py
View File

@ -4,7 +4,10 @@ import os
import numpy as np import numpy as np
import pandas as pd import pandas as pd
import sklearn import sklearn
from keras.src.regularizers import L1L2
from matplotlib import pyplot as plt from matplotlib import pyplot as plt
from pandas import DataFrame
from sklearn.dummy import DummyClassifier
from pipeline import ( from pipeline import (
load_dataset, load_dataset,
@ -13,7 +16,8 @@ from pipeline import (
prepare_user_data, prepare_user_data,
train_models, train_models,
evaluate_models, evaluate_models,
prepare_data_for_model, model_type_gru, model_type_lstm, model_type_bilstm, train_models_v2, train_one_model
prepare_data_for_model, model_type_gru, model_type_lstm, model_type_bilstm, train_models_v2, train_one_model,
eval_metrics
) )
year_str = 'Year' year_str = 'Year'
@ -29,6 +33,7 @@ timespan_str = 'time used'
hour_timespan_str = '1HR' hour_timespan_str = '1HR'
min_timespan_str = '15MIN' min_timespan_str = '15MIN'
sequence_length_str = 'sequence length' sequence_length_str = 'sequence length'
accuracy_str = 'accuracy'
precision_str = 'precision' precision_str = 'precision'
recall_str = 'recall' recall_str = 'recall'
f1_string = 'f1 score' f1_string = 'f1 score'
@ -332,7 +337,7 @@ def manual_tuning(model_type):
df = load_dataset(file_path) df = load_dataset(file_path)
df = remove_covid_data(df) df = remove_covid_data(df)
tr, val, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=20)
tr, val, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=100)
tr = reduce_columns(tr, data_filename) tr = reduce_columns(tr, data_filename)
val = reduce_columns(val, data_filename) val = reduce_columns(val, data_filename)
te = reduce_columns(te, data_filename) te = reduce_columns(te, data_filename)
@ -342,15 +347,18 @@ def manual_tuning(model_type):
# fit and evaluate model # fit and evaluate model
# config # config
repeats = 5
n_batch = 4
repeats = 3
n_batch = 1024
n_epochs = 500 n_epochs = 500
n_neurons = 1
n_neurons = 16
l_rate = 1e-4
reg = L1L2(l1=0.0, l2=0.0)
history_list = list() history_list = list()
# run diagnostic tests # run diagnostic tests
for i in range(repeats): for i in range(repeats):
history = train_one_model(user_data_train, user_data_val, n_batch, n_epochs, n_neurons,
history = train_one_model(user_data_train, user_data_val, n_batch, n_epochs,
n_neurons, l_rate, reg,
sequence_length=sequence_length, sequence_length=sequence_length,
model_type=model_type) model_type=model_type)
history_list.append(history) history_list.append(history)
@ -358,11 +366,45 @@ def manual_tuning(model_type):
for history in history_list: for history in history_list:
plt.plot(history['train_'+metric], color='blue') plt.plot(history['train_'+metric], color='blue')
plt.plot(history['test_'+metric], color='orange') plt.plot(history['test_'+metric], color='orange')
plt.savefig(figure_path+metric+'_epochs_diagnostic.png')
plt.savefig(figure_path+metric+'_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+
str(n_batch)+'_l'+str(l_rate)+'_diagnostic.png')
plt.clf() plt.clf()
print('Done') print('Done')
def calculate_baselines():
file_combinations = [(hour_timespan_str, with_threshold_str,'ALL32USERS1HR_WITHTHRESHOLD.xlsx'),
(min_timespan_str, with_threshold_str, 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx'),
(min_timespan_str, without_threshold_str, 'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx'),
(hour_timespan_str, without_threshold_str, 'ALLUSERS_32_1HR_WITHOUT_THRESHOLD.xlsx'),
]
baseline_res = pd.DataFrame()
for timespan_id, threshold_id, filename in file_combinations:
file_path = os.path.join(dataset_path, filename)
df = load_dataset(file_path)
df = remove_covid_data(df)
_, _, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=20)
te = reduce_columns(te, filename)
user_data_te = prepare_user_data(te)
for sequence_length in range(5,30, 5):
x, y = prepare_data_for_model(user_data=user_data_te, sequence_length=sequence_length)
for strategy in ['most_frequent', 'stratified', 'uniform']:
cls = DummyClassifier(strategy=strategy)
cls.fit(x,y)
y_pred = cls.predict(x)
acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred)
baseline_res = pd.concat([baseline_res,
DataFrame({ 'strategy':[strategy], threshold_str:[threshold_id],
timespan_str:[timespan_id], sequence_length_str:[sequence_length],
accuracy_str:[acc],precision_str:[p],recall_str:[r],
f1_string:f1})], ignore_index=True)
baseline_res.to_json('baseline_results.json')
print('Done')
if __name__ == "__main__": if __name__ == "__main__":
# main_two_v1() # main_two_v1()
# visualise_results_v1() # visualise_results_v1()
@ -370,4 +412,5 @@ if __name__ == "__main__":
# main_two_v2(model_type=model_type_gru) # main_two_v2(model_type=model_type_gru)
#visualise_results_v2() #visualise_results_v2()
manual_tuning(model_type=model_type_lstm) manual_tuning(model_type=model_type_lstm)
#calculate_baselines()
print('Done') print('Done')

32
pipeline.py
View File

@ -1,3 +1,5 @@
import random
import keras_tuner import keras_tuner
import numpy as np import numpy as np
import pandas as pd import pandas as pd
@ -78,6 +80,8 @@ def prepare_data_for_model(user_data, sequence_length):
x_new, y_new = make_sequences(data, sequence_length) x_new, y_new = make_sequences(data, sequence_length)
x = x + x_new x = x + x_new
y = y + y_new y = y + y_new
random.Random(17).shuffle(x)
random.Random(17).shuffle(y)
x = np.array(x) x = np.array(x)
y = np.array(y) y = np.array(y)
return x,y return x,y
@ -205,7 +209,7 @@ def train_models_v2(user_data, user_data_val, sequence_length, model_type):
return tuner.get_best_models(num_models=1)[0] return tuner.get_best_models(num_models=1)[0]
def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, sequence_length, model_type):
def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, l_rate, reg, sequence_length, model_type):
x, y = prepare_data_for_model(user_data=train_data, sequence_length=sequence_length) x, y = prepare_data_for_model(user_data=train_data, sequence_length=sequence_length)
n_features = x.shape[2] n_features = x.shape[2]
users = list(train_data.keys()) users = list(train_data.keys())
@ -213,7 +217,7 @@ def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, sequence
# prepare model # prepare model
def build_model(): def build_model():
model = Sequential() model = Sequential()
model.add(Input(shape=(sequence_length, n_features), batch_size=n_batch))
model.add(Input(shape=(sequence_length, n_features), batch_size=n_batch, bias_regularizer=reg))
# if model_type == model_type_bilstm: # if model_type == model_type_bilstm:
# model.add(Bidirectional(units=units_hp)) # model.add(Bidirectional(units=units_hp))
if model_type == model_type_lstm: if model_type == model_type_lstm:
@ -225,7 +229,7 @@ def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, sequence
# model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.2, step=0.1))) # model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.2, step=0.1)))
model.add(Dense(len(users), activation='softmax')) model.add(Dense(len(users), activation='softmax'))
model.compile( model.compile(
optimizer=Adam(learning_rate=1e-5),
optimizer=Adam(learning_rate=l_rate),
loss=SparseCategoricalCrossentropy(), loss=SparseCategoricalCrossentropy(),
metrics=[SparseCategoricalAccuracy()], metrics=[SparseCategoricalAccuracy()],
) )
@ -234,21 +238,24 @@ def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, sequence
model = build_model() model = build_model()
# fit model # fit model
train_p, test_p, train_r, test_r, train_f1, test_f1 = list(), list(),list(), list(),list(), list()
train_acc, test_acc, train_p, test_p, train_r, test_r, train_f1, test_f1 = list(), list(),list(), list(),list(), list(),list(), list()
for i in range(n_epochs): for i in range(n_epochs):
model.fit(x, y, batch_size=n_batch, epochs=1, verbose=0, shuffle=False) model.fit(x, y, batch_size=n_batch, epochs=1, verbose=0, shuffle=False)
# evaluate model on train data # evaluate model on train data
p, r, f1 = evaluate(model, train_data, sequence_length, n_batch)
acc, p, r, f1 = evaluate(model, train_data, sequence_length, n_batch)
train_acc.append(acc)
train_p.append(p) train_p.append(p)
train_r.append(r) train_r.append(r)
train_f1.append(f1) train_f1.append(f1)
# evaluate model on test data # evaluate model on test data
p, r, f1 = evaluate(model, val_data, sequence_length, n_batch)
acc, p, r, f1 = evaluate(model, val_data, sequence_length, n_batch)
test_acc.append(acc)
test_p.append(p) test_p.append(p)
test_r.append(r) test_r.append(r)
test_f1.append(f1) test_f1.append(f1)
history = DataFrame() history = DataFrame()
history['train_acc'], history['test_acc'] = train_acc, test_acc
history['train_p'], history['test_p'] = train_p, test_p history['train_p'], history['test_p'] = train_p, test_p
history['train_r'], history['test_r'] = train_r, test_r history['train_r'], history['test_r'] = train_r, test_r
history['train_f1'], history['test_f1'] = train_f1, test_f1 history['train_f1'], history['test_f1'] = train_f1, test_f1
@ -262,11 +269,16 @@ def evaluate(model, df, sequence_length, batch_size):
y_pred = model.predict(x, verbose=0, batch_size=batch_size) y_pred = model.predict(x, verbose=0, batch_size=batch_size)
y_pred_classes = np.argmax(y_pred, axis=1) y_pred_classes = np.argmax(y_pred, axis=1)
f1 = f1_score(y_true=y_true, y_pred=y_pred_classes, average='weighted')
p = precision_score(y_true=y_true, y_pred=y_pred_classes, average='weighted')
r = recall_score(y_true=y_true, y_pred=y_pred_classes, average='weighted')
return p, r, f1
return eval_metrics(y_true=y_true, y_pred=y_pred_classes)
def eval_metrics(y_true, y_pred):
f1 = f1_score(y_true=y_true, y_pred=y_pred, average='weighted')
p = precision_score(y_true=y_true, y_pred=y_pred, average='weighted')
r = recall_score(y_true=y_true, y_pred=y_pred, average='weighted')
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
return acc, p, r, f1
# === Evaluation === # === Evaluation ===
def evaluate_models(best_models, df_test, sequence_lengths, output_excel_path, ALLUSERS32_15MIN_WITHOUTTHREHOLD): def evaluate_models(best_models, df_test, sequence_lengths, output_excel_path, ALLUSERS32_15MIN_WITHOUTTHREHOLD):

Write Preview
Loading…
Cancel
Save