bs/Step_Data_Project_India
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated code for different tests

Browse Source
This commit is contained in:
Bianca Steffes
2026-01-21 09:20:43 +01:00
parent 05808cf8f3
commit 98831ce4f3
3 changed files with 112 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipeline.py
+33 -13
View File
@@ -14,13 +14,14 @@ from tensorflow.keras.layers import LSTM, Dense, Dropout, Bidirectional,GRU
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
from keras_tuner import RandomSearch
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix
epochs = 5#50
model_type_gru = 'GRU'
model_type_lstm = 'LSTM'
model_type_bilstm = 'BiLSTM'
# === Display functions ===
def display_warning_about_2020_data():
print("\n⚠️ Warning: 2020 data after February is excluded due to COVID-19.")
@@ -67,11 +68,11 @@ def prepare_user_data(df):
def make_sequences(data, sequence_length):
x, y = [], []
features = data.drop('user', axis=1).values
features = features.astype(int)
#features = features.astype(int)
labels = data['user'].values
for i in range(len(features) - sequence_length):
for i in range(len(features) - sequence_length+1):
x.append(features[i:i + sequence_length])
y.append(labels[i + sequence_length])
y.append(labels[i + sequence_length-1])
return x, y
def prepare_data_for_model(user_data, sequence_length):
@@ -209,7 +210,7 @@ def train_models_v2(user_data, user_data_val, sequence_length, model_type):
return tuner.get_best_models(num_models=1)[0]
def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, l_rate, sequence_length, model_type):
def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons,n_neurons2,n_neurons3,n_neurons4, l_rate, d1, r1, reg1, r2, reg2, sequence_length, model_type):
x, y = prepare_data_for_model(user_data=train_data, sequence_length=sequence_length)
n_features = x.shape[2]
users = list(train_data.keys())
@@ -218,15 +219,19 @@ def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, l_rate,
def build_model():
model = Sequential()
model.add(Input(shape=(sequence_length, n_features), batch_size=n_batch))
# if model_type == model_type_bilstm:
# model.add(Bidirectional(units=units_hp))
if model_type == model_type_bilstm:
model.add(Bidirectional(LSTM(n_neurons)))
if model_type == model_type_lstm:
# model.add(LSTM(n_neurons, kernel_regularizer=reg1, return_sequences=True))
model.add(LSTM(n_neurons))
# if model_type == model_type_gru:
# model.add(GRU(units=units_hp))
# model.add(LSTM(n_neurons2))
# model.add(LSTM(n_neurons3, return_sequences=True))
# model.add(LSTM(n_neurons4))
if model_type == model_type_gru:
model.add(GRU(n_neurons))
# TODO: add another dense layer
#model.add(Dense(256, activation='relu'))
# model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.2, step=0.1)))
#model.add(Dense(n_neurons, activation='relu'))
#model.add(Dropout(d1))
model.add(Dense(len(users), activation='softmax'))
model.compile(
optimizer=Adam(learning_rate=l_rate),
@@ -248,7 +253,8 @@ def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, l_rate,
train_r.append(r)
train_f1.append(f1)
# evaluate model on test data
acc, p, r, f1 = evaluate(model, val_data, sequence_length, n_batch)
savename = 'cf_matrix_'+get_save_id(n_epochs, n_neurons, n_neurons2,n_neurons3, n_neurons4, n_batch, l_rate,d1,r1, r2)+'.json'
acc, p, r, f1 = evaluate(model, val_data, sequence_length, n_batch, save_name=savename)
test_acc.append(acc)
test_p.append(p)
test_r.append(r)
@@ -262,13 +268,27 @@ def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, l_rate,
return history
def evaluate(model, df, sequence_length, batch_size):
def get_save_id(n_epochs, n_neurons, n_neurons2,n_neurons3,n_neurons4, n_batch, l_rate, d1,r1, r2):
return '_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+ str(n_batch)
#'x'+str(n_neurons3)+'x'+str(n_neurons4)
#+'_l'+str(l_rate)+'_r'+str(r1)+'xx'+str(r2)
def evaluate(model, df, sequence_length, batch_size, save_name=None):
x, y = prepare_data_for_model(user_data=df, sequence_length=sequence_length)
x = np.array(x)
y_true = np.array(y)
y_pred = model.predict(x, verbose=0, batch_size=batch_size)
y_pred_classes = np.argmax(y_pred, axis=1)
cf_matrix = pd.DataFrame(confusion_matrix(y_true, y_pred_classes))
if save_name is not None:
cf_matrix.to_json('results/'+save_name)
true_counts = pd.DataFrame(y).value_counts()
print('Top true occurrences', true_counts[:6])
predicted_counts = pd.DataFrame(y_pred_classes).value_counts()
print('Top predicted occurrences', predicted_counts[:6])
return eval_metrics(y_true=y_true, y_pred=y_pred_classes)