diff --git a/.gitignore b/.gitignore index 9ff7e30..8d77471 100644 --- a/.gitignore +++ b/.gitignore @@ -1,147 +1,5 @@ -# ---> Python -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -.idea -working/tuner -working +.idea +__pycache__ figures -baseline_results.json -baseline_results_v3.json results +working diff --git a/Europe.zip b/Europe.zip deleted file mode 100644 index 71ffa62..0000000 Binary files a/Europe.zip and /dev/null differ diff --git a/Rest_of_the_World.zip b/Rest_of_the_World.zip deleted file mode 100644 index 0b3de71..0000000 Binary files a/Rest_of_the_World.zip and /dev/null differ diff --git a/baseline_results.json b/baseline_results.json new file mode 100644 index 0000000..36c59ce --- /dev/null +++ b/baseline_results.json @@ -0,0 +1 @@ +{"strategy":{"0":"most_frequent","1":"stratified","2":"uniform","3":"most_frequent","4":"stratified","5":"uniform","6":"most_frequent","7":"stratified","8":"uniform","9":"most_frequent","10":"stratified","11":"uniform","12":"most_frequent","13":"stratified","14":"uniform","15":"most_frequent","16":"stratified","17":"uniform"},"time used":{"0":"1HR","1":"1HR","2":"1HR","3":"1HR","4":"1HR","5":"1HR","6":"1HR","7":"1HR","8":"1HR","9":"1HR","10":"1HR","11":"1HR","12":"1HR","13":"1HR","14":"1HR","15":"1HR","16":"1HR","17":"1HR"},"sequence length":{"0":1,"1":1,"2":1,"3":6,"4":6,"5":6,"6":11,"7":11,"8":11,"9":16,"10":16,"11":16,"12":21,"13":21,"14":21,"15":26,"16":26,"17":26},"accuracy":{"0":0.0465710356,"1":0.0387109452,"2":0.0322263706,"3":0.0471161657,"4":0.0337124289,"5":0.0314784728,"6":0.0476990964,"7":0.0355116621,"8":0.0310989704,"9":0.0483239007,"10":0.035263387,"11":0.0343926861,"12":0.0489952585,"13":0.0363513208,"14":0.0325129826,"15":0.0497185741,"16":0.0356472795,"17":0.0300187617},"precision":{"0":0.0021688614,"1":0.038723827,"2":0.035357886,"3":0.0022199331,"4":0.0344564588,"5":0.0368125232,"6":0.0022752038,"7":0.0355258063,"8":0.034249105,"9":0.0023351994,"10":0.0353594615,"11":0.038321258,"12":0.0024005354,"13":0.0363413163,"14":0.0372264035,"15":0.0024719366,"16":0.0354808739,"17":0.0381814576},"recall":{"0":0.0465710356,"1":0.0387109452,"2":0.0322263706,"3":0.0471161657,"4":0.0337124289,"5":0.0314784728,"6":0.0476990964,"7":0.0355116621,"8":0.0310989704,"9":0.0483239007,"10":0.035263387,"11":0.0343926861,"12":0.0489952585,"13":0.0363513208,"14":0.0325129826,"15":0.0497185741,"16":0.0356472795,"17":0.0300187617},"f1 score":{"0":0.0041446997,"1":0.0386459741,"2":0.0329666692,"3":0.0042400894,"4":0.0340329362,"5":0.033074915,"6":0.004343239,"7":0.0354644534,"8":0.0315466073,"9":0.0044551105,"10":0.0352542225,"11":0.0349449461,"12":0.0045768279,"13":0.0362991035,"14":0.0333112368,"15":0.004709713,"16":0.0355062319,"17":0.0327189637}} \ No newline at end of file diff --git a/main.py b/main.py index cfdd27a..fe5d403 100644 --- a/main.py +++ b/main.py @@ -1,29 +1,23 @@ -import json import os -import math import numpy as np import pandas as pd -import sklearn -from keras.src.regularizers import L1L2 from matplotlib import pyplot as plt from pandas import DataFrame from sklearn.dummy import DummyClassifier +from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier +from sklearn.model_selection import GridSearchCV +from sklearn.naive_bayes import BernoulliNB from sklearn.preprocessing import MinMaxScaler from pipeline import ( - load_dataset, - filter_data, - filter_test_data, - prepare_user_data, - train_models, - evaluate_models, - prepare_data_for_model, model_type_gru, model_type_lstm, model_type_bilstm, train_models_v2, train_one_model, - eval_metrics, get_save_id + prepare_data_for_neural_model, model_type_gru, model_type_lstm, model_type_bilstm, + eval_metrics, prepare_data_for_basic_algorithm, train_one_model, ) year_str = 'Year' month_str = 'Month' +day_str = 'Day' date_str = 'Date' time_str = 'Time' day_of_week_str = 'DayOfWeek' @@ -31,9 +25,6 @@ user_str = 'user' split_str = 'split type' data_split_str = 'data percentages' month_split_str = 'month percentages' -threshold_str = 'threshold used' -with_threshold_str = 'WITH' -without_threshold_str = 'WITHOUT' timespan_str = 'time used' hour_timespan_str = '1HR' min_timespan_str = '15MIN' @@ -46,30 +37,11 @@ model_type_str = 'model type' week_column_names = ['DayOfWeek_' + day for day in ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday' ]] figure_path = 'figures/' -predicitons_path = 'preds/' # === Configurable Parameters === dataset_path = './Datasets/' dataset_hrs_path = './Datasets/hours.json' dataset_min_path = './Datasets/minutes.json' -DATA_PATH = dataset_path +'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx' -OUTPUT_EXCEL_PATH = './working/evaluation_results.xlsx' -result_filename_v1 = './working/evaluation_results.json' -result_filename_v2 = './working/evaluation_results_v2.json' -SEQUENCE_LENGTHS = [30, 25, 20, 15, 10, 5] # You can add more: [20, 25, 30] - -TRAINING_SCENARIO = [(2018, list(range(1, 13))), (2019, list(range(1, 10)))] -VALIDATION_SCENARIO = [(2019, [10, 11, 12])] -TEST_SCENARIO = [(2020, [1, 2])] # Jan–Feb 2020 only - -# === Optional display only === -predefined_training_scenarios = { - "Scenario 1": {"years_months": [(2018, list(range(1, 13))), (2019, list(range(1, 10)))]}, - "Scenario 2": {"years_months": [(2017, list(range(1, 13))), (2018, list(range(1, 13))), (2019, list(range(1, 10)))]} -} -predefined_validation_scenarios = { - "Scenario A": {"years_months": [(2019, [10, 11, 12])]} -} def create_dir(path): """ @@ -81,424 +53,74 @@ def create_dir(path): os.makedirs(path) def remove_covid_data(df): + """ + Removes covid data from dataframe because the steps from these times will most likely differ from before + :param df: Dataframe with the data + :return: the data without covid time data + """ df = df[~(df[year_str]>=2020)] return df -def split_data_by_month_percentage(df, percentages): - train_p, valid_p, test_p = percentages - ids = df[[year_str, month_str]].drop_duplicates().sort_values([year_str, month_str]) - tr, va, te = np.split(ids, [int((train_p/100) * len(ids)), int(((train_p + valid_p)/100) * len(ids))]) - return df.merge(tr, on=[year_str, month_str], how='inner'), df.merge(va, on=[year_str, month_str], how='inner'), df.merge(te, on=[year_str, month_str], how='inner') - def split_data_by_userdata_percentage(df, percentages, sample=100): + """ + Splits data by userdata percentages. Each users data will be split according to the given percentages along the time axis + + :param df: Data with all users data + :param percentages: triple with percentages + :param sample: overall percentage if less data should be used. Use only for testing. Has an error!! + :return: the split data + """ train_p, valid_p, test_p = percentages tr, va, te = pd.DataFrame(), pd.DataFrame(), pd.DataFrame() for user_id in df[user_str].unique(): - user_data = df[df[user_str]==user_id].sample(frac=sample/ 100).sort_values([year_str, month_str]) + # !! following sample creates gaps in data if sample smaller 100 + user_data = df[df[user_str]==user_id].sample(frac=sample/ 100).sort_values([date_str]) # have to sort for time shift u_tr, u_va, u_te = np.split(user_data, [int((train_p/100)*len(user_data)), int(((train_p+valid_p)/100)*len(user_data))]) tr = pd.concat([tr, u_tr], ignore_index=True) va = pd.concat([va, u_va], ignore_index=True) te = pd.concat([te, u_te], ignore_index=True) return tr, va, te - -def main(): - # print("=== Training Scenario Setup ===") - # display_warning_about_2020_data() - # display_warnings_for_scenarios("training", predefined_training_scenarios, predefined_validation_scenarios) - - # print("\n=== Validation Scenario Setup ===") - # display_warning_about_2020_data() - # display_warnings_for_scenarios("validation", predefined_training_scenarios, predefined_validation_scenarios) - - # === Load and preprocess === - df = load_dataset(DATA_PATH) - - ALLUSERS32_15MIN_WITHOUTTHREHOLD = False - if('ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx' in DATA_PATH): - ALLUSERS32_15MIN_WITHOUTTHREHOLD = True - - training_data = filter_data(df, TRAINING_SCENARIO, ALLUSERS32_15MIN_WITHOUTTHREHOLD) - validation_data = filter_data(df, VALIDATION_SCENARIO, ALLUSERS32_15MIN_WITHOUTTHREHOLD) - - user_data_train = prepare_user_data(training_data) - user_data_val = prepare_user_data(validation_data) - - # === Train models === - best_models = train_models(user_data_train, user_data_val, sequence_lengths=SEQUENCE_LENGTHS) - - # === Load and evaluate test === - test_df = filter_test_data(df, TEST_SCENARIO) - evaluate_models(best_models, test_df, SEQUENCE_LENGTHS, OUTPUT_EXCEL_PATH, ALLUSERS32_15MIN_WITHOUTTHREHOLD) - - print(f"\n✅ All evaluations completed. Results saved to: {OUTPUT_EXCEL_PATH}") - - -def reduce_columns(df, filename): - if min_timespan_str in filename: - return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'] + week_column_names, errors='ignore') - else: - return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'], errors='ignore') - - -def reduce_columns_v3(df): +def reduce_columns(df): + """ + Removes unnecessary columns from dataframe. + :param df: Dataframe with the data + :return: dataframe without unnecessary columns + """ return df.drop(columns=[month_str, year_str, date_str]) -def load_previous_results(filename): - results = pd.DataFrame() - if os.path.exists(filename): - results = pd.DataFrame(json.load(open(filename))) - return results - -def main_two_v2(model_type): - seq_length = range(10,31, 5) - for sequence_length in seq_length: - for data_filename in os.listdir(dataset_path): - timespan_id = hour_timespan_str - threshold_id = with_threshold_str - if min_timespan_str in data_filename: - timespan_id = min_timespan_str - if without_threshold_str in data_filename: - threshold_id = without_threshold_str - - results = load_previous_results(result_filename_v2) - if len(results) > 0: - if len(results[(results[timespan_str]==timespan_id) & - (results[threshold_str]==threshold_id) & - (results[sequence_length_str]==sequence_length) & - (results[model_type_str]==model_type)]) > 0: - continue - - file_path = os.path.join(dataset_path, data_filename) - df = load_dataset(file_path) - df = remove_covid_data(df) - - tr,val,te = split_data_by_userdata_percentage(df, percentages=(80,10,10)) - tr = reduce_columns(tr, data_filename) - val = reduce_columns(val, data_filename) - te = reduce_columns(te, data_filename) - - user_data_train = prepare_user_data(tr) - user_data_val = prepare_user_data(val) - - best_model = train_models_v2(user_data_train, user_data_val, - sequence_length=sequence_length, - model_type=model_type) - - results = load_previous_results(result_filename_v2) - results = pd.concat([results, - evaluate_model_on_test_data(model=best_model, - test_df=te, - sequence_length=sequence_length, - time_span_id=timespan_id, - threshold_id=threshold_id, - model_type=model_type, - split_id=data_split_str)], - ignore_index=True) - results.to_json(result_filename_v2) - -def main_two_v1(): - seq_length = [30, 25, 20, 15, 10, 5] # You can add more: [20, 25, 30] - results = pd.DataFrame() - if os.path.exists(result_filename_v1): - results = pd.DataFrame(json.load(open(result_filename_v1))) - for sequence_length in seq_length: - for data_filename in os.listdir(dataset_path): - for split_id, split_method in [(data_split_str, split_data_by_userdata_percentage),(month_split_str, split_data_by_month_percentage)]: - for model_type in [model_type_lstm, model_type_bilstm, model_type_gru]: - timespan_id = hour_timespan_str - threshold_id = with_threshold_str - if min_timespan_str in data_filename: - timespan_id = min_timespan_str - if without_threshold_str in data_filename: - threshold_id = without_threshold_str - if len(results) > 0: - if len(results[(results[split_str]==split_id) & - (results[timespan_str]==timespan_id) & - (results[threshold_str]==threshold_id) & - (results[sequence_length_str]==sequence_length) & - (results[model_type_str]==model_type)]) > 0: - continue - - file_path = os.path.join(dataset_path, data_filename) - df = load_dataset(file_path) - df = remove_covid_data(df) - tr,val,te = split_method(df, percentages=(80,10,10)) - tr = reduce_columns(tr, data_filename) - val = reduce_columns(val, data_filename) - te = reduce_columns(te, data_filename) - - user_data_train = prepare_user_data(tr) - user_data_val = prepare_user_data(val) - - best_models = train_models(user_data_train, user_data_val, sequence_lengths=[sequence_length], model_type=model_type) - - results = pd.concat([results, - evaluate_model_on_test_data(model=best_models[sequence_length]['model'], - test_df=te, split_id=split_id, - sequence_length=sequence_length, - time_span_id=timespan_id, - threshold_id=threshold_id, - model_type=model_type)], ignore_index=True) - results.to_json(result_filename_v1) - -# === Evaluation === -def evaluate_model_on_test_data(model, test_df,sequence_length, split_id, threshold_id, time_span_id, model_type): - user_data = prepare_user_data(test_df) - x, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length) - - y_pred = model.predict(x, verbose=0) - y_pred_classes = np.argmax(y_pred, axis=1) - - recall = sklearn.metrics.recall_score(y, y_pred_classes, average='weighted') - precision = sklearn.metrics.precision_score(y, y_pred_classes, average='weighted') - f1_score = sklearn.metrics.f1_score(y, y_pred_classes, average='weighted') - return pd.DataFrame({split_str:[split_id], threshold_str:[threshold_id], timespan_str:[time_span_id], - sequence_length_str:[sequence_length], - model_type_str:[model_type], recall_str:[recall], - precision_str:[precision], f1_string:[f1_score]}) - -def visualise_results_v1(): - results = pd.DataFrame(json.load(open(result_filename_v1))) - # Month split ist immer schlechter - results = results[results[split_str] == data_split_str] - with_threshold = results[results[threshold_str] == with_threshold_str] - without_threshold = results[results[threshold_str] == without_threshold_str] - fig, axes = plt.subplots(2, 3) - ax_col_id = 0 - ax_row_id = -1 - for timespan in [hour_timespan_str,min_timespan_str]: - ax_row_id +=1 - for model in [model_type_lstm, model_type_bilstm, model_type_gru]: - with_sub = with_threshold[(with_threshold[timespan_str] == timespan) & (with_threshold[model_type_str] == model)] - without_sub = without_threshold[(without_threshold[timespan_str] == timespan) & (without_threshold[model_type_str] == model)] - ax = axes[ax_row_id, ax_col_id] - ax.set_title(model+' '+timespan) - ax.plot(with_sub[sequence_length_str], with_sub[f1_string], label=with_threshold_str) - ax.plot(without_sub[sequence_length_str], without_sub[f1_string], label=without_threshold_str) - ax.legend() - ax_col_id +=1 - ax_col_id %= 3 - fig.tight_layout() - fig.savefig(figure_path+'v1_results.svg') - # Fazit: keine eindeutig besseren Versionen erkennbar - - -def visualise_results_v2(): - results = pd.DataFrame(json.load(open(result_filename_v2))) - with_threshold = results[results[threshold_str] == with_threshold_str] - without_threshold = results[results[threshold_str] == without_threshold_str] - fig, axes = plt.subplots(2, 3) - ax_col_id = 0 - ax_row_id = -1 - for timespan in [hour_timespan_str,min_timespan_str]: - ax_row_id +=1 - for model in [model_type_lstm, model_type_bilstm, model_type_gru]: - with_sub = with_threshold[(with_threshold[timespan_str] == timespan) & (with_threshold[model_type_str] == model)] - without_sub = without_threshold[(without_threshold[timespan_str] == timespan) & (without_threshold[model_type_str] == model)] - with_sub = with_sub.sort_values(sequence_length_str) - without_sub = without_sub.sort_values(sequence_length_str) - ax = axes[ax_row_id, ax_col_id] - ax.set_title(model+' '+timespan) - ax.plot(with_sub[sequence_length_str], with_sub[f1_string], label=with_threshold_str) - ax.plot(without_sub[sequence_length_str], without_sub[f1_string], label=without_threshold_str) - ax.legend() - ax_col_id +=1 - ax_col_id %= 3 - fig.tight_layout() - fig.savefig(figure_path+'v2_results.svg') - # Fazit: keine eindeutig besseren Versionen erkennbar - - -def test(model_type): - sequence_length = 20 - data_filename = os.listdir(dataset_path)[0] - timespan_id = hour_timespan_str - threshold_id = with_threshold_str - - file_path = os.path.join(dataset_path, data_filename) - df = load_dataset(file_path) - df = remove_covid_data(df) - results = pd.DataFrame() - - for percentage in [33,66,100]: - print('Percentage:', percentage) - tr,val,te = split_data_by_userdata_percentage(df, percentages=(80,10,10),sample=percentage) - tr = reduce_columns(tr, data_filename) - val = reduce_columns(val, data_filename) - te = reduce_columns(te, data_filename) - - user_data_train = prepare_user_data(tr) - user_data_val = prepare_user_data(val) - - best_model = train_models_v2(user_data_train, user_data_val, - sequence_length=sequence_length, - model_type=model_type) - - results = pd.concat([results, - evaluate_model_on_test_data(model=best_model, - test_df=te, - sequence_length=sequence_length, - time_span_id=timespan_id, - threshold_id=threshold_id, - model_type=model_type, - split_id=data_split_str)], - ignore_index=True) - print(results) - -def manual_tuning(model_type): - # load dataset - sequence_length = 20 - data_filename = 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx' - timespan_id = min_timespan_str - threshold_id = with_threshold_str - - file_path = os.path.join(dataset_path, data_filename) - df = load_dataset(file_path) - df = remove_covid_data(df) - - tr, val, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=100) - tr = reduce_columns(tr, data_filename) - val = reduce_columns(val, data_filename) - te = reduce_columns(te, data_filename) - - user_data_train = prepare_user_data(tr) - user_data_val = prepare_user_data(val) - - # fit and evaluate model - # config - repeats = 3 - n_batch = 1024 - n_epochs = 500 - n_neurons = 16 - l_rate = 1e-4 - reg = L1L2(l1=0.0, l2=0.0) - - history_list = list() - # run diagnostic tests - for i in range(repeats): - history = train_one_model(user_data_train, user_data_val, n_batch, n_epochs, - n_neurons, l_rate, reg, - sequence_length=sequence_length, - model_type=model_type) - history_list.append(history) - for metric in ['p', 'r', 'f1']: - for history in history_list: - plt.plot(history['train_'+metric], color='blue') - plt.plot(history['test_'+metric], color='orange') - plt.savefig(figure_path+metric+'_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+ - str(n_batch)+'_l'+str(l_rate)+'_diagnostic.png') - plt.clf() - print('Done') - - -def upsampling(df): - max_user_data = df[user_str].value_counts().max() - for user in df[user_str].unique(): - user_data = df[df[user_str]==user] - user_count = user_data.shape[0] - times = max_user_data / user_count - before_comma = math.floor(times) - after_comma = times % 1 - after_comma_data = user_data.sample(frac=after_comma) - for i in range(1, before_comma): - df = pd.concat([df, user_data], ignore_index=True) - df = pd.concat([df, after_comma_data], ignore_index=True) - return df - - -def manual_tuning_v3(model_type): - # TODO: hrs/min - sequence_length = 1 - - tr, val, te = get_prepared_data_v3(dataset_hrs_path) - - # fit and evaluate model - # config - repeats = 3 - n_batch = 1024 - n_epochs = 10 - n_neurons = 256 - n_neurons2 = 512 - n_neurons3 = 512 - n_neurons4 = 128 - l_rate = 1e-2 - d1 = 256 - reg1 = L1L2(l1=0.0, l2=0.001) - r1 = '0001' - reg2 = L1L2(l1=0.0, l2=0.1) - r2 = '01' - - history_list = list() - # run diagnostic tests - for i in range(repeats): - history = train_one_model(tr, val, n_batch, n_epochs, - n_neurons,n_neurons2, n_neurons3, n_neurons4, l_rate, d1, r1, reg1, r2, reg2, - sequence_length=sequence_length, - model_type=model_type) - history_list.append(history) - for metric in ['acc', 'p', 'r', 'f1']: - for history in history_list: - plt.plot(history['train_'+metric], color='blue') - plt.plot(history['test_'+metric], color='orange') - plt.savefig(figure_path+'v3/'+metric+get_save_id(n_epochs, n_neurons, n_neurons2, n_neurons3,n_neurons4, n_batch, l_rate, d1, r1, r2) - +'.png') - plt.clf() - print('Done') - - - -def calculate_baselines(): - file_combinations = [(hour_timespan_str, with_threshold_str,'ALL32USERS1HR_WITHTHRESHOLD.xlsx'), - (min_timespan_str, with_threshold_str, 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx'), - (min_timespan_str, without_threshold_str, 'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx'), - (hour_timespan_str, without_threshold_str, 'ALLUSERS_32_1HR_WITHOUT_THRESHOLD.xlsx'), - ] - baseline_res = pd.DataFrame() - for timespan_id, threshold_id, filename in file_combinations: - file_path = os.path.join(dataset_path, filename) - df = load_dataset(file_path) - df = remove_covid_data(df) - - _, _, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=20) - te = reduce_columns(te, filename) - user_data_te = prepare_user_data(te) - for sequence_length in range(5,30, 5): - x, y = prepare_data_for_model(user_data=user_data_te, sequence_length=sequence_length) - - for strategy in ['most_frequent', 'stratified', 'uniform']: - cls = DummyClassifier(strategy=strategy) - cls.fit(x,y) - y_pred = cls.predict(x) - acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred) - baseline_res = pd.concat([baseline_res, - DataFrame({ 'strategy':[strategy], threshold_str:[threshold_id], - timespan_str:[timespan_id], sequence_length_str:[sequence_length], - accuracy_str:[acc],precision_str:[p],recall_str:[r], - f1_string:f1})], ignore_index=True) - baseline_res.to_json('baseline_results.json') - print('Done') +def filter_and_preprocess_data(filename, sample=100, print_unique=False): + """ + Preprocesses data. Removes users with too little data or which subsume another, bins the step data and creates train, valid, test splits -def get_prepared_data_v3(filename, sample=100): + :param filename: Name of the file for loading + :param sample: percentage of the sample in case less data is wanted (e.g., for testing) + :param print_unique: To print the number of unique users + :return: train, valid, test splits as dicts from user_id to dataframe + """ df = pd.read_json(filename) df = remove_covid_data(df) - # remove users with too little data (optional) - value_counts = df[user_str].value_counts() - # df = df[df[user_str].isin(value_counts[value_counts>1000].index)] - - adjusted_df = pd.DataFrame() - # adjust labels - new_id = 0 - for user_id in df[user_str].unique(): - user_data = df[df[user_str]==user_id] - user_data[user_str] = new_id - adjusted_df = pd.concat([adjusted_df, user_data], ignore_index=True) - new_id += 1 + # remove users which are a complete subset of another user (but keep one) + users_to_remove = [] + for user_a in df[user_str].unique(): + for user_b in df[user_str].unique(): + if user_a != user_b: + data = pd.concat([df[df[user_str]==user_a], df[df[user_str]==user_b]]) + columns = data.columns.tolist() + columns.remove(user_str) + + no_dup = data.drop_duplicates(columns, keep=False) + if len(no_dup[no_dup[user_str]==user_a]) == 0: + if print_unique: + print(user_a, 'is subset of',user_b) + if user_b not in users_to_remove: + users_to_remove.append(user_a) + df = df[~df[user_str].isin(users_to_remove)] # bin steps per hour TODO: adjust for minutes for hour in ['Hour_'+str(i) for i in range(24)]: - hour_data = adjusted_df[hour] + hour_data = df[hour] # smaller 1000 - round to 10 a = ((hour_data[hour_data<1000]/10).round()*10) # between 1000 and 10000 - round to next 100 @@ -507,66 +129,306 @@ def get_prepared_data_v3(filename, sample=100): c = hour_data[hour_data > 10000] c = pd.Series(data={ind:10000 for ind in c.index}, index=c.index) new = pd.concat([a, b, c]).sort_index().astype(int) - adjusted_df[hour] = new - - tr, val, te = split_data_by_userdata_percentage(adjusted_df, percentages=(70, 15, 15), sample=sample) - tr = reduce_columns_v3(tr) - val = reduce_columns_v3(val) - te = reduce_columns_v3(te) - + df[hour] = new + + # remove users with too little data + min_datapoints = 500 # 500 leads to at least 75 datapoints in the valid set + users_to_remove = set() + cols = df.columns.tolist() + cols.remove(user_str) + reduced = df.drop_duplicates(subset=cols, keep=False) + for user_id in df[user_str].unique(): + subset = df[df[user_str] == user_id] + reduced_subset = reduced[reduced[user_str] == user_id] + if print_unique: + print(user_id, len(subset), len(reduced_subset)) + if len(reduced_subset) < min_datapoints: + users_to_remove.add(user_id) + if print_unique: + print('removing', user_id) + df = df[~df[user_str].isin(users_to_remove)] + + tr, val, te = split_data_by_userdata_percentage(df, percentages=(70, 15, 15), sample=sample) + tr = reduce_columns(tr) + val = reduce_columns(val) + te = reduce_columns(te) + + if print_unique: + print('Train: Users', len(tr[user_str].unique()), 'mean num datapoins:', tr[user_str].value_counts().mean()) + print('Valid: Users', len(val[user_str].unique()), 'mean num datapoins:', val[user_str].value_counts().mean()) + print('Test: Users', len(te[user_str].unique()), 'mean num datapoins:', te[user_str].value_counts().mean()) + + tr, val, te = add_features(tr), add_features(val), add_features(te) scaler = MinMaxScaler() scaler.fit(tr.drop(columns=[user_str])) return scale_dataset(scaler, tr), scale_dataset(scaler, val), scale_dataset(scaler, te) - def scale_dataset(scaler, df): + """ + Data scaling function + :param scaler: The scaler object + :param df: data to scale + :return: the scaled data + """ y = df[user_str] x_scaled = scaler.transform(df.drop(columns=[user_str])) + x_scaled = pd.DataFrame(x_scaled) + x_scaled.columns = df.drop(columns=[user_str]).columns - df_scaled = pd.concat([pd.DataFrame(x_scaled), pd.DataFrame(y)], axis=1) - df_scaled.columns = df.columns - return prepare_user_data(df_scaled) + df_scaled = pd.concat([x_scaled, pd.DataFrame(y.reset_index()[user_str])], axis=1) + return convert_to_user_dict(df_scaled) +def convert_to_user_dict(df): + """ + Converts the dataframe to a dict of dataframes with the key the user id + + :param df: Complete dataframe + :return: the dict of dataframes + """ + users = df[user_str].unique() + return {user: df[df[user_str] == user] for user in users} -def calculate_baselines_v3(): +def calculate_baselines(): + """ + Calculates very simple baselines for the scenario and saves them + :return the calculated baselines + """ file_combinations = [(hour_timespan_str, dataset_hrs_path), - # (min_timespan_str, dataset_min_path), # TODO: dataset bining not ready for minutes + # (min_timespan_str, dataset_min_path), # TODO: dataset binning not ready for minutes yet, rerun this method when that works ] - baseline_res = pd.DataFrame() - for timespan_id, filename in file_combinations: - _, _, te = get_prepared_data_v3(filename) - for sequence_length in range(1,30,5): - x, y = prepare_data_for_model(user_data=te, sequence_length=sequence_length) - - for strategy in ['most_frequent', 'stratified', 'uniform']: - cls = DummyClassifier(strategy=strategy) - cls.fit(x,y) - y_pred = cls.predict(x) - acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred) - baseline_res = pd.concat([baseline_res, - DataFrame({ 'strategy':[strategy], - timespan_str:[timespan_id], sequence_length_str:[sequence_length], - accuracy_str:[acc],precision_str:[p],recall_str:[r], - f1_string:f1})], ignore_index=True) - baseline_res.to_json('baseline_results_v3.json') + str_result_filename = 'baseline_results.json' + if os.path.exists(str_result_filename): + baseline_res = pd.read_json(str_result_filename) + else: + baseline_res = pd.DataFrame() + + for timespan_id, filename in file_combinations: + _, _, te = filter_and_preprocess_data(filename) + for sequence_length in range(1,30,5): + x, y = prepare_data_for_neural_model(user_data=te, sequence_length=sequence_length) + + for strategy in ['most_frequent', 'stratified', 'uniform']: + cls = DummyClassifier(strategy=strategy) + cls.fit(x,y) + y_pred = cls.predict(x) + acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred) + baseline_res = pd.concat([baseline_res, + DataFrame({ 'strategy':[strategy], + timespan_str:[timespan_id], sequence_length_str:[sequence_length], + accuracy_str:[acc],precision_str:[p],recall_str:[r], + f1_string:f1})], ignore_index=True) + baseline_res.to_json(str_result_filename) + return baseline_res + +def hypertune_basic_algorithms(): + """ + Function can be used to hypertune basic sklearn algorithms. Takes very long. + """ + # TODO: mnake it run for minutes, iterate over sequence lengths + sequence_length = 7 + + tr, val, te = filter_and_preprocess_data(dataset_hrs_path) + + x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) + x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) + + random_state = 17 + results = pd.DataFrame() + for tag, clf, grid in [ + ('GradientBoosting', GradientBoostingClassifier(random_state=random_state), + {'loss': ['log_loss', 'exponential'], + 'learning_rate': [0.1, 0.5, 1.0,2.0, 5.0], + 'n_estimators': [10, 50, 100, 150, 200], + 'subsample': [0.1, 0.5, 1.0], + 'criterion': ['friedman_mse', 'squared_error'], + 'min_samples_split': [2, 10, 100], + 'min_samples_leaf': [1, 5, 10], + 'min_weight_fraction_leaf': [0.0, 0.1, 0.5], + 'max_depth': [None, 2, 10, 100], + 'min_impurity_decrease': [0.0, 0.1, 0.5], + 'max_features': ['sqrt', 'log2', None, 10, 20], + 'max_leaf_nodes': [None, 1, 5, 10], + }), + ('Bernoulli', BernoulliNB(), {'fit_prior': [True, False], + 'binarize': [0.0, 0.1, 0.25, 0.5, 0.75], + 'force_alpha': [True, False], + 'alpha':[0.0, 0.25, 0.5, 0.75, 1.0]}), + ('extra trees', ExtraTreesClassifier(random_state=random_state, n_jobs=1), + {'n_estimators': [10, 50, 100, 150, 200], + 'criterion': ['gini', 'entropy', 'log_loss'], + 'max_depth': [None, 2, 10, 100], + 'min_samples_split': [2, 10, 100], + 'min_samples_leaf': [1, 5, 10], + 'min_weight_fraction_leaf': [0.0, 0.1, 0.5], + 'max_features': ['sqrt', 'log2', None, 10, 20], + 'max_leaf_nodes': [None, 1, 5, 10], + 'min_impurity_decrease': [0.0, 0.1, 0.5], + 'bootstrap': [True, False], + 'class_weight': [None, 'balanced', 'balanced_subsample'], + 'max_samples': [None, 0.1, 0.2, 0.3]} + ), + ('random forest', RandomForestClassifier(random_state=random_state, n_jobs=1), + {'n_estimators':[10, 50, 100, 150, 200], + 'criterion':['gini', 'entropy', 'log_loss'], + 'max_depth':[None, 2, 10,100], + 'min_samples_split': [2,10,100], + 'min_samples_leaf':[1,5,10], + 'min_weight_fraction_leaf':[0.0,0.1, 0.5], + 'max_features':['sqrt', 'log2', None, 10, 20], + 'max_leaf_nodes':[None, 1, 5, 10], + 'min_impurity_decrease':[0.0, 0.1, 0.5], + 'bootstrap':[True, False], + 'class_weight':[None, 'balanced', 'balanced_subsample'], + 'max_samples':[None,0.1, 0.2, 0.3]}) + ]: + grid_search = GridSearchCV( + estimator=clf, param_grid=grid, scoring='f1_weighted', cv=5, n_jobs=1) + grid_search.fit(x_tr, y_tr) + + best_model = grid_search.best_estimator_ + y_pred = best_model.predict(x_val) + acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) + results = pd.concat([results, DataFrame({ 'params': str(grid_search.best_params_), + 'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) + results.to_json('basic_ht_results.json') print('Done') +def test_basic_algorithms(): + """ + Method for testing basic algorithms from the sklearn library. Tested more, but those 4 were the best + :return: The calculated values + """ + # TODO: also check for minutes + # TODO: iterate over sequence lengths + sequence_length = 21 + + tr, val, te = filter_and_preprocess_data(dataset_hrs_path) + + x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) + x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) + + random_state = 17 + results = pd.DataFrame() + for tag, clf in [ + ('Bernoulli', BernoulliNB()), + ('GradientBoosting', GradientBoostingClassifier(random_state=random_state)), + ('extra trees', ExtraTreesClassifier(random_state=random_state)), + ('random forest', RandomForestClassifier(random_state=random_state)) + ]: + clf.fit(x_tr, y_tr) + y_pred = clf.predict(x_val) + acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) + results = pd.concat([results, DataFrame({ 'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) + return results + +def add_features(df): + """ + Functiion for adding additional features to the dataframe + + :param df: dataframe + :return: dataframe with features added + """ + # indicator weekend + df['weekend'] = df[day_of_week_str + '_Saturday']+df[day_of_week_str + '_Sunday'] + # sum of steps per day + df['day_total'] = sum([df['Hour_'+str(i)] for i in range(23)]) + # sum of steps morning, afternoon, evening, night + df['morning_total'] = sum([df['Hour_' + str(i)] for i in range(6,13)]) + df['afternoon_total'] = sum([df['Hour_' + str(i)] for i in range(13,19)]) + df['evening_total'] = sum([df['Hour_' + str(i)] for i in range(19,23)]) + df['night_total'] = sum([df['Hour_' + str(i)] for i in [23,0,1,2,3,4,5]]) + return df + +def test_basic_algorithm_on_sequence_lengths(clf = RandomForestClassifier(random_state=17)): + """ + Runs a basic algorith from scikit learn on different sequence lengths. Plots an image for the results. + :param clf: the sklearn classifier + """ + tr, val, te = filter_and_preprocess_data(dataset_hrs_path) + + results_train = pd.DataFrame() + results_valid = pd.DataFrame() + # iterate over sequence lengths + for sequence_length in range(1, 60, 5): + x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) + x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) + clf.fit(x_tr, y_tr) + acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=clf.predict(x_val)) + results_valid = pd.concat([results_valid, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) + acc, p, r, f1 = eval_metrics(y_true=y_tr, y_pred=clf.predict(x_tr)) + results_train = pd.concat([results_train, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) + + fig = plt.figure() + + for frame in [results_train, results_valid]: + plt.plot(frame[sequence_length_str], frame[f1_string]) + + plt.show() + print('') + +def tune_neural_network(model_type): + """ + Tunes a neural model for different sequence lengths. Plots the results + :param model_type: either lstm, gru or bilstm, use set strings + """ + # TODO: Also do this for minute data + tr, val, te = filter_and_preprocess_data(dataset_hrs_path) + n_epochs= 20 + n_neurons = 1024 + n_batch = 1024 + results_train = pd.DataFrame() + results_valid = pd.DataFrame() + # iterate over sequence lengths + for sequence_length in range(1, 50, 5): + train_data = prepare_data_for_neural_model(user_data=tr, sequence_length=sequence_length) + val_data = prepare_data_for_neural_model(user_data=val, sequence_length=sequence_length) + + # fit and evaluate model + history_list = list() + repeats = 3 + # run diagnostic tests + for i in range(repeats): + history = train_one_model(train_data, val_data, n_batch, n_epochs, + n_neurons, sequence_length=sequence_length, + model_type=model_type) + history_list.append(history) + results = pd.concat([history.tail(1) for history in history_list]).mean() + results_train = pd.concat([results_train, + DataFrame({sequence_length_str:[sequence_length], + accuracy_str:[results['train_acc']], + precision_str:[results['train_p']], + recall_str:[results['train_r']], + f1_string:[results['train_f1']]})], ignore_index=True) + results_valid = pd.concat([results_valid, + DataFrame({sequence_length_str:[sequence_length], + accuracy_str:[results['test_acc']], + precision_str:[results['test_p']], + recall_str:[results['test_r']], + f1_string:[results['test_f1']]})], ignore_index=True) + + fig = plt.figure() + for frame in [results_train, results_valid]: + plt.plot(frame[sequence_length_str], frame[f1_string]) + + plt.show() + print('Done') + + if __name__ == "__main__": - # Ordner erstellen, die benötigt werden + # create needed directories create_dir('results/') create_dir(figure_path) + pd.options.mode.copy_on_write = True + + baselines = calculate_baselines() + + # use basic algorithms from scikit learn + test_basic_algorithms() + hypertune_basic_algorithms() + test_basic_algorithm_on_sequence_lengths() - # main_two_v1() - # visualise_results_v1() - #test(model_type=model_type_gru) - # main_two_v2(model_type=model_type_gru) - #visualise_results_v2() - #manual_tuning(model_type=model_type_lstm) - #calculate_baselines() - - #### Ab hier aktuell (21.01.2026) - #calculate_baselines_v3() - manual_tuning_v3(model_type=model_type_lstm) + # tune a neural network + tune_neural_network(model_type=model_type_lstm) print('Done') \ No newline at end of file diff --git a/DATA_PREPROCESSING_CODE.ipynb b/old/DATA_PREPROCESSING_CODE.ipynb similarity index 100% rename from DATA_PREPROCESSING_CODE.ipynb rename to old/DATA_PREPROCESSING_CODE.ipynb diff --git a/MODIFICATIONSANDINSTRUCTIONS.pdf b/old/MODIFICATIONSANDINSTRUCTIONS.pdf similarity index 100% rename from MODIFICATIONSANDINSTRUCTIONS.pdf rename to old/MODIFICATIONSANDINSTRUCTIONS.pdf diff --git a/old/baseline_results.json b/old/baseline_results.json new file mode 100644 index 0000000..df8a859 --- /dev/null +++ b/old/baseline_results.json @@ -0,0 +1 @@ +{"strategy":{"0":"most_frequent","1":"stratified","2":"uniform","3":"most_frequent","4":"stratified","5":"uniform","6":"most_frequent","7":"stratified","8":"uniform","9":"most_frequent","10":"stratified","11":"uniform","12":"most_frequent","13":"stratified","14":"uniform","15":"most_frequent","16":"stratified","17":"uniform","18":"most_frequent","19":"stratified","20":"uniform","21":"most_frequent","22":"stratified","23":"uniform","24":"most_frequent","25":"stratified","26":"uniform","27":"most_frequent","28":"stratified","29":"uniform","30":"most_frequent","31":"stratified","32":"uniform","33":"most_frequent","34":"stratified","35":"uniform","36":"most_frequent","37":"stratified","38":"uniform","39":"most_frequent","40":"stratified","41":"uniform","42":"most_frequent","43":"stratified","44":"uniform","45":"most_frequent","46":"stratified","47":"uniform","48":"most_frequent","49":"stratified","50":"uniform","51":"most_frequent","52":"stratified","53":"uniform","54":"most_frequent","55":"stratified","56":"uniform","57":"most_frequent","58":"stratified","59":"uniform"},"threshold used":{"0":"WITH","1":"WITH","2":"WITH","3":"WITH","4":"WITH","5":"WITH","6":"WITH","7":"WITH","8":"WITH","9":"WITH","10":"WITH","11":"WITH","12":"WITH","13":"WITH","14":"WITH","15":"WITH","16":"WITH","17":"WITH","18":"WITH","19":"WITH","20":"WITH","21":"WITH","22":"WITH","23":"WITH","24":"WITH","25":"WITH","26":"WITH","27":"WITH","28":"WITH","29":"WITH","30":"WITHOUT","31":"WITHOUT","32":"WITHOUT","33":"WITHOUT","34":"WITHOUT","35":"WITHOUT","36":"WITHOUT","37":"WITHOUT","38":"WITHOUT","39":"WITHOUT","40":"WITHOUT","41":"WITHOUT","42":"WITHOUT","43":"WITHOUT","44":"WITHOUT","45":"WITHOUT","46":"WITHOUT","47":"WITHOUT","48":"WITHOUT","49":"WITHOUT","50":"WITHOUT","51":"WITHOUT","52":"WITHOUT","53":"WITHOUT","54":"WITHOUT","55":"WITHOUT","56":"WITHOUT","57":"WITHOUT","58":"WITHOUT","59":"WITHOUT"},"time used":{"0":"1HR","1":"1HR","2":"1HR","3":"1HR","4":"1HR","5":"1HR","6":"1HR","7":"1HR","8":"1HR","9":"1HR","10":"1HR","11":"1HR","12":"1HR","13":"1HR","14":"1HR","15":"15MIN","16":"15MIN","17":"15MIN","18":"15MIN","19":"15MIN","20":"15MIN","21":"15MIN","22":"15MIN","23":"15MIN","24":"15MIN","25":"15MIN","26":"15MIN","27":"15MIN","28":"15MIN","29":"15MIN","30":"15MIN","31":"15MIN","32":"15MIN","33":"15MIN","34":"15MIN","35":"15MIN","36":"15MIN","37":"15MIN","38":"15MIN","39":"15MIN","40":"15MIN","41":"15MIN","42":"15MIN","43":"15MIN","44":"15MIN","45":"1HR","46":"1HR","47":"1HR","48":"1HR","49":"1HR","50":"1HR","51":"1HR","52":"1HR","53":"1HR","54":"1HR","55":"1HR","56":"1HR","57":"1HR","58":"1HR","59":"1HR"},"sequence length":{"0":5,"1":5,"2":5,"3":10,"4":10,"5":10,"6":15,"7":15,"8":15,"9":20,"10":20,"11":20,"12":25,"13":25,"14":25,"15":5,"16":5,"17":5,"18":10,"19":10,"20":10,"21":15,"22":15,"23":15,"24":20,"25":20,"26":20,"27":25,"28":25,"29":25,"30":5,"31":5,"32":5,"33":10,"34":10,"35":10,"36":15,"37":15,"38":15,"39":20,"40":20,"41":20,"42":25,"43":25,"44":25,"45":5,"46":5,"47":5,"48":10,"49":10,"50":10,"51":15,"52":15,"53":15,"54":20,"55":20,"56":20,"57":25,"58":25,"59":25},"accuracy":{"0":0.0527306968,"1":0.0357815443,"2":0.0301318267,"3":0.061827957,"4":0.0376344086,"5":0.0510752688,"6":0.0789473684,"7":0.048245614,"8":0.0350877193,"9":0.1056910569,"10":0.0487804878,"11":0.0406504065,"12":0.1454545455,"13":0.0909090909,"14":0.0727272727,"15":0.0527306968,"16":0.0301318267,"17":0.0376647834,"18":0.061827957,"19":0.0322580645,"20":0.0215053763,"21":0.0789473684,"22":0.048245614,"23":0.0394736842,"24":0.1056910569,"25":0.0731707317,"26":0.081300813,"27":0.1454545455,"28":0.1272727273,"29":0.1454545455,"30":0.0527306968,"31":0.0602636535,"32":0.0414312618,"33":0.061827957,"34":0.0403225806,"35":0.0376344086,"36":0.0789473684,"37":0.048245614,"38":0.0350877193,"39":0.1056910569,"40":0.0406504065,"41":0.0975609756,"42":0.1454545455,"43":0.1818181818,"44":0.0545454545,"45":0.0527306968,"46":0.0508474576,"47":0.0433145009,"48":0.061827957,"49":0.0268817204,"50":0.0268817204,"51":0.0789473684,"52":0.0526315789,"53":0.0438596491,"54":0.1056910569,"55":0.0975609756,"56":0.0487804878,"57":0.1454545455,"58":0.1090909091,"59":0.1090909091},"precision":{"0":0.0027805264,"1":0.0341690256,"2":0.0311942747,"3":0.0038226963,"4":0.0351651701,"5":0.0683324292,"6":0.006232687,"7":0.0522057707,"8":0.0467348137,"9":0.0111705995,"10":0.0374479219,"11":0.0413279133,"12":0.0211570248,"13":0.0916017316,"14":0.0688888889,"15":0.0027805264,"16":0.0274117382,"17":0.0420429845,"18":0.0038226963,"19":0.0286652192,"20":0.0284115997,"21":0.006232687,"22":0.0518483709,"23":0.0421175767,"24":0.0111705995,"25":0.0739379861,"26":0.110298103,"27":0.0211570248,"28":0.1035353535,"29":0.1476262626,"30":0.0027805264,"31":0.0578291271,"32":0.0451266983,"33":0.0038226963,"34":0.0372897448,"35":0.0396207093,"36":0.006232687,"37":0.0432283852,"38":0.0377911691,"39":0.0111705995,"40":0.0459930314,"41":0.0966382759,"42":0.0211570248,"43":0.1691460055,"44":0.0541125541,"45":0.0027805264,"46":0.0482160586,"47":0.0444993315,"48":0.0038226963,"49":0.0246880889,"50":0.0267291641,"51":0.006232687,"52":0.051154457,"53":0.0653357722,"54":0.0111705995,"55":0.104870739,"56":0.0836720867,"57":0.0211570248,"58":0.1104978355,"59":0.0940981241},"recall":{"0":0.0527306968,"1":0.0357815443,"2":0.0301318267,"3":0.061827957,"4":0.0376344086,"5":0.0510752688,"6":0.0789473684,"7":0.048245614,"8":0.0350877193,"9":0.1056910569,"10":0.0487804878,"11":0.0406504065,"12":0.1454545455,"13":0.0909090909,"14":0.0727272727,"15":0.0527306968,"16":0.0301318267,"17":0.0376647834,"18":0.061827957,"19":0.0322580645,"20":0.0215053763,"21":0.0789473684,"22":0.048245614,"23":0.0394736842,"24":0.1056910569,"25":0.0731707317,"26":0.081300813,"27":0.1454545455,"28":0.1272727273,"29":0.1454545455,"30":0.0527306968,"31":0.0602636535,"32":0.0414312618,"33":0.061827957,"34":0.0403225806,"35":0.0376344086,"36":0.0789473684,"37":0.048245614,"38":0.0350877193,"39":0.1056910569,"40":0.0406504065,"41":0.0975609756,"42":0.1454545455,"43":0.1818181818,"44":0.0545454545,"45":0.0527306968,"46":0.0508474576,"47":0.0433145009,"48":0.061827957,"49":0.0268817204,"50":0.0268817204,"51":0.0789473684,"52":0.0526315789,"53":0.0438596491,"54":0.1056910569,"55":0.0975609756,"56":0.0487804878,"57":0.1454545455,"58":0.1090909091,"59":0.1090909091},"f1 score":{"0":0.0052825027,"1":0.0347429612,"2":0.0299888052,"3":0.0072002178,"4":0.0361083348,"5":0.0552944412,"6":0.0115532734,"7":0.0496540923,"8":0.0385668014,"9":0.0202056432,"10":0.0413486151,"11":0.0391147245,"12":0.0369408369,"13":0.0883746556,"14":0.0641479732,"15":0.0052825027,"16":0.0285925373,"17":0.0384109852,"18":0.0072002178,"19":0.0301958901,"20":0.0233679998,"21":0.0115532734,"22":0.0495366536,"23":0.0370203194,"24":0.0202056432,"25":0.07241401,"26":0.0810068644,"27":0.0369408369,"28":0.1110889111,"29":0.1433100233,"30":0.0052825027,"31":0.0580513337,"32":0.0421405072,"33":0.0072002178,"34":0.0385343397,"35":0.0365603478,"36":0.0115532734,"37":0.0453510925,"38":0.0329189581,"39":0.0202056432,"40":0.0428921458,"41":0.0874960487,"42":0.0369408369,"43":0.1680995475,"44":0.0541046832,"45":0.0052825027,"46":0.0491226418,"47":0.0423109157,"48":0.0072002178,"49":0.0255968184,"50":0.0261026052,"51":0.0115532734,"52":0.0513571421,"53":0.0484257097,"54":0.0202056432,"55":0.099825225,"56":0.0598149686,"57":0.0369408369,"58":0.1061904762,"59":0.0965892693}} \ No newline at end of file diff --git a/data_preprocessing.py b/old/data_preprocessing.py similarity index 100% rename from data_preprocessing.py rename to old/data_preprocessing.py diff --git a/data_preprocessing_main.py b/old/data_preprocessing_main.py similarity index 100% rename from data_preprocessing_main.py rename to old/data_preprocessing_main.py diff --git a/final-32-automated-code-new(1).ipynb b/old/final-32-automated-code-new(1).ipynb similarity index 100% rename from final-32-automated-code-new(1).ipynb rename to old/final-32-automated-code-new(1).ipynb diff --git a/old/main_old.py b/old/main_old.py new file mode 100644 index 0000000..ae2c697 --- /dev/null +++ b/old/main_old.py @@ -0,0 +1,848 @@ +import json +import os + +import math +import numpy as np +import pandas as pd +import sklearn +from keras.src.regularizers import L1L2 +from matplotlib import pyplot as plt +from pandas import DataFrame +from sklearn.calibration import CalibratedClassifierCV +from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis, LinearDiscriminantAnalysis +from sklearn.dummy import DummyClassifier +from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier, VotingClassifier, \ + GradientBoostingClassifier, AdaBoostClassifier +from sklearn.gaussian_process import GaussianProcessClassifier +from sklearn.linear_model import PassiveAggressiveClassifier, RidgeClassifier, RidgeClassifierCV, SGDClassifier, \ + LogisticRegression, LogisticRegressionCV, Perceptron +from sklearn.metrics import confusion_matrix +from sklearn.mixture import GaussianMixture +from sklearn.model_selection import GridSearchCV +from sklearn.naive_bayes import GaussianNB, BernoulliNB, MultinomialNB +from sklearn.neighbors import KNeighborsClassifier, NearestCentroid +from sklearn.neural_network import MLPClassifier +from sklearn.preprocessing import MinMaxScaler +from sklearn.semi_supervised import LabelSpreading, LabelPropagation +from sklearn.svm import LinearSVC, SVC, OneClassSVM +from sklearn.tree import ExtraTreeClassifier, DecisionTreeClassifier + +from pipeline_old import ( + load_dataset, + filter_data, + filter_test_data, + prepare_user_data, + train_models, + evaluate_models, + prepare_data_for_model, model_type_gru, model_type_lstm, model_type_bilstm, train_models_v2, train_one_model, + eval_metrics, get_save_id, prepare_data_for_basic_algorithm, train_one_model_v2, +) + +year_str = 'Year' +month_str = 'Month' +day_str = 'Day' +date_str = 'Date' +time_str = 'Time' +day_of_week_str = 'DayOfWeek' +user_str = 'user' +split_str = 'split type' +data_split_str = 'data percentages' +month_split_str = 'month percentages' +threshold_str = 'threshold used' +with_threshold_str = 'WITH' +without_threshold_str = 'WITHOUT' +timespan_str = 'time used' +hour_timespan_str = '1HR' +min_timespan_str = '15MIN' +sequence_length_str = 'sequence length' +accuracy_str = 'accuracy' +precision_str = 'precision' +recall_str = 'recall' +f1_string = 'f1 score' +model_type_str = 'model type' +week_column_names = ['DayOfWeek_' + day for day in + ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday' ]] +figure_path = 'figures/' +predicitons_path = 'preds/' + +# === Configurable Parameters === +dataset_path = './Datasets/' +dataset_hrs_path = './Datasets/hours.json' +dataset_min_path = './Datasets/minutes.json' +DATA_PATH = dataset_path +'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx' +OUTPUT_EXCEL_PATH = './working/evaluation_results.xlsx' +result_filename_v1 = './working/evaluation_results.json' +result_filename_v2 = './working/evaluation_results_v2.json' +SEQUENCE_LENGTHS = [30, 25, 20, 15, 10, 5] # You can add more: [20, 25, 30] + +TRAINING_SCENARIO = [(2018, list(range(1, 13))), (2019, list(range(1, 10)))] +VALIDATION_SCENARIO = [(2019, [10, 11, 12])] +TEST_SCENARIO = [(2020, [1, 2])] # Jan–Feb 2020 only + +# === Optional display only === +predefined_training_scenarios = { + "Scenario 1": {"years_months": [(2018, list(range(1, 13))), (2019, list(range(1, 10)))]}, + "Scenario 2": {"years_months": [(2017, list(range(1, 13))), (2018, list(range(1, 13))), (2019, list(range(1, 10)))]} +} +predefined_validation_scenarios = { + "Scenario A": {"years_months": [(2019, [10, 11, 12])]} +} + +def create_dir(path): + """ + Creates a directory if it doesn't exist yet. + + :param path: The path to the directory + """ + if not os.path.exists(path): + os.makedirs(path) + +def remove_covid_data(df): + df = df[~(df[year_str]>=2020)] + return df + +def split_data_by_month_percentage(df, percentages): + train_p, valid_p, test_p = percentages + ids = df[[year_str, month_str]].drop_duplicates().sort_values([year_str, month_str]) + tr, va, te = np.split(ids, [int((train_p/100) * len(ids)), int(((train_p + valid_p)/100) * len(ids))]) + return df.merge(tr, on=[year_str, month_str], how='inner'), df.merge(va, on=[year_str, month_str], how='inner'), df.merge(te, on=[year_str, month_str], how='inner') + +def split_data_by_userdata_percentage(df, percentages, sample=100): + train_p, valid_p, test_p = percentages + tr, va, te = pd.DataFrame(), pd.DataFrame(), pd.DataFrame() + for user_id in df[user_str].unique(): + # !! following sample creates gaps in data if sample smaller 100 + user_data = df[df[user_str]==user_id].sample(frac=sample/ 100).sort_values([date_str]) # have to sort for time shift + u_tr, u_va, u_te = np.split(user_data, [int((train_p/100)*len(user_data)), int(((train_p+valid_p)/100)*len(user_data))]) + tr = pd.concat([tr, u_tr], ignore_index=True) + va = pd.concat([va, u_va], ignore_index=True) + te = pd.concat([te, u_te], ignore_index=True) + return tr, va, te + + +def main(): + # print("=== Training Scenario Setup ===") + # display_warning_about_2020_data() + # display_warnings_for_scenarios("training", predefined_training_scenarios, predefined_validation_scenarios) + + # print("\n=== Validation Scenario Setup ===") + # display_warning_about_2020_data() + # display_warnings_for_scenarios("validation", predefined_training_scenarios, predefined_validation_scenarios) + + # === Load and preprocess === + df = load_dataset(DATA_PATH) + + ALLUSERS32_15MIN_WITHOUTTHREHOLD = False + if('ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx' in DATA_PATH): + ALLUSERS32_15MIN_WITHOUTTHREHOLD = True + + training_data = filter_data(df, TRAINING_SCENARIO, ALLUSERS32_15MIN_WITHOUTTHREHOLD) + validation_data = filter_data(df, VALIDATION_SCENARIO, ALLUSERS32_15MIN_WITHOUTTHREHOLD) + + user_data_train = prepare_user_data(training_data) + user_data_val = prepare_user_data(validation_data) + + # === Train models === + best_models = train_models(user_data_train, user_data_val, sequence_lengths=SEQUENCE_LENGTHS) + + # === Load and evaluate test === + test_df = filter_test_data(df, TEST_SCENARIO) + evaluate_models(best_models, test_df, SEQUENCE_LENGTHS, OUTPUT_EXCEL_PATH, ALLUSERS32_15MIN_WITHOUTTHREHOLD) + + print(f"\n✅ All evaluations completed. Results saved to: {OUTPUT_EXCEL_PATH}") + + +def reduce_columns(df, filename): + if min_timespan_str in filename: + return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'] + week_column_names, errors='ignore') + else: + return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'], errors='ignore') + + +def reduce_columns_v3(df): + return df.drop(columns=[month_str, year_str, date_str]) + +def load_previous_results(filename): + results = pd.DataFrame() + if os.path.exists(filename): + results = pd.DataFrame(json.load(open(filename))) + return results + +def main_two_v2(model_type): + seq_length = range(10,31, 5) + for sequence_length in seq_length: + for data_filename in os.listdir(dataset_path): + timespan_id = hour_timespan_str + threshold_id = with_threshold_str + if min_timespan_str in data_filename: + timespan_id = min_timespan_str + if without_threshold_str in data_filename: + threshold_id = without_threshold_str + + results = load_previous_results(result_filename_v2) + if len(results) > 0: + if len(results[(results[timespan_str]==timespan_id) & + (results[threshold_str]==threshold_id) & + (results[sequence_length_str]==sequence_length) & + (results[model_type_str]==model_type)]) > 0: + continue + + file_path = os.path.join(dataset_path, data_filename) + df = load_dataset(file_path) + df = remove_covid_data(df) + + tr,val,te = split_data_by_userdata_percentage(df, percentages=(80,10,10)) + tr = reduce_columns(tr, data_filename) + val = reduce_columns(val, data_filename) + te = reduce_columns(te, data_filename) + + user_data_train = prepare_user_data(tr) + user_data_val = prepare_user_data(val) + + best_model = train_models_v2(user_data_train, user_data_val, + sequence_length=sequence_length, + model_type=model_type) + + results = load_previous_results(result_filename_v2) + results = pd.concat([results, + evaluate_model_on_test_data(model=best_model, + test_df=te, + sequence_length=sequence_length, + time_span_id=timespan_id, + threshold_id=threshold_id, + model_type=model_type, + split_id=data_split_str)], + ignore_index=True) + results.to_json(result_filename_v2) + +def main_two_v1(): + seq_length = [30, 25, 20, 15, 10, 5] # You can add more: [20, 25, 30] + results = pd.DataFrame() + if os.path.exists(result_filename_v1): + results = pd.DataFrame(json.load(open(result_filename_v1))) + for sequence_length in seq_length: + for data_filename in os.listdir(dataset_path): + for split_id, split_method in [(data_split_str, split_data_by_userdata_percentage),(month_split_str, split_data_by_month_percentage)]: + for model_type in [model_type_lstm, model_type_bilstm, model_type_gru]: + timespan_id = hour_timespan_str + threshold_id = with_threshold_str + if min_timespan_str in data_filename: + timespan_id = min_timespan_str + if without_threshold_str in data_filename: + threshold_id = without_threshold_str + if len(results) > 0: + if len(results[(results[split_str]==split_id) & + (results[timespan_str]==timespan_id) & + (results[threshold_str]==threshold_id) & + (results[sequence_length_str]==sequence_length) & + (results[model_type_str]==model_type)]) > 0: + continue + + file_path = os.path.join(dataset_path, data_filename) + df = load_dataset(file_path) + df = remove_covid_data(df) + tr,val,te = split_method(df, percentages=(80,10,10)) + tr = reduce_columns(tr, data_filename) + val = reduce_columns(val, data_filename) + te = reduce_columns(te, data_filename) + + user_data_train = prepare_user_data(tr) + user_data_val = prepare_user_data(val) + + best_models = train_models(user_data_train, user_data_val, sequence_lengths=[sequence_length], model_type=model_type) + + results = pd.concat([results, + evaluate_model_on_test_data(model=best_models[sequence_length]['model'], + test_df=te, split_id=split_id, + sequence_length=sequence_length, + time_span_id=timespan_id, + threshold_id=threshold_id, + model_type=model_type)], ignore_index=True) + results.to_json(result_filename_v1) + +# === Evaluation === +def evaluate_model_on_test_data(model, test_df,sequence_length, split_id, threshold_id, time_span_id, model_type): + user_data = prepare_user_data(test_df) + x, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length) + + y_pred = model.predict(x, verbose=0) + y_pred_classes = np.argmax(y_pred, axis=1) + + recall = sklearn.metrics.recall_score(y, y_pred_classes, average='weighted') + precision = sklearn.metrics.precision_score(y, y_pred_classes, average='weighted') + f1_score = sklearn.metrics.f1_score(y, y_pred_classes, average='weighted') + return pd.DataFrame({split_str:[split_id], threshold_str:[threshold_id], timespan_str:[time_span_id], + sequence_length_str:[sequence_length], + model_type_str:[model_type], recall_str:[recall], + precision_str:[precision], f1_string:[f1_score]}) + +def visualise_results_v1(): + results = pd.DataFrame(json.load(open(result_filename_v1))) + # Month split ist immer schlechter + results = results[results[split_str] == data_split_str] + with_threshold = results[results[threshold_str] == with_threshold_str] + without_threshold = results[results[threshold_str] == without_threshold_str] + fig, axes = plt.subplots(2, 3) + ax_col_id = 0 + ax_row_id = -1 + for timespan in [hour_timespan_str,min_timespan_str]: + ax_row_id +=1 + for model in [model_type_lstm, model_type_bilstm, model_type_gru]: + with_sub = with_threshold[(with_threshold[timespan_str] == timespan) & (with_threshold[model_type_str] == model)] + without_sub = without_threshold[(without_threshold[timespan_str] == timespan) & (without_threshold[model_type_str] == model)] + ax = axes[ax_row_id, ax_col_id] + ax.set_title(model+' '+timespan) + ax.plot(with_sub[sequence_length_str], with_sub[f1_string], label=with_threshold_str) + ax.plot(without_sub[sequence_length_str], without_sub[f1_string], label=without_threshold_str) + ax.legend() + ax_col_id +=1 + ax_col_id %= 3 + fig.tight_layout() + fig.savefig(figure_path+'v1_results.svg') + # Fazit: keine eindeutig besseren Versionen erkennbar + + +def visualise_results_v2(): + results = pd.DataFrame(json.load(open(result_filename_v2))) + with_threshold = results[results[threshold_str] == with_threshold_str] + without_threshold = results[results[threshold_str] == without_threshold_str] + fig, axes = plt.subplots(2, 3) + ax_col_id = 0 + ax_row_id = -1 + for timespan in [hour_timespan_str,min_timespan_str]: + ax_row_id +=1 + for model in [model_type_lstm, model_type_bilstm, model_type_gru]: + with_sub = with_threshold[(with_threshold[timespan_str] == timespan) & (with_threshold[model_type_str] == model)] + without_sub = without_threshold[(without_threshold[timespan_str] == timespan) & (without_threshold[model_type_str] == model)] + with_sub = with_sub.sort_values(sequence_length_str) + without_sub = without_sub.sort_values(sequence_length_str) + ax = axes[ax_row_id, ax_col_id] + ax.set_title(model+' '+timespan) + ax.plot(with_sub[sequence_length_str], with_sub[f1_string], label=with_threshold_str) + ax.plot(without_sub[sequence_length_str], without_sub[f1_string], label=without_threshold_str) + ax.legend() + ax_col_id +=1 + ax_col_id %= 3 + fig.tight_layout() + fig.savefig(figure_path+'v2_results.svg') + # Fazit: keine eindeutig besseren Versionen erkennbar + + +def test(model_type): + sequence_length = 20 + data_filename = os.listdir(dataset_path)[0] + timespan_id = hour_timespan_str + threshold_id = with_threshold_str + + file_path = os.path.join(dataset_path, data_filename) + df = load_dataset(file_path) + df = remove_covid_data(df) + results = pd.DataFrame() + + for percentage in [33,66,100]: + print('Percentage:', percentage) + tr,val,te = split_data_by_userdata_percentage(df, percentages=(80,10,10),sample=percentage) + tr = reduce_columns(tr, data_filename) + val = reduce_columns(val, data_filename) + te = reduce_columns(te, data_filename) + + user_data_train = prepare_user_data(tr) + user_data_val = prepare_user_data(val) + + best_model = train_models_v2(user_data_train, user_data_val, + sequence_length=sequence_length, + model_type=model_type) + + results = pd.concat([results, + evaluate_model_on_test_data(model=best_model, + test_df=te, + sequence_length=sequence_length, + time_span_id=timespan_id, + threshold_id=threshold_id, + model_type=model_type, + split_id=data_split_str)], + ignore_index=True) + print(results) + +def manual_tuning(model_type): + # load dataset + sequence_length = 20 + data_filename = 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx' + timespan_id = min_timespan_str + threshold_id = with_threshold_str + + file_path = os.path.join(dataset_path, data_filename) + df = load_dataset(file_path) + df = remove_covid_data(df) + + tr, val, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=100) + tr = reduce_columns(tr, data_filename) + val = reduce_columns(val, data_filename) + te = reduce_columns(te, data_filename) + + user_data_train = prepare_user_data(tr) + user_data_val = prepare_user_data(val) + + # fit and evaluate model + # config + repeats = 3 + n_batch = 1024 + n_epochs = 500 + n_neurons = 16 + l_rate = 1e-4 + reg = L1L2(l1=0.0, l2=0.0) + + history_list = list() + # run diagnostic tests + for i in range(repeats): + history = train_one_model(user_data_train, user_data_val, n_batch, n_epochs, + n_neurons, l_rate, reg, + sequence_length=sequence_length, + model_type=model_type) + history_list.append(history) + for metric in ['p', 'r', 'f1']: + for history in history_list: + plt.plot(history['train_'+metric], color='blue') + plt.plot(history['test_'+metric], color='orange') + plt.savefig(figure_path+metric+'_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+ + str(n_batch)+'_l'+str(l_rate)+'_diagnostic.png') + plt.clf() + print('Done') + + +def upsampling(df): + max_user_data = df[user_str].value_counts().max() + for user in df[user_str].unique(): + user_data = df[df[user_str]==user] + user_count = user_data.shape[0] + times = max_user_data / user_count + before_comma = math.floor(times) + after_comma = times % 1 + after_comma_data = user_data.sample(frac=after_comma) + for i in range(1, before_comma): + df = pd.concat([df, user_data], ignore_index=True) + df = pd.concat([df, after_comma_data], ignore_index=True) + return df + + +def manual_tuning_v3(model_type): + # TODO: hrs/min + sequence_length = 1 + + tr, val, te = get_prepared_data_v3(dataset_hrs_path) + + # fit and evaluate model + # config + repeats = 3 + n_batch = 1024 + n_epochs = 10 + n_neurons = 256 + n_neurons2 = 512 + n_neurons3 = 512 + n_neurons4 = 128 + l_rate = 1e-2 + d1 = 256 + reg1 = L1L2(l1=0.0, l2=0.001) + r1 = '0001' + reg2 = L1L2(l1=0.0, l2=0.1) + r2 = '01' + + history_list = list() + # run diagnostic tests + for i in range(repeats): + history = train_one_model(tr, val, n_batch, n_epochs, + n_neurons,n_neurons2, n_neurons3, n_neurons4, l_rate, d1, r1, reg1, r2, reg2, + sequence_length=sequence_length, + model_type=model_type) + history_list.append(history) + for metric in ['acc', 'p', 'r', 'f1']: + for history in history_list: + plt.plot(history['train_'+metric], color='blue') + plt.plot(history['test_'+metric], color='orange') + plt.savefig(figure_path+'v3/'+metric+get_save_id(n_epochs, n_neurons, n_neurons2, n_neurons3,n_neurons4, n_batch, l_rate, d1, r1, r2) + +'.png') + plt.clf() + print('Done') + + + +def calculate_baselines(): + file_combinations = [(hour_timespan_str, with_threshold_str,'ALL32USERS1HR_WITHTHRESHOLD.xlsx'), + (min_timespan_str, with_threshold_str, 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx'), + (min_timespan_str, without_threshold_str, 'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx'), + (hour_timespan_str, without_threshold_str, 'ALLUSERS_32_1HR_WITHOUT_THRESHOLD.xlsx'), + ] + baseline_res = pd.DataFrame() + for timespan_id, threshold_id, filename in file_combinations: + file_path = os.path.join(dataset_path, filename) + df = load_dataset(file_path) + df = remove_covid_data(df) + + _, _, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=20) + te = reduce_columns(te, filename) + user_data_te = prepare_user_data(te) + for sequence_length in range(5,30, 5): + x, y = prepare_data_for_model(user_data=user_data_te, sequence_length=sequence_length) + + for strategy in ['most_frequent', 'stratified', 'uniform']: + cls = DummyClassifier(strategy=strategy) + cls.fit(x,y) + y_pred = cls.predict(x) + acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred) + baseline_res = pd.concat([baseline_res, + DataFrame({ 'strategy':[strategy], threshold_str:[threshold_id], + timespan_str:[timespan_id], sequence_length_str:[sequence_length], + accuracy_str:[acc],precision_str:[p],recall_str:[r], + f1_string:f1})], ignore_index=True) + baseline_res.to_json('baseline_results.json') + print('Done') + +def get_prepared_data_v3(filename, sample=100, print_unique=False): + df = pd.read_json(filename) + df = remove_covid_data(df) + + # remove users which are a complete subset of another user (but keep one) + users_to_remove = [] + for user_a in df[user_str].unique(): + for user_b in df[user_str].unique(): + if user_a != user_b: + data = pd.concat([df[df[user_str]==user_a], df[df[user_str]==user_b]]) + columns = data.columns.tolist() + columns.remove(user_str) + + no_dup = data.drop_duplicates(columns, keep=False) + if len(no_dup[no_dup[user_str]==user_a]) == 0: + if print_unique: + print(user_a, 'is subset of',user_b) + if user_b not in users_to_remove: + users_to_remove.append(user_a) + df = df[~df[user_str].isin(users_to_remove)] + + # bin steps per hour TODO: adjust for minutes + for hour in ['Hour_'+str(i) for i in range(24)]: + hour_data = df[hour] + # smaller 1000 - round to 10 + a = ((hour_data[hour_data<1000]/10).round()*10) + # between 1000 and 10000 - round to next 100 + b = ((hour_data[(hour_data>=1000)& (hour_data<10000)]/100).round()*100) + # higher or equal 10000 - one class + c = hour_data[hour_data > 10000] + c = pd.Series(data={ind:10000 for ind in c.index}, index=c.index) + new = pd.concat([a, b, c]).sort_index().astype(int) + df[hour] = new + + # remove users with too little data (optional) + #value_counts = df[user_str].value_counts() + #df = df[df[user_str].isin(value_counts[value_counts>200].index)] + min_datapoints = 500 # 500 leads to at least 75 datapoints in the valid set + users_to_remove = set() + cols = df.columns.tolist() + cols.remove(user_str) + reduced = df.drop_duplicates(subset=cols, keep=False) + for user_id in df[user_str].unique(): + subset = df[df[user_str] == user_id] + reduced_subset = reduced[reduced[user_str] == user_id] + if print_unique: + print(user_id, len(subset), len(reduced_subset)) + if len(reduced_subset) < min_datapoints: + users_to_remove.add(user_id) + if print_unique: + print('removing', user_id) + + df = df[~df[user_str].isin(users_to_remove)] + + tr, val, te = split_data_by_userdata_percentage(df, percentages=(70, 15, 15), sample=sample) + tr = reduce_columns_v3(tr) + val = reduce_columns_v3(val) + te = reduce_columns_v3(te) + + + + print('Train: Users', len(tr[user_str].unique()), 'mean num datapoins:', tr[user_str].value_counts().mean()) + print('Valid: Users', len(val[user_str].unique()), 'mean num datapoins:', val[user_str].value_counts().mean()) + print('Test: Users', len(te[user_str].unique()), 'mean num datapoins:', te[user_str].value_counts().mean()) + + tr, val, te = add_features(tr), add_features(val), add_features(te) + + scaler = MinMaxScaler() + scaler.fit(tr.drop(columns=[user_str])) + + return scale_dataset(scaler, tr), scale_dataset(scaler, val), scale_dataset(scaler, te) + + +def scale_dataset(scaler, df): + y = df[user_str] + x_scaled = scaler.transform(df.drop(columns=[user_str])) + x_scaled = pd.DataFrame(x_scaled) + x_scaled.columns = df.drop(columns=[user_str]).columns + + df_scaled = pd.concat([x_scaled, pd.DataFrame(y.reset_index()[user_str])], axis=1) +# df_scaled.columns = df.columns + return prepare_user_data(df_scaled) + + +def calculate_baselines_v3(): + file_combinations = [(hour_timespan_str, dataset_hrs_path), + # (min_timespan_str, dataset_min_path), # TODO: dataset bining not ready for minutes + ] + baseline_res = pd.DataFrame() + for timespan_id, filename in file_combinations: + _, _, te = get_prepared_data_v3(filename) + for sequence_length in range(1,30,5): + x, y = prepare_data_for_model(user_data=te, sequence_length=sequence_length) + + for strategy in ['most_frequent', 'stratified', 'uniform']: + cls = DummyClassifier(strategy=strategy) + cls.fit(x,y) + y_pred = cls.predict(x) + acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred) + baseline_res = pd.concat([baseline_res, + DataFrame({ 'strategy':[strategy], + timespan_str:[timespan_id], sequence_length_str:[sequence_length], + accuracy_str:[acc],precision_str:[p],recall_str:[r], + f1_string:f1})], ignore_index=True) + baseline_res.to_json('baseline_results_v3.json') + print('Done') + + +def hypertune_basic_algorithms(): + # TODO: hrs/min + # iterate over sequence lengths + sequence_length = 7 + + tr, val, te = get_prepared_data_v3(dataset_hrs_path) + + x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) + x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) + + random_state = 17 + results = pd.DataFrame() + for tag, clf, grid in [ + ('GradientBoosting', GradientBoostingClassifier(random_state=random_state), + {'loss': ['log_loss', 'exponential'], + 'learning_rate': [0.1, 0.5, 1.0,2.0, 5.0], + 'n_estimators': [10, 50, 100, 150, 200], + 'subsample': [0.1, 0.5, 1.0], + 'criterion': ['friedman_mse', 'squared_error'], + 'min_samples_split': [2, 10, 100], + 'min_samples_leaf': [1, 5, 10], + 'min_weight_fraction_leaf': [0.0, 0.1, 0.5], + 'max_depth': [None, 2, 10, 100], + 'min_impurity_decrease': [0.0, 0.1, 0.5], + 'max_features': ['sqrt', 'log2', None, 10, 20], + 'max_leaf_nodes': [None, 1, 5, 10], + }), + ('Bernoulli', BernoulliNB(), {'fit_prior': [True, False], + 'binarize': [0.0, 0.1, 0.25, 0.5, 0.75], + 'force_alpha': [True, False], + 'alpha':[0.0, 0.25, 0.5, 0.75, 1.0]}), + ('extra trees', ExtraTreesClassifier(random_state=random_state, n_jobs=1), + {'n_estimators': [10, 50, 100, 150, 200], + 'criterion': ['gini', 'entropy', 'log_loss'], + 'max_depth': [None, 2, 10, 100], + 'min_samples_split': [2, 10, 100], + 'min_samples_leaf': [1, 5, 10], + 'min_weight_fraction_leaf': [0.0, 0.1, 0.5], + 'max_features': ['sqrt', 'log2', None, 10, 20], + 'max_leaf_nodes': [None, 1, 5, 10], + 'min_impurity_decrease': [0.0, 0.1, 0.5], + 'bootstrap': [True, False], + 'class_weight': [None, 'balanced', 'balanced_subsample'], + 'max_samples': [None, 0.1, 0.2, 0.3]} + ), + ('random forest', RandomForestClassifier(random_state=random_state, n_jobs=1), + {'n_estimators':[10, 50, 100, 150, 200], + 'criterion':['gini', 'entropy', 'log_loss'], + 'max_depth':[None, 2, 10,100], + 'min_samples_split': [2,10,100], + 'min_samples_leaf':[1,5,10], + 'min_weight_fraction_leaf':[0.0,0.1, 0.5], + 'max_features':['sqrt', 'log2', None, 10, 20], + 'max_leaf_nodes':[None, 1, 5, 10], + 'min_impurity_decrease':[0.0, 0.1, 0.5], + 'bootstrap':[True, False], + 'class_weight':[None, 'balanced', 'balanced_subsample'], + 'max_samples':[None,0.1, 0.2, 0.3]}) + ]: + grid_search = GridSearchCV( + estimator=clf, param_grid=grid, scoring='f1_weighted', cv=5, n_jobs=1) + grid_search.fit(x_tr, y_tr) + + best_model = grid_search.best_estimator_ + y_pred = best_model.predict(x_val) + acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) + results = pd.concat([results, DataFrame({ 'params': str(grid_search.best_params_), + 'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) + results.to_json('basic_ht_results.json') + print('Done') + +def test_basic_algorithms(): + # TODO: hrs/min + # TODO: iterate over sequence lengths + sequence_length = 21 + + tr, val, te = get_prepared_data_v3(dataset_hrs_path) + + x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) + x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) + + random_state = 17 + results = pd.DataFrame() + for tag, clf in [ + # ('Label Propagation', LabelPropagation()), + # ('Label Spreading', LabelSpreading()), + # ('VBGMM', GaussianMixture(random_state=random_state)), + # ('linear discrimenant analysis', LinearDiscriminantAnalysis()), + # ('discriminent analysis', QuadraticDiscriminantAnalysis()), + # ('oneclassSVM', OneClassSVM()), + # ('mlp', MLPClassifier(random_state=random_state)), + # ('Perceptron', Perceptron(random_state=random_state)), + # ('SVC', SVC(random_state=random_state)), + #('logisticRegression', LogisticRegression(random_state=random_state)), + #('logisticRegressionCV', LogisticRegressionCV(random_state=random_state)), + #('multinomialNB', MultinomialNB()), + #('nearestCentroid', NearestCentroid()), + #('linearSVC', LinearSVC(random_state=random_state)), + #('ada boost', AdaBoostClassifier(random_state=random_state)), + #('GradientBoosting', GradientBoostingClassifier(random_state=random_state)), + #('Bernoulli', BernoulliNB()), + #('claibrated', CalibratedClassifierCV()), + #('naive Bayes', GaussianNB()), + #('sgd', SGDClassifier(random_state=random_state)), + #('ridgeCV', RidgeClassifierCV()), + # ('ridge', RidgeClassifier(random_state=random_state)), + # ('passiveAggressive', PassiveAggressiveClassifier(random_state=random_state)), + # ('knn', KNeighborsClassifier()), + # ('bagging', BaggingClassifier(random_state=random_state)), + # ('decision tree', DecisionTreeClassifier(random_state=random_state)), + # ('extra tree', ExtraTreeClassifier(random_state=random_state)), + # ('extra trees', ExtraTreesClassifier(random_state=random_state)), + ('random forest', RandomForestClassifier(random_state=random_state)) + ]: + clf.fit(x_tr, y_tr) + y_pred = clf.predict(x_val) + acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) + results = pd.concat([results, DataFrame({ 'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) + print('Done') + + +def add_features(df): + # indicator weekend + df['weekend'] = df[day_of_week_str + '_Saturday']+df[day_of_week_str + '_Sunday'] + # sum of steps per day + df['day_total'] = sum([df['Hour_'+str(i)] for i in range(23)]) + # sum of steps morning, afternoon, evening, night + df['morning_total'] = sum([df['Hour_' + str(i)] for i in range(6,13)]) + df['afternoon_total'] = sum([df['Hour_' + str(i)] for i in range(13,19)]) + df['evening_total'] = sum([df['Hour_' + str(i)] for i in range(19,23)]) + df['night_total'] = sum([df['Hour_' + str(i)] for i in [23,0,1,2,3,4,5]]) + return df + + +def feature_engineering(): + sequence_length = 1 + + tr, val, te = get_prepared_data_v3(dataset_hrs_path, print_unique=True) + + x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) + x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) + + random_state = 17 + clf=RandomForestClassifier(random_state=random_state) + clf.fit(x_tr, y_tr) + y_pred = clf.predict(x_val) + acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) + cf = confusion_matrix(y_pred=y_pred, y_true=y_val) + # TODO: welche funktionieren schlecht? warum? + # TODO: auf minutes umändern + + print('Done') + + +def test_sequence_length_on_approach(clf = RandomForestClassifier(random_state=17)): + tr, val, te = get_prepared_data_v3(dataset_hrs_path) + + results_train = pd.DataFrame() + results_valid = pd.DataFrame() + for sequence_length in range(1, 60, 5): + x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) + x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) + clf.fit(x_tr, y_tr) + acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=clf.predict(x_val)) + results_valid = pd.concat([results_valid, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) + acc, p, r, f1 = eval_metrics(y_true=y_tr, y_pred=clf.predict(x_tr)) + results_train = pd.concat([results_train, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) + + fig = plt.figure() + + for frame in [results_train, results_valid]: + plt.plot(frame[sequence_length_str], frame[f1_string]) + + plt.show() + print('') + +def manual_tuning_v4(model_type): + # TODO: hrs/min + tr, val, te = get_prepared_data_v3(dataset_hrs_path) + n_epochs= 20 + n_neurons = 1024 + results_train = pd.DataFrame() + results_valid = pd.DataFrame() + for sequence_length in range(1, 50, 5): + train_data = prepare_data_for_model(user_data=tr, sequence_length=sequence_length) + val_data = prepare_data_for_model(user_data=val, sequence_length=sequence_length) + + # fit and evaluate model + history_list = list() + repeats = 3 + # run diagnostic tests + for i in range(repeats): + history = train_one_model_v2(train_data, val_data, 1024, n_epochs, + n_neurons, sequence_length=sequence_length, + model_type=model_type) + history_list.append(history) + results = pd.concat([history.tail(1) for history in history_list]).mean() + results_train = pd.concat([results_train, + DataFrame({sequence_length_str:[sequence_length], + accuracy_str:[results['train_acc']], + precision_str:[results['train_p']], + recall_str:[results['train_r']], + f1_string:[results['train_f1']]})], ignore_index=True) + results_valid = pd.concat([results_valid, + DataFrame({sequence_length_str:[sequence_length], + accuracy_str:[results['test_acc']], + precision_str:[results['test_p']], + recall_str:[results['test_r']], + f1_string:[results['test_f1']]})], ignore_index=True) + + fig = plt.figure() + for frame in [results_train, results_valid]: + plt.plot(frame[sequence_length_str], frame[f1_string]) + + plt.show() + print('Done') + + +if __name__ == "__main__": + # Ordner erstellen, die benötigt werden + create_dir('results/') + create_dir(figure_path) + pd.options.mode.copy_on_write = True + + main_two_v1() + visualise_results_v1() + #test(model_type=model_type_gru) + # main_two_v2(model_type=model_type_gru) + #visualise_results_v2() + #manual_tuning(model_type=model_type_lstm) + #calculate_baselines() + + #### Ab hier aktuell (21.01.2026) + #calculate_baselines() +# manual_tuning_v3(model_type=model_type_lstm) + #test_basic_algorithms() + # test_basic_algorithm_on_sequence_lengths() + manual_tuning_v4(model_type=model_type_lstm) + #feature_engineering() + #hypertune_basic_algorithms() + print('Done') \ No newline at end of file diff --git a/non_jupyter_version.py b/old/non_jupyter_version.py similarity index 100% rename from non_jupyter_version.py rename to old/non_jupyter_version.py diff --git a/old/pipeline_old.py b/old/pipeline_old.py new file mode 100644 index 0000000..87e5408 --- /dev/null +++ b/old/pipeline_old.py @@ -0,0 +1,466 @@ +import random + +import keras_tuner +import numpy as np +import pandas as pd +import shutil + +from keras import Input +from keras.src.losses import SparseCategoricalCrossentropy +from keras.src.metrics import F1Score, Precision, Recall, Accuracy, SparseCategoricalAccuracy +from pandas import ExcelWriter, DataFrame +from tensorflow.keras.models import Sequential +from tensorflow.keras.layers import LSTM, Dense, Dropout, Bidirectional,GRU +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping +from keras_tuner import RandomSearch +from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix + +epochs = 5#50 +model_type_gru = 'GRU' +model_type_lstm = 'LSTM' +model_type_bilstm = 'BiLSTM' + + +# === Data functions === +def load_dataset(file_path): + return pd.read_excel(file_path) + +def filter_data(df, scenario, ALLUSERS32_15MIN_WITHOUTREHOLD): + filtered = pd.DataFrame() + for year, months in scenario: + filtered = pd.concat([filtered, df[(df['Year'] == year) & (df['Month'].isin(months))]]) + + if ALLUSERS32_15MIN_WITHOUTREHOLD: + return filtered.drop(columns=['Month', 'Year', 'date', 'DayOfWeek']) + else: + return filtered.drop(columns=['Month', 'Year', 'date']) + +def filter_test_data(df, scenario): + data_parts = [] + for year, months in scenario: + part = df[(df['Year'] == year) & (df['Month'].isin(months))] + data_parts.append(part) + return pd.concat(data_parts, ignore_index=True) + +def prepare_user_data(df): + #df_sorted = df.sort_values(by='user').reset_index(drop=True) + users = df['user'].unique() + return {user: df[df['user'] == user] for user in users} + +def make_sequences(data, sequence_length): + x, y = [], [] + features = data.drop('user', axis=1).values + labels = data['user'].values + for i in range(len(features) - sequence_length+1): # with overlap on days +# for i in range(0, len(features) - sequence_length + 1, sequence_length): # without overlap on days + x.append(features[i:i + sequence_length]) + y.append(labels[i + sequence_length-1]) + return x, y + +def prepare_data_for_basic_algorithm(user_data, sequence_length): + combined = pd.DataFrame() + for user, data in user_data.items(): + x_new, y_new = make_sequences(data, sequence_length) + if len(x_new)>0: + var = [[pd.DataFrame(a[s]) for s in range(sequence_length)] for a in x_new] + df_var = pd.concat([pd.concat(seq_list).T for seq_list in var]) + df_var['user'] = user + combined = pd.concat([combined, df_var], ignore_index=True) + return combined.drop(columns=['user']), combined['user'] + +def prepare_data_for_model(user_data, sequence_length, print_counts=False): + x, y = [], [] + combined = pd.DataFrame() + for user, data in user_data.items(): + x_new, y_new = make_sequences(data, sequence_length) + x = x + x_new + y = y + y_new + if print_counts and len(x_new)>0: + var = [[pd.DataFrame(a[s])for s in range(sequence_length)] for a in x_new ] + df_var = pd.concat([pd.concat(seq_list).T for seq_list in var]) + df_var['user'] = user + combined = pd.concat([combined, df_var], ignore_index=True) + if print_counts: + combined_ohne = combined.drop('user', axis=1) + print('Alle', len(combined)) + print('Unique mit user', len(combined.drop_duplicates())) + print('Unique ohne user', len(combined_ohne.drop_duplicates())) + print('Unique') + print(combined.drop_duplicates()['user'].value_counts()) + print('Alle') + print(combined['user'].value_counts()) + random.Random(17).shuffle(x) + random.Random(17).shuffle(y) + x = np.array(x) + y = np.array(y) + return x,y + +# === Training & Validation === +def train_models(user_data, user_data_val, sequence_lengths, tuner_dir="./working/tuner", model_type=model_type_lstm): + best_models = {} + early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True) + lr_scheduler = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, verbose=1) + users = list(user_data.keys()) + + shutil.rmtree(tuner_dir, ignore_errors=True) + + for sequence_length in sequence_lengths: + print(f"\n=== Training for Sequence Length: {sequence_length} ===") + X, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length) + X_val, y_val = prepare_data_for_model(user_data=user_data_val, sequence_length=sequence_length) + + if X.shape[0] == 0 or X_val.shape[0] == 0: + print(f"⚠️ Skipped sequence length {sequence_length} due to insufficient data.") + continue + + n_features = X.shape[2] + + def build_model(hp): + model = Sequential() + if model_type==model_type_bilstm: + model.add(Bidirectional(LSTM(units=hp.Int('units', 32, 256, step=2), + input_shape=(sequence_length, n_features)))) + if model_type==model_type_lstm: + model.add(LSTM(units=hp.Int('units', 32, 256, step=2), + input_shape=(sequence_length, n_features))) + if model_type==model_type_gru: + model.add(GRU(units=hp.Int('units', 32, 256, step=2), + input_shape=(sequence_length, n_features))) + model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.5, step=0.1))) + model.add(Dense(len(users), activation='softmax')) + model.compile( + optimizer=Adam(learning_rate=hp.Choice('learning_rate', [1e-2, 1e-3, 1e-4])), + loss='sparse_categorical_crossentropy', + metrics=['accuracy'] + ) + return model + + tuner = RandomSearch( + build_model, + objective='val_loss', + max_trials=30, + executions_per_trial=2, + directory=tuner_dir, + project_name=f'lstm_seq_{sequence_length}' + ) + + tuner.search(X, y, epochs=epochs, validation_data=(X_val, y_val), + callbacks=[early_stopping, lr_scheduler], verbose=0) + + best_hps = tuner.get_best_hyperparameters(1)[0] + best_model = tuner.hypermodel.build(best_hps) + best_model.fit(X, y, epochs=epochs, validation_data=(X_val, y_val), + callbacks=[early_stopping, lr_scheduler], verbose=0) + + best_models[sequence_length] = { + 'model': best_model, + 'best_hyperparameters': { + 'units': best_hps.get('units'), + 'dropout_rate': best_hps.get('dropout_rate'), + 'learning_rate': best_hps.get('learning_rate') + } + } + + return best_models + +# === Training & Validation === +def train_models_v2(user_data, user_data_val, sequence_length, model_type): + tuner_dir = "./working/tuner/"+model_type + #val_metric = 'val_f1' + val_metric = 'val_precision' + + early_stopping = EarlyStopping(monitor=val_metric, patience=3, restore_best_weights=True) + lr_scheduler = ReduceLROnPlateau(monitor=val_metric, factor=0.5, patience=2) + + shutil.rmtree(tuner_dir, ignore_errors=True) + + x, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length) + x_val, y_val = prepare_data_for_model(user_data=user_data_val, sequence_length=sequence_length) + + n_features = x.shape[2] + users = list(user_data.keys()) + + #y_val = np.array(y_val).reshape(-1, 1) + #y = np.array(y).reshape(-1, 1) + + def build_model(hp): + units_hp = hp.Int('units', 2, 8, step=2, sampling="log") +# units_hp = hp.Int('units', 2, 256, step=2, sampling="log") + + model = Sequential() + model.add(Input((sequence_length, n_features))) + if model_type==model_type_bilstm: + model.add(Bidirectional(LSTM(units=units_hp))) + if model_type==model_type_lstm: + model.add(LSTM(units=units_hp)) + if model_type==model_type_gru: + model.add(GRU(units=units_hp)) + model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.2, step=0.1))) + model.add(Dense(len(users), activation='softmax')) + model.compile( + optimizer=Adam(learning_rate=hp.Choice('learning_rate', [1e-5])), + loss='sparse_categorical_crossentropy', + metrics=[#F1Score(name='f1', average='weighted'), + Precision(), #Recall(), Accuracy() + ] + ) + return model + + tuner = RandomSearch( + build_model, + objective=keras_tuner.Objective(val_metric, direction="max"), + max_trials=120, + directory=tuner_dir, + ) + + tuner.search(x, y, epochs=epochs, validation_data=(x_val, y_val), + callbacks=[early_stopping, lr_scheduler]) + return tuner.get_best_models(num_models=1)[0] + + +def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons,n_neurons2,n_neurons3,n_neurons4, l_rate, d1, r1, reg1, r2, reg2, sequence_length, model_type): + x, y = prepare_data_for_model(user_data=train_data, sequence_length=sequence_length) + n_features = x.shape[2] + users = list(train_data.keys()) + + # prepare model + def build_model(): + model = Sequential() + model.add(Input(shape=(sequence_length, n_features), batch_size=n_batch)) + if model_type == model_type_bilstm: + model.add(Bidirectional(LSTM(n_neurons))) + if model_type == model_type_lstm: +# model.add(LSTM(n_neurons, kernel_regularizer=reg1, return_sequences=True)) + model.add(LSTM(n_neurons)) + # model.add(LSTM(n_neurons2)) + if model_type == model_type_gru: + model.add(GRU(n_neurons)) + #model.add(Dense(n_neurons, activation='relu')) + #model.add(Dropout(d1)) + model.add(Dense(len(users), activation='softmax')) + model.compile( + optimizer=Adam(learning_rate=l_rate), + loss=SparseCategoricalCrossentropy(), + metrics=[SparseCategoricalAccuracy()], + ) + return model + + model = build_model() + + # fit model + train_acc, test_acc, train_p, test_p, train_r, test_r, train_f1, test_f1 = list(), list(),list(), list(),list(), list(),list(), list() + for i in range(n_epochs): + model.fit(x, y, batch_size=n_batch, epochs=1, verbose=0, shuffle=False) + # evaluate model on train data + acc, p, r, f1 = evaluate(model, train_data, sequence_length, n_batch) + train_acc.append(acc) + train_p.append(p) + train_r.append(r) + train_f1.append(f1) + # evaluate model on test data + savename = 'cf_matrix_'+get_save_id(n_epochs, n_neurons, n_neurons2,n_neurons3, n_neurons4, n_batch, l_rate,d1,r1, r2)+'.json' + acc, p, r, f1 = evaluate(model, val_data, sequence_length, n_batch, save_name=savename) + test_acc.append(acc) + test_p.append(p) + test_r.append(r) + test_f1.append(f1) + + history = DataFrame() + history['train_acc'], history['test_acc'] = train_acc, test_acc + history['train_p'], history['test_p'] = train_p, test_p + history['train_r'], history['test_r'] = train_r, test_r + history['train_f1'], history['test_f1'] = train_f1, test_f1 + return history + + +def train_one_model_v2(train_data, val_data, n_batch, n_epochs, n_neurons, sequence_length, model_type): + x, y = train_data + x_v, y_v = val_data + users = list(set(y)) + + # renumber users + user_map = {users[i]:i for i in range(len(users))} + y = np.array([user_map[x] for x in y]) + y_v = np.array([user_map[x] for x in y_v]) + n_features = x.shape[2] + user_num = len(users) + + # prepare model + def build_model(): + model = Sequential() + model.add(Input(shape=(sequence_length, n_features), batch_size=n_batch)) + if model_type == model_type_bilstm: + model.add(Bidirectional(LSTM(n_neurons))) + if model_type == model_type_lstm: + model.add(LSTM(n_neurons)) + if model_type == model_type_gru: + model.add(GRU(n_neurons)) + #model.add(Dense(n_neurons, activation='relu')) + #model.add(Dropout(d1)) + model.add(Dense(user_num, activation='softmax')) + model.compile( + optimizer=Adam(), + loss=SparseCategoricalCrossentropy(), + metrics=[SparseCategoricalAccuracy()], + ) + return model + + model = build_model() + + # fit model + train_acc, test_acc, train_p, test_p, train_r, test_r, train_f1, test_f1 = list(), list(),list(), list(),list(), list(),list(), list() + for i in range(n_epochs): + model.fit(x, y, batch_size=n_batch, epochs=1, verbose=0, shuffle=False) + # evaluate model on train data + acc, p, r, f1 = evaluate_v2(model, (x,y), sequence_length, n_batch) + train_acc.append(acc) + train_p.append(p) + train_r.append(r) + train_f1.append(f1) + # evaluate model on test data + savename = 'cf_matrix_'+get_save_id(n_epochs, n_neurons, '', '', '', n_batch, '','','','')+'.json' + acc, p, r, f1 = evaluate_v2(model, (x_v, y_v), sequence_length, n_batch, save_name=savename) + test_acc.append(acc) + test_p.append(p) + test_r.append(r) + test_f1.append(f1) + + history = DataFrame() + history['train_acc'], history['test_acc'] = train_acc, test_acc + history['train_p'], history['test_p'] = train_p, test_p + history['train_r'], history['test_r'] = train_r, test_r + history['train_f1'], history['test_f1'] = train_f1, test_f1 + return history + +def get_save_id(n_epochs, n_neurons, n_neurons2,n_neurons3,n_neurons4, n_batch, l_rate, d1,r1, r2): + return '_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+ str(n_batch) +#'x'+str(n_neurons3)+'x'+str(n_neurons4) + #+'_l'+str(l_rate)+'_r'+str(r1)+'xx'+str(r2) + + +def evaluate(model, df, sequence_length, batch_size, save_name=None): + x, y = prepare_data_for_model(user_data=df, sequence_length=sequence_length) + x = np.array(x) + y_true = np.array(y) + + y_pred = model.predict(x, verbose=0, batch_size=batch_size) + y_pred_classes = np.argmax(y_pred, axis=1) + cf_matrix = pd.DataFrame(confusion_matrix(y_true, y_pred_classes)) + if save_name is not None: + cf_matrix.to_json('results/'+save_name) + true_counts = pd.DataFrame(y).value_counts() + print('Top true occurrences', true_counts[:6]) + predicted_counts = pd.DataFrame(y_pred_classes).value_counts() + print('Top predicted occurrences', predicted_counts[:6]) + + return eval_metrics(y_true=y_true, y_pred=y_pred_classes) + +def evaluate_v2(model, data, sequence_length, batch_size, save_name=None): + x, y_true = data + y_pred = model.predict(x, verbose=0, batch_size=batch_size) + y_pred_classes = np.argmax(y_pred, axis=1) + cf_matrix = pd.DataFrame(confusion_matrix(y_true, y_pred_classes)) + if save_name is not None: + cf_matrix.to_json('results/'+save_name) + true_counts = pd.DataFrame(y_true).value_counts() + print('Top true occurrences', true_counts[:6]) + predicted_counts = pd.DataFrame(y_pred_classes).value_counts() + print('Top predicted occurrences', predicted_counts[:6]) + + return eval_metrics(y_true=y_true, y_pred=y_pred_classes) + + +def eval_metrics(y_true, y_pred): + f1 = f1_score(y_true=y_true, y_pred=y_pred, average='weighted') + p = precision_score(y_true=y_true, y_pred=y_pred, average='weighted') + r = recall_score(y_true=y_true, y_pred=y_pred, average='weighted') + acc = accuracy_score(y_true=y_true, y_pred=y_pred) + return acc, p, r, f1 + +# === Evaluation === +def evaluate_models(best_models, df_test, sequence_lengths, output_excel_path, ALLUSERS32_15MIN_WITHOUTTHREHOLD): + print("\n🧪 Evaluating on Test Data...") + with ExcelWriter(output_excel_path) as writer: + for sequence_length in sequence_lengths: + if sequence_length not in best_models: + continue + evaluate_model_on_test_data(best_models[sequence_length]['model'], df_test.copy(), + sequence_length, writer, ALLUSERS32_15MIN_WITHOUTTHREHOLD) + +def evaluate_model_on_test_data(model, test_df, sequence_length, excel_writer, ALLUSERS32_15MIN_WITHOUTTHREHOLD): + if(ALLUSERS32_15MIN_WITHOUTTHREHOLD): + test_df = test_df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek']) + else: + test_df = test_df.drop(columns=['Month', 'Year', 'date']) + + test_df = test_df.sort_values(by='user').reset_index(drop=True) + + users = test_df['user'].unique() + results = [] + accuracy_above_50 = 0 + + for user in users: + user_df = test_df[test_df['user'] == user] + X, y_true = [], [] + user_features = user_df.drop(columns=['user']).values + user_labels = user_df['user'].values + + if len(user_df) <= sequence_length: + print(f"Skipping User {user} (not enough data for sequence length {sequence_length})") + continue + + for i in range(len(user_df) - sequence_length): + seq_x = user_features[i:i + sequence_length] + seq_y = user_labels[i + sequence_length] + X.append(seq_x) + y_true.append(seq_y) + + X = np.array(X) + y_true = np.array(y_true) + + if len(X) == 0: + continue + + y_pred = model.predict(X, verbose=0) + y_pred_classes = np.argmax(y_pred, axis=1) + + # counts which class was predicted how often + unique_pred, counts_pred = np.unique(y_pred_classes, return_counts=True) + label_counts_pred = dict(zip(unique_pred, counts_pred)) + + # counts which class should have been predicted how often (only one class for the user) + unique_true, counts_true = np.unique(y_true, return_counts=True) + label_counts_true = dict(zip(unique_true, counts_true)) + + # the fraction of correctly classified samples + acc = accuracy_score(y_true, y_pred_classes) + if acc > 0.5: + accuracy_above_50 += 1 + + results.append({ + 'User': user, + 'Accuracy (%)': acc * 100, + 'Predicted Class Distribution': str(label_counts_pred), + 'Actual Class Distribution': str(label_counts_true) + }) + + print(f"\n=== User {user} ===") + print(f"✅ Accuracy: {acc * 100:.2f}%") + print("📊 Predicted Class Distribution:", label_counts_pred) + print("📌 Actual Class Distribution: ", label_counts_true) + + final_accuracy_percent = (accuracy_above_50 / 32) * 100 + print(f"\n🟩 Final Evaluation Summary for Sequence Length {sequence_length}:") + print(f"Users with >50% Accuracy: {accuracy_above_50} / 32") + print(f"✅ Final Success Rate: {final_accuracy_percent:.2f}%") + + results.append({ + 'User': 'TOTAL', + 'Accuracy (%)': '', + 'Predicted Class Distribution': f'Users >50% Acc: {accuracy_above_50}/32', + 'Actual Class Distribution': f'Success Rate: {final_accuracy_percent:.2f}%' + }) + + df_results = pd.DataFrame(results) + df_results.to_excel(excel_writer, sheet_name=f"SeqLen_{sequence_length}", index=False) diff --git a/requirements.txt b/old/requirements.txt similarity index 91% rename from requirements.txt rename to old/requirements.txt index f0a4af0..173213c 100644 --- a/requirements.txt +++ b/old/requirements.txt @@ -37,8 +37,7 @@ scipy==1.16.0 six==1.17.0 tensorboard==2.19.0 tensorboard-data-server==0.7.2 -tensorflow==2.19.0 -tensorflow-io-gcs-filesystem==0.31.0 +tensorflow==2.20.0 termcolor==3.1.0 threadpoolctl==3.6.0 typing_extensions==4.14.1 diff --git a/pipeline.py b/pipeline.py index 728020d..5163e81 100644 --- a/pipeline.py +++ b/pipeline.py @@ -1,19 +1,15 @@ import random -import keras_tuner import numpy as np import pandas as pd -import shutil from keras import Input from keras.src.losses import SparseCategoricalCrossentropy -from keras.src.metrics import F1Score, Precision, Recall, Accuracy, SparseCategoricalAccuracy -from pandas import ExcelWriter, DataFrame +from keras.src.metrics import SparseCategoricalAccuracy +from pandas import DataFrame from tensorflow.keras.models import Sequential -from tensorflow.keras.layers import LSTM, Dense, Dropout, Bidirectional,GRU +from tensorflow.keras.layers import LSTM, Dense, Bidirectional,GRU from tensorflow.keras.optimizers import Adam -from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping -from keras_tuner import RandomSearch from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix epochs = 5#50 @@ -21,61 +17,49 @@ model_type_gru = 'GRU' model_type_lstm = 'LSTM' model_type_bilstm = 'BiLSTM' - -# === Display functions === -def display_warning_about_2020_data(): - print("\n⚠️ Warning: 2020 data after February is excluded due to COVID-19.") - print("✅ Only Jan and Feb 2020 are used for testing. Do not use them in training/validation.") - -def display_warnings_for_scenarios(scenario_type, predefined_training_scenarios, predefined_validation_scenarios): - if scenario_type == "training": - print("\n⚠️ Predefined Training Scenarios (for reference only):") - for name, scenario in predefined_training_scenarios.items(): - parts = [f"{year}-{months}" for year, months in scenario['years_months']] - print(f" {name}: {', '.join(parts)}") - elif scenario_type == "validation": - print("\n⚠️ Predefined Validation Scenario:") - for name, scenario in predefined_validation_scenarios.items(): - parts = [f"{year}-{months}" for year, months in scenario['years_months']] - print(f" {name}: {', '.join(parts)}") - -# === Data functions === -def load_dataset(file_path): - return pd.read_excel(file_path) - -def filter_data(df, scenario, ALLUSERS32_15MIN_WITHOUTREHOLD): - filtered = pd.DataFrame() - for year, months in scenario: - filtered = pd.concat([filtered, df[(df['Year'] == year) & (df['Month'].isin(months))]]) - - if ALLUSERS32_15MIN_WITHOUTREHOLD: - return filtered.drop(columns=['Month', 'Year', 'date', 'DayOfWeek']) - else: - return filtered.drop(columns=['Month', 'Year', 'date']) - -def filter_test_data(df, scenario): - data_parts = [] - for year, months in scenario: - part = df[(df['Year'] == year) & (df['Month'].isin(months))] - data_parts.append(part) - return pd.concat(data_parts, ignore_index=True) - -def prepare_user_data(df): - #df_sorted = df.sort_values(by='user').reset_index(drop=True) - users = df['user'].unique() - return {user: df[df['user'] == user] for user in users} - def make_sequences(data, sequence_length): + """ + Converts the data into sequences of the given length + + :param data: Original data + :param sequence_length: length of intended sequences + :return: x,y for the sequences + """ x, y = [], [] features = data.drop('user', axis=1).values labels = data['user'].values -# for i in range(len(features) - sequence_length+1): # with overlap on days - for i in range(0, len(features) - sequence_length + 1, sequence_length): # without overlap on days + for i in range(len(features) - sequence_length+1): # with overlap on days +# for i in range(0, len(features) - sequence_length + 1, sequence_length): # without overlap on days x.append(features[i:i + sequence_length]) y.append(labels[i + sequence_length-1]) return x, y -def prepare_data_for_model(user_data, sequence_length, print_counts=False): +def prepare_data_for_basic_algorithm(user_data, sequence_length): + """ + Converts the data into a format the sklearn algorithms can work with. Does not change the data, only the structure + :param user_data: the dict of dataframe with the data + :param sequence_length: intended sequence length + :return: the formatted data + """ + combined = pd.DataFrame() + for user, data in user_data.items(): + x_new, y_new = make_sequences(data, sequence_length) + if len(x_new)>0: + var = [[pd.DataFrame(a[s]) for s in range(sequence_length)] for a in x_new] + df_var = pd.concat([pd.concat(seq_list).T for seq_list in var]) + df_var['user'] = user + combined = pd.concat([combined, df_var], ignore_index=True) + return combined.drop(columns=['user']), combined['user'] + +def prepare_data_for_neural_model(user_data, sequence_length, print_counts=False): + """ + Converts the data into a format the neural model can work with. Does not change the data, only the structure + + :param print_counts: Whether to print some additional debug data + :param user_data: the dict of dataframe with the data + :param sequence_length: intended sequence length + :return: the formatted data + """ x, y = [], [] combined = pd.DataFrame() for user, data in user_data.items(): @@ -102,133 +86,17 @@ def prepare_data_for_model(user_data, sequence_length, print_counts=False): y = np.array(y) return x,y -# === Training & Validation === -def train_models(user_data, user_data_val, sequence_lengths, tuner_dir="./working/tuner", model_type=model_type_lstm): - best_models = {} - early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True) - lr_scheduler = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, verbose=1) - users = list(user_data.keys()) - - shutil.rmtree(tuner_dir, ignore_errors=True) - - for sequence_length in sequence_lengths: - print(f"\n=== Training for Sequence Length: {sequence_length} ===") - X, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length) - X_val, y_val = prepare_data_for_model(user_data=user_data_val, sequence_length=sequence_length) - - if X.shape[0] == 0 or X_val.shape[0] == 0: - print(f"⚠️ Skipped sequence length {sequence_length} due to insufficient data.") - continue - - n_features = X.shape[2] - - def build_model(hp): - model = Sequential() - if model_type==model_type_bilstm: - model.add(Bidirectional(LSTM(units=hp.Int('units', 32, 256, step=2), - input_shape=(sequence_length, n_features)))) - if model_type==model_type_lstm: - model.add(LSTM(units=hp.Int('units', 32, 256, step=2), - input_shape=(sequence_length, n_features))) - if model_type==model_type_gru: - model.add(GRU(units=hp.Int('units', 32, 256, step=2), - input_shape=(sequence_length, n_features))) - model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.5, step=0.1))) - model.add(Dense(len(users), activation='softmax')) - model.compile( - optimizer=Adam(learning_rate=hp.Choice('learning_rate', [1e-2, 1e-3, 1e-4])), - loss='sparse_categorical_crossentropy', - metrics=['accuracy'] - ) - return model - - tuner = RandomSearch( - build_model, - objective='val_loss', - max_trials=30, - executions_per_trial=2, - directory=tuner_dir, - project_name=f'lstm_seq_{sequence_length}' - ) - - tuner.search(X, y, epochs=epochs, validation_data=(X_val, y_val), - callbacks=[early_stopping, lr_scheduler], verbose=0) - - best_hps = tuner.get_best_hyperparameters(1)[0] - best_model = tuner.hypermodel.build(best_hps) - best_model.fit(X, y, epochs=epochs, validation_data=(X_val, y_val), - callbacks=[early_stopping, lr_scheduler], verbose=0) - - best_models[sequence_length] = { - 'model': best_model, - 'best_hyperparameters': { - 'units': best_hps.get('units'), - 'dropout_rate': best_hps.get('dropout_rate'), - 'learning_rate': best_hps.get('learning_rate') - } - } - - return best_models - -# === Training & Validation === -def train_models_v2(user_data, user_data_val, sequence_length, model_type): - tuner_dir = "./working/tuner/"+model_type - #val_metric = 'val_f1' - val_metric = 'val_precision' - - early_stopping = EarlyStopping(monitor=val_metric, patience=3, restore_best_weights=True) - lr_scheduler = ReduceLROnPlateau(monitor=val_metric, factor=0.5, patience=2) - - shutil.rmtree(tuner_dir, ignore_errors=True) - - x, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length) - x_val, y_val = prepare_data_for_model(user_data=user_data_val, sequence_length=sequence_length) +def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, sequence_length, model_type): + x, y = train_data + x_v, y_v = val_data + users = list(set(y)) + # renumber users + user_map = {users[i]:i for i in range(len(users))} + y = np.array([user_map[x] for x in y]) + y_v = np.array([user_map[x] for x in y_v]) n_features = x.shape[2] - users = list(user_data.keys()) - - #y_val = np.array(y_val).reshape(-1, 1) - #y = np.array(y).reshape(-1, 1) - - def build_model(hp): - units_hp = hp.Int('units', 2, 8, step=2, sampling="log") -# units_hp = hp.Int('units', 2, 256, step=2, sampling="log") - - model = Sequential() - model.add(Input((sequence_length, n_features))) - if model_type==model_type_bilstm: - model.add(Bidirectional(LSTM(units=units_hp))) - if model_type==model_type_lstm: - model.add(LSTM(units=units_hp)) - if model_type==model_type_gru: - model.add(GRU(units=units_hp)) - model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.2, step=0.1))) - model.add(Dense(len(users), activation='softmax')) - model.compile( - optimizer=Adam(learning_rate=hp.Choice('learning_rate', [1e-5])), - loss='sparse_categorical_crossentropy', - metrics=[#F1Score(name='f1', average='weighted'), - Precision(), #Recall(), Accuracy() - ] - ) - return model - - tuner = RandomSearch( - build_model, - objective=keras_tuner.Objective(val_metric, direction="max"), - max_trials=120, - directory=tuner_dir, - ) - - tuner.search(x, y, epochs=epochs, validation_data=(x_val, y_val), - callbacks=[early_stopping, lr_scheduler]) - return tuner.get_best_models(num_models=1)[0] - - -def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons,n_neurons2,n_neurons3,n_neurons4, l_rate, d1, r1, reg1, r2, reg2, sequence_length, model_type): - x, y = prepare_data_for_model(user_data=train_data, sequence_length=sequence_length) - n_features = x.shape[2] - users = list(train_data.keys()) + user_num = len(users) # prepare model def build_model(): @@ -237,16 +105,12 @@ def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons,n_neurons if model_type == model_type_bilstm: model.add(Bidirectional(LSTM(n_neurons))) if model_type == model_type_lstm: -# model.add(LSTM(n_neurons, kernel_regularizer=reg1, return_sequences=True)) model.add(LSTM(n_neurons)) - # model.add(LSTM(n_neurons2)) if model_type == model_type_gru: model.add(GRU(n_neurons)) - #model.add(Dense(n_neurons, activation='relu')) - #model.add(Dropout(d1)) - model.add(Dense(len(users), activation='softmax')) + model.add(Dense(user_num, activation='softmax')) model.compile( - optimizer=Adam(learning_rate=l_rate), + optimizer=Adam(), loss=SparseCategoricalCrossentropy(), metrics=[SparseCategoricalAccuracy()], ) @@ -259,14 +123,14 @@ def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons,n_neurons for i in range(n_epochs): model.fit(x, y, batch_size=n_batch, epochs=1, verbose=0, shuffle=False) # evaluate model on train data - acc, p, r, f1 = evaluate(model, train_data, sequence_length, n_batch) + acc, p, r, f1 = evaluate(model, (x, y), sequence_length, n_batch) train_acc.append(acc) train_p.append(p) train_r.append(r) train_f1.append(f1) # evaluate model on test data - savename = 'cf_matrix_'+get_save_id(n_epochs, n_neurons, n_neurons2,n_neurons3, n_neurons4, n_batch, l_rate,d1,r1, r2)+'.json' - acc, p, r, f1 = evaluate(model, val_data, sequence_length, n_batch, save_name=savename) + savename = 'cf_matrix_'+get_save_id(n_epochs, n_neurons, n_batch)+'.json' + acc, p, r, f1 = evaluate(model, (x_v, y_v), n_batch, save_name=savename) test_acc.append(acc) test_p.append(p) test_r.append(r) @@ -279,24 +143,26 @@ def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons,n_neurons history['train_f1'], history['test_f1'] = train_f1, test_f1 return history - -def get_save_id(n_epochs, n_neurons, n_neurons2,n_neurons3,n_neurons4, n_batch, l_rate, d1,r1, r2): +def get_save_id(n_epochs, n_neurons, n_batch): return '_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+ str(n_batch) -#'x'+str(n_neurons3)+'x'+str(n_neurons4) - #+'_l'+str(l_rate)+'_r'+str(r1)+'xx'+str(r2) - - -def evaluate(model, df, sequence_length, batch_size, save_name=None): - x, y = prepare_data_for_model(user_data=df, sequence_length=sequence_length) - x = np.array(x) - y_true = np.array(y) +def evaluate(model, data, batch_size, save_name=None): + """ + GIven a model, the data is used for prediction and then evaluated. + + :param model: Model to use with a .predict() call + :param data: x, y_true of the data, already prepared for the model + :param batch_size: batch size for prediction + :param save_name: if provided, results will be saved to a json file of that name + :return: the evaluation results + """ + x, y_true = data y_pred = model.predict(x, verbose=0, batch_size=batch_size) y_pred_classes = np.argmax(y_pred, axis=1) cf_matrix = pd.DataFrame(confusion_matrix(y_true, y_pred_classes)) if save_name is not None: cf_matrix.to_json('results/'+save_name) - true_counts = pd.DataFrame(y).value_counts() + true_counts = pd.DataFrame(y_true).value_counts() print('Top true occurrences', true_counts[:6]) predicted_counts = pd.DataFrame(y_pred_classes).value_counts() print('Top predicted occurrences', predicted_counts[:6]) @@ -304,97 +170,15 @@ def evaluate(model, df, sequence_length, batch_size, save_name=None): return eval_metrics(y_true=y_true, y_pred=y_pred_classes) - def eval_metrics(y_true, y_pred): + """ + Calculate the evaluation metrics + :param y_true: + :param y_pred: + :return: acc, p, r, f1 + """ f1 = f1_score(y_true=y_true, y_pred=y_pred, average='weighted') p = precision_score(y_true=y_true, y_pred=y_pred, average='weighted') r = recall_score(y_true=y_true, y_pred=y_pred, average='weighted') acc = accuracy_score(y_true=y_true, y_pred=y_pred) return acc, p, r, f1 - -# === Evaluation === -def evaluate_models(best_models, df_test, sequence_lengths, output_excel_path, ALLUSERS32_15MIN_WITHOUTTHREHOLD): - print("\n🧪 Evaluating on Test Data...") - with ExcelWriter(output_excel_path) as writer: - for sequence_length in sequence_lengths: - if sequence_length not in best_models: - continue - evaluate_model_on_test_data(best_models[sequence_length]['model'], df_test.copy(), - sequence_length, writer, ALLUSERS32_15MIN_WITHOUTTHREHOLD) - -def evaluate_model_on_test_data(model, test_df, sequence_length, excel_writer, ALLUSERS32_15MIN_WITHOUTTHREHOLD): - if(ALLUSERS32_15MIN_WITHOUTTHREHOLD): - test_df = test_df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek']) - else: - test_df = test_df.drop(columns=['Month', 'Year', 'date']) - - test_df = test_df.sort_values(by='user').reset_index(drop=True) - - users = test_df['user'].unique() - results = [] - accuracy_above_50 = 0 - - for user in users: - user_df = test_df[test_df['user'] == user] - X, y_true = [], [] - user_features = user_df.drop(columns=['user']).values - user_labels = user_df['user'].values - - if len(user_df) <= sequence_length: - print(f"Skipping User {user} (not enough data for sequence length {sequence_length})") - continue - - for i in range(len(user_df) - sequence_length): - seq_x = user_features[i:i + sequence_length] - seq_y = user_labels[i + sequence_length] - X.append(seq_x) - y_true.append(seq_y) - - X = np.array(X) - y_true = np.array(y_true) - - if len(X) == 0: - continue - - y_pred = model.predict(X, verbose=0) - y_pred_classes = np.argmax(y_pred, axis=1) - - # counts which class was predicted how often - unique_pred, counts_pred = np.unique(y_pred_classes, return_counts=True) - label_counts_pred = dict(zip(unique_pred, counts_pred)) - - # counts which class should have been predicted how often (only one class for the user) - unique_true, counts_true = np.unique(y_true, return_counts=True) - label_counts_true = dict(zip(unique_true, counts_true)) - - # the fraction of correctly classified samples - acc = accuracy_score(y_true, y_pred_classes) - if acc > 0.5: - accuracy_above_50 += 1 - - results.append({ - 'User': user, - 'Accuracy (%)': acc * 100, - 'Predicted Class Distribution': str(label_counts_pred), - 'Actual Class Distribution': str(label_counts_true) - }) - - print(f"\n=== User {user} ===") - print(f"✅ Accuracy: {acc * 100:.2f}%") - print("📊 Predicted Class Distribution:", label_counts_pred) - print("📌 Actual Class Distribution: ", label_counts_true) - - final_accuracy_percent = (accuracy_above_50 / 32) * 100 - print(f"\n🟩 Final Evaluation Summary for Sequence Length {sequence_length}:") - print(f"Users with >50% Accuracy: {accuracy_above_50} / 32") - print(f"✅ Final Success Rate: {final_accuracy_percent:.2f}%") - - results.append({ - 'User': 'TOTAL', - 'Accuracy (%)': '', - 'Predicted Class Distribution': f'Users >50% Acc: {accuracy_above_50}/32', - 'Actual Class Distribution': f'Success Rate: {final_accuracy_percent:.2f}%' - }) - - df_results = pd.DataFrame(results) - df_results.to_excel(excel_writer, sheet_name=f"SeqLen_{sequence_length}", index=False) diff --git a/preprocessing_new.py b/preprocessing.py similarity index 98% rename from preprocessing_new.py rename to preprocessing.py index 07cd7dd..7cd32f8 100644 --- a/preprocessing_new.py +++ b/preprocessing.py @@ -66,7 +66,6 @@ def process_file_15_min(file_path, user_label): # Load the dataset df = pd.read_csv(file_path, delimiter=';', low_memory=False) - # TODO: evtl. nicht nur iPhone date nutzen # Filter for iPhone devices iphone_df = df[df['device'].str.contains('iPhone', na=False)]