17 changed files with 1725 additions and 907 deletions
-
148.gitignore
-
BINEurope.zip
-
BINRest_of_the_World.zip
-
1baseline_results.json
-
806main.py
-
0old/DATA_PREPROCESSING_CODE.ipynb
-
0old/MODIFICATIONSANDINSTRUCTIONS.pdf
-
1old/baseline_results.json
-
0old/data_preprocessing.py
-
0old/data_preprocessing_main.py
-
0old/final-32-automated-code-new(1).ipynb
-
848old/main_old.py
-
0old/non_jupyter_version.py
-
466old/pipeline_old.py
-
3old/requirements.txt
-
358pipeline.py
-
1preprocessing.py
@ -1,147 +1,5 @@ |
|||
# ---> Python |
|||
# Byte-compiled / optimized / DLL files |
|||
__pycache__/ |
|||
*.py[cod] |
|||
*$py.class |
|||
|
|||
# C extensions |
|||
*.so |
|||
|
|||
# Distribution / packaging |
|||
.Python |
|||
build/ |
|||
develop-eggs/ |
|||
dist/ |
|||
downloads/ |
|||
eggs/ |
|||
.eggs/ |
|||
lib/ |
|||
lib64/ |
|||
parts/ |
|||
sdist/ |
|||
var/ |
|||
wheels/ |
|||
share/python-wheels/ |
|||
*.egg-info/ |
|||
.installed.cfg |
|||
*.egg |
|||
MANIFEST |
|||
|
|||
# PyInstaller |
|||
# Usually these files are written by a python script from a template |
|||
# before PyInstaller builds the exe, so as to inject date/other infos into it. |
|||
*.manifest |
|||
*.spec |
|||
|
|||
# Installer logs |
|||
pip-log.txt |
|||
pip-delete-this-directory.txt |
|||
|
|||
# Unit test / coverage reports |
|||
htmlcov/ |
|||
.tox/ |
|||
.nox/ |
|||
.coverage |
|||
.coverage.* |
|||
.cache |
|||
nosetests.xml |
|||
coverage.xml |
|||
*.cover |
|||
*.py,cover |
|||
.hypothesis/ |
|||
.pytest_cache/ |
|||
cover/ |
|||
|
|||
# Translations |
|||
*.mo |
|||
*.pot |
|||
|
|||
# Django stuff: |
|||
*.log |
|||
local_settings.py |
|||
db.sqlite3 |
|||
db.sqlite3-journal |
|||
|
|||
# Flask stuff: |
|||
instance/ |
|||
.webassets-cache |
|||
|
|||
# Scrapy stuff: |
|||
.scrapy |
|||
|
|||
# Sphinx documentation |
|||
docs/_build/ |
|||
|
|||
# PyBuilder |
|||
.pybuilder/ |
|||
target/ |
|||
|
|||
# Jupyter Notebook |
|||
.ipynb_checkpoints |
|||
|
|||
# IPython |
|||
profile_default/ |
|||
ipython_config.py |
|||
|
|||
# pyenv |
|||
# For a library or package, you might want to ignore these files since the code is |
|||
# intended to run in multiple environments; otherwise, check them in: |
|||
# .python-version |
|||
|
|||
# pipenv |
|||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. |
|||
# However, in case of collaboration, if having platform-specific dependencies or dependencies |
|||
# having no cross-platform support, pipenv may install dependencies that don't work, or not |
|||
# install all needed dependencies. |
|||
#Pipfile.lock |
|||
|
|||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow |
|||
__pypackages__/ |
|||
|
|||
# Celery stuff |
|||
celerybeat-schedule |
|||
celerybeat.pid |
|||
|
|||
# SageMath parsed files |
|||
*.sage.py |
|||
|
|||
# Environments |
|||
.env |
|||
.venv |
|||
env/ |
|||
venv/ |
|||
ENV/ |
|||
env.bak/ |
|||
venv.bak/ |
|||
|
|||
# Spyder project settings |
|||
.spyderproject |
|||
.spyproject |
|||
|
|||
# Rope project settings |
|||
.ropeproject |
|||
|
|||
# mkdocs documentation |
|||
/site |
|||
|
|||
# mypy |
|||
.mypy_cache/ |
|||
.dmypy.json |
|||
dmypy.json |
|||
|
|||
# Pyre type checker |
|||
.pyre/ |
|||
|
|||
# pytype static type analyzer |
|||
.pytype/ |
|||
|
|||
# Cython debug symbols |
|||
cython_debug/ |
|||
|
|||
.idea |
|||
working/tuner |
|||
working |
|||
.idea |
|||
__pycache__ |
|||
figures |
|||
baseline_results.json |
|||
baseline_results_v3.json |
|||
results |
|||
working |
|||
@ -0,0 +1 @@ |
|||
{"strategy":{"0":"most_frequent","1":"stratified","2":"uniform","3":"most_frequent","4":"stratified","5":"uniform","6":"most_frequent","7":"stratified","8":"uniform","9":"most_frequent","10":"stratified","11":"uniform","12":"most_frequent","13":"stratified","14":"uniform","15":"most_frequent","16":"stratified","17":"uniform"},"time used":{"0":"1HR","1":"1HR","2":"1HR","3":"1HR","4":"1HR","5":"1HR","6":"1HR","7":"1HR","8":"1HR","9":"1HR","10":"1HR","11":"1HR","12":"1HR","13":"1HR","14":"1HR","15":"1HR","16":"1HR","17":"1HR"},"sequence length":{"0":1,"1":1,"2":1,"3":6,"4":6,"5":6,"6":11,"7":11,"8":11,"9":16,"10":16,"11":16,"12":21,"13":21,"14":21,"15":26,"16":26,"17":26},"accuracy":{"0":0.0465710356,"1":0.0387109452,"2":0.0322263706,"3":0.0471161657,"4":0.0337124289,"5":0.0314784728,"6":0.0476990964,"7":0.0355116621,"8":0.0310989704,"9":0.0483239007,"10":0.035263387,"11":0.0343926861,"12":0.0489952585,"13":0.0363513208,"14":0.0325129826,"15":0.0497185741,"16":0.0356472795,"17":0.0300187617},"precision":{"0":0.0021688614,"1":0.038723827,"2":0.035357886,"3":0.0022199331,"4":0.0344564588,"5":0.0368125232,"6":0.0022752038,"7":0.0355258063,"8":0.034249105,"9":0.0023351994,"10":0.0353594615,"11":0.038321258,"12":0.0024005354,"13":0.0363413163,"14":0.0372264035,"15":0.0024719366,"16":0.0354808739,"17":0.0381814576},"recall":{"0":0.0465710356,"1":0.0387109452,"2":0.0322263706,"3":0.0471161657,"4":0.0337124289,"5":0.0314784728,"6":0.0476990964,"7":0.0355116621,"8":0.0310989704,"9":0.0483239007,"10":0.035263387,"11":0.0343926861,"12":0.0489952585,"13":0.0363513208,"14":0.0325129826,"15":0.0497185741,"16":0.0356472795,"17":0.0300187617},"f1 score":{"0":0.0041446997,"1":0.0386459741,"2":0.0329666692,"3":0.0042400894,"4":0.0340329362,"5":0.033074915,"6":0.004343239,"7":0.0354644534,"8":0.0315466073,"9":0.0044551105,"10":0.0352542225,"11":0.0349449461,"12":0.0045768279,"13":0.0362991035,"14":0.0333112368,"15":0.004709713,"16":0.0355062319,"17":0.0327189637}} |
|||
1
old/baseline_results.json
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,848 @@ |
|||
import json |
|||
import os |
|||
|
|||
import math |
|||
import numpy as np |
|||
import pandas as pd |
|||
import sklearn |
|||
from keras.src.regularizers import L1L2 |
|||
from matplotlib import pyplot as plt |
|||
from pandas import DataFrame |
|||
from sklearn.calibration import CalibratedClassifierCV |
|||
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis, LinearDiscriminantAnalysis |
|||
from sklearn.dummy import DummyClassifier |
|||
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier, VotingClassifier, \ |
|||
GradientBoostingClassifier, AdaBoostClassifier |
|||
from sklearn.gaussian_process import GaussianProcessClassifier |
|||
from sklearn.linear_model import PassiveAggressiveClassifier, RidgeClassifier, RidgeClassifierCV, SGDClassifier, \ |
|||
LogisticRegression, LogisticRegressionCV, Perceptron |
|||
from sklearn.metrics import confusion_matrix |
|||
from sklearn.mixture import GaussianMixture |
|||
from sklearn.model_selection import GridSearchCV |
|||
from sklearn.naive_bayes import GaussianNB, BernoulliNB, MultinomialNB |
|||
from sklearn.neighbors import KNeighborsClassifier, NearestCentroid |
|||
from sklearn.neural_network import MLPClassifier |
|||
from sklearn.preprocessing import MinMaxScaler |
|||
from sklearn.semi_supervised import LabelSpreading, LabelPropagation |
|||
from sklearn.svm import LinearSVC, SVC, OneClassSVM |
|||
from sklearn.tree import ExtraTreeClassifier, DecisionTreeClassifier |
|||
|
|||
from pipeline_old import ( |
|||
load_dataset, |
|||
filter_data, |
|||
filter_test_data, |
|||
prepare_user_data, |
|||
train_models, |
|||
evaluate_models, |
|||
prepare_data_for_model, model_type_gru, model_type_lstm, model_type_bilstm, train_models_v2, train_one_model, |
|||
eval_metrics, get_save_id, prepare_data_for_basic_algorithm, train_one_model_v2, |
|||
) |
|||
|
|||
year_str = 'Year' |
|||
month_str = 'Month' |
|||
day_str = 'Day' |
|||
date_str = 'Date' |
|||
time_str = 'Time' |
|||
day_of_week_str = 'DayOfWeek' |
|||
user_str = 'user' |
|||
split_str = 'split type' |
|||
data_split_str = 'data percentages' |
|||
month_split_str = 'month percentages' |
|||
threshold_str = 'threshold used' |
|||
with_threshold_str = 'WITH' |
|||
without_threshold_str = 'WITHOUT' |
|||
timespan_str = 'time used' |
|||
hour_timespan_str = '1HR' |
|||
min_timespan_str = '15MIN' |
|||
sequence_length_str = 'sequence length' |
|||
accuracy_str = 'accuracy' |
|||
precision_str = 'precision' |
|||
recall_str = 'recall' |
|||
f1_string = 'f1 score' |
|||
model_type_str = 'model type' |
|||
week_column_names = ['DayOfWeek_' + day for day in |
|||
['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday' ]] |
|||
figure_path = 'figures/' |
|||
predicitons_path = 'preds/' |
|||
|
|||
# === Configurable Parameters === |
|||
dataset_path = './Datasets/' |
|||
dataset_hrs_path = './Datasets/hours.json' |
|||
dataset_min_path = './Datasets/minutes.json' |
|||
DATA_PATH = dataset_path +'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx' |
|||
OUTPUT_EXCEL_PATH = './working/evaluation_results.xlsx' |
|||
result_filename_v1 = './working/evaluation_results.json' |
|||
result_filename_v2 = './working/evaluation_results_v2.json' |
|||
SEQUENCE_LENGTHS = [30, 25, 20, 15, 10, 5] # You can add more: [20, 25, 30] |
|||
|
|||
TRAINING_SCENARIO = [(2018, list(range(1, 13))), (2019, list(range(1, 10)))] |
|||
VALIDATION_SCENARIO = [(2019, [10, 11, 12])] |
|||
TEST_SCENARIO = [(2020, [1, 2])] # Jan–Feb 2020 only |
|||
|
|||
# === Optional display only === |
|||
predefined_training_scenarios = { |
|||
"Scenario 1": {"years_months": [(2018, list(range(1, 13))), (2019, list(range(1, 10)))]}, |
|||
"Scenario 2": {"years_months": [(2017, list(range(1, 13))), (2018, list(range(1, 13))), (2019, list(range(1, 10)))]} |
|||
} |
|||
predefined_validation_scenarios = { |
|||
"Scenario A": {"years_months": [(2019, [10, 11, 12])]} |
|||
} |
|||
|
|||
def create_dir(path): |
|||
""" |
|||
Creates a directory if it doesn't exist yet. |
|||
|
|||
:param path: The path to the directory |
|||
""" |
|||
if not os.path.exists(path): |
|||
os.makedirs(path) |
|||
|
|||
def remove_covid_data(df): |
|||
df = df[~(df[year_str]>=2020)] |
|||
return df |
|||
|
|||
def split_data_by_month_percentage(df, percentages): |
|||
train_p, valid_p, test_p = percentages |
|||
ids = df[[year_str, month_str]].drop_duplicates().sort_values([year_str, month_str]) |
|||
tr, va, te = np.split(ids, [int((train_p/100) * len(ids)), int(((train_p + valid_p)/100) * len(ids))]) |
|||
return df.merge(tr, on=[year_str, month_str], how='inner'), df.merge(va, on=[year_str, month_str], how='inner'), df.merge(te, on=[year_str, month_str], how='inner') |
|||
|
|||
def split_data_by_userdata_percentage(df, percentages, sample=100): |
|||
train_p, valid_p, test_p = percentages |
|||
tr, va, te = pd.DataFrame(), pd.DataFrame(), pd.DataFrame() |
|||
for user_id in df[user_str].unique(): |
|||
# !! following sample creates gaps in data if sample smaller 100 |
|||
user_data = df[df[user_str]==user_id].sample(frac=sample/ 100).sort_values([date_str]) # have to sort for time shift |
|||
u_tr, u_va, u_te = np.split(user_data, [int((train_p/100)*len(user_data)), int(((train_p+valid_p)/100)*len(user_data))]) |
|||
tr = pd.concat([tr, u_tr], ignore_index=True) |
|||
va = pd.concat([va, u_va], ignore_index=True) |
|||
te = pd.concat([te, u_te], ignore_index=True) |
|||
return tr, va, te |
|||
|
|||
|
|||
def main(): |
|||
# print("=== Training Scenario Setup ===") |
|||
# display_warning_about_2020_data() |
|||
# display_warnings_for_scenarios("training", predefined_training_scenarios, predefined_validation_scenarios) |
|||
|
|||
# print("\n=== Validation Scenario Setup ===") |
|||
# display_warning_about_2020_data() |
|||
# display_warnings_for_scenarios("validation", predefined_training_scenarios, predefined_validation_scenarios) |
|||
|
|||
# === Load and preprocess === |
|||
df = load_dataset(DATA_PATH) |
|||
|
|||
ALLUSERS32_15MIN_WITHOUTTHREHOLD = False |
|||
if('ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx' in DATA_PATH): |
|||
ALLUSERS32_15MIN_WITHOUTTHREHOLD = True |
|||
|
|||
training_data = filter_data(df, TRAINING_SCENARIO, ALLUSERS32_15MIN_WITHOUTTHREHOLD) |
|||
validation_data = filter_data(df, VALIDATION_SCENARIO, ALLUSERS32_15MIN_WITHOUTTHREHOLD) |
|||
|
|||
user_data_train = prepare_user_data(training_data) |
|||
user_data_val = prepare_user_data(validation_data) |
|||
|
|||
# === Train models === |
|||
best_models = train_models(user_data_train, user_data_val, sequence_lengths=SEQUENCE_LENGTHS) |
|||
|
|||
# === Load and evaluate test === |
|||
test_df = filter_test_data(df, TEST_SCENARIO) |
|||
evaluate_models(best_models, test_df, SEQUENCE_LENGTHS, OUTPUT_EXCEL_PATH, ALLUSERS32_15MIN_WITHOUTTHREHOLD) |
|||
|
|||
print(f"\n✅ All evaluations completed. Results saved to: {OUTPUT_EXCEL_PATH}") |
|||
|
|||
|
|||
def reduce_columns(df, filename): |
|||
if min_timespan_str in filename: |
|||
return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'] + week_column_names, errors='ignore') |
|||
else: |
|||
return df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek'], errors='ignore') |
|||
|
|||
|
|||
def reduce_columns_v3(df): |
|||
return df.drop(columns=[month_str, year_str, date_str]) |
|||
|
|||
def load_previous_results(filename): |
|||
results = pd.DataFrame() |
|||
if os.path.exists(filename): |
|||
results = pd.DataFrame(json.load(open(filename))) |
|||
return results |
|||
|
|||
def main_two_v2(model_type): |
|||
seq_length = range(10,31, 5) |
|||
for sequence_length in seq_length: |
|||
for data_filename in os.listdir(dataset_path): |
|||
timespan_id = hour_timespan_str |
|||
threshold_id = with_threshold_str |
|||
if min_timespan_str in data_filename: |
|||
timespan_id = min_timespan_str |
|||
if without_threshold_str in data_filename: |
|||
threshold_id = without_threshold_str |
|||
|
|||
results = load_previous_results(result_filename_v2) |
|||
if len(results) > 0: |
|||
if len(results[(results[timespan_str]==timespan_id) & |
|||
(results[threshold_str]==threshold_id) & |
|||
(results[sequence_length_str]==sequence_length) & |
|||
(results[model_type_str]==model_type)]) > 0: |
|||
continue |
|||
|
|||
file_path = os.path.join(dataset_path, data_filename) |
|||
df = load_dataset(file_path) |
|||
df = remove_covid_data(df) |
|||
|
|||
tr,val,te = split_data_by_userdata_percentage(df, percentages=(80,10,10)) |
|||
tr = reduce_columns(tr, data_filename) |
|||
val = reduce_columns(val, data_filename) |
|||
te = reduce_columns(te, data_filename) |
|||
|
|||
user_data_train = prepare_user_data(tr) |
|||
user_data_val = prepare_user_data(val) |
|||
|
|||
best_model = train_models_v2(user_data_train, user_data_val, |
|||
sequence_length=sequence_length, |
|||
model_type=model_type) |
|||
|
|||
results = load_previous_results(result_filename_v2) |
|||
results = pd.concat([results, |
|||
evaluate_model_on_test_data(model=best_model, |
|||
test_df=te, |
|||
sequence_length=sequence_length, |
|||
time_span_id=timespan_id, |
|||
threshold_id=threshold_id, |
|||
model_type=model_type, |
|||
split_id=data_split_str)], |
|||
ignore_index=True) |
|||
results.to_json(result_filename_v2) |
|||
|
|||
def main_two_v1(): |
|||
seq_length = [30, 25, 20, 15, 10, 5] # You can add more: [20, 25, 30] |
|||
results = pd.DataFrame() |
|||
if os.path.exists(result_filename_v1): |
|||
results = pd.DataFrame(json.load(open(result_filename_v1))) |
|||
for sequence_length in seq_length: |
|||
for data_filename in os.listdir(dataset_path): |
|||
for split_id, split_method in [(data_split_str, split_data_by_userdata_percentage),(month_split_str, split_data_by_month_percentage)]: |
|||
for model_type in [model_type_lstm, model_type_bilstm, model_type_gru]: |
|||
timespan_id = hour_timespan_str |
|||
threshold_id = with_threshold_str |
|||
if min_timespan_str in data_filename: |
|||
timespan_id = min_timespan_str |
|||
if without_threshold_str in data_filename: |
|||
threshold_id = without_threshold_str |
|||
if len(results) > 0: |
|||
if len(results[(results[split_str]==split_id) & |
|||
(results[timespan_str]==timespan_id) & |
|||
(results[threshold_str]==threshold_id) & |
|||
(results[sequence_length_str]==sequence_length) & |
|||
(results[model_type_str]==model_type)]) > 0: |
|||
continue |
|||
|
|||
file_path = os.path.join(dataset_path, data_filename) |
|||
df = load_dataset(file_path) |
|||
df = remove_covid_data(df) |
|||
tr,val,te = split_method(df, percentages=(80,10,10)) |
|||
tr = reduce_columns(tr, data_filename) |
|||
val = reduce_columns(val, data_filename) |
|||
te = reduce_columns(te, data_filename) |
|||
|
|||
user_data_train = prepare_user_data(tr) |
|||
user_data_val = prepare_user_data(val) |
|||
|
|||
best_models = train_models(user_data_train, user_data_val, sequence_lengths=[sequence_length], model_type=model_type) |
|||
|
|||
results = pd.concat([results, |
|||
evaluate_model_on_test_data(model=best_models[sequence_length]['model'], |
|||
test_df=te, split_id=split_id, |
|||
sequence_length=sequence_length, |
|||
time_span_id=timespan_id, |
|||
threshold_id=threshold_id, |
|||
model_type=model_type)], ignore_index=True) |
|||
results.to_json(result_filename_v1) |
|||
|
|||
# === Evaluation === |
|||
def evaluate_model_on_test_data(model, test_df,sequence_length, split_id, threshold_id, time_span_id, model_type): |
|||
user_data = prepare_user_data(test_df) |
|||
x, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length) |
|||
|
|||
y_pred = model.predict(x, verbose=0) |
|||
y_pred_classes = np.argmax(y_pred, axis=1) |
|||
|
|||
recall = sklearn.metrics.recall_score(y, y_pred_classes, average='weighted') |
|||
precision = sklearn.metrics.precision_score(y, y_pred_classes, average='weighted') |
|||
f1_score = sklearn.metrics.f1_score(y, y_pred_classes, average='weighted') |
|||
return pd.DataFrame({split_str:[split_id], threshold_str:[threshold_id], timespan_str:[time_span_id], |
|||
sequence_length_str:[sequence_length], |
|||
model_type_str:[model_type], recall_str:[recall], |
|||
precision_str:[precision], f1_string:[f1_score]}) |
|||
|
|||
def visualise_results_v1(): |
|||
results = pd.DataFrame(json.load(open(result_filename_v1))) |
|||
# Month split ist immer schlechter |
|||
results = results[results[split_str] == data_split_str] |
|||
with_threshold = results[results[threshold_str] == with_threshold_str] |
|||
without_threshold = results[results[threshold_str] == without_threshold_str] |
|||
fig, axes = plt.subplots(2, 3) |
|||
ax_col_id = 0 |
|||
ax_row_id = -1 |
|||
for timespan in [hour_timespan_str,min_timespan_str]: |
|||
ax_row_id +=1 |
|||
for model in [model_type_lstm, model_type_bilstm, model_type_gru]: |
|||
with_sub = with_threshold[(with_threshold[timespan_str] == timespan) & (with_threshold[model_type_str] == model)] |
|||
without_sub = without_threshold[(without_threshold[timespan_str] == timespan) & (without_threshold[model_type_str] == model)] |
|||
ax = axes[ax_row_id, ax_col_id] |
|||
ax.set_title(model+' '+timespan) |
|||
ax.plot(with_sub[sequence_length_str], with_sub[f1_string], label=with_threshold_str) |
|||
ax.plot(without_sub[sequence_length_str], without_sub[f1_string], label=without_threshold_str) |
|||
ax.legend() |
|||
ax_col_id +=1 |
|||
ax_col_id %= 3 |
|||
fig.tight_layout() |
|||
fig.savefig(figure_path+'v1_results.svg') |
|||
# Fazit: keine eindeutig besseren Versionen erkennbar |
|||
|
|||
|
|||
def visualise_results_v2(): |
|||
results = pd.DataFrame(json.load(open(result_filename_v2))) |
|||
with_threshold = results[results[threshold_str] == with_threshold_str] |
|||
without_threshold = results[results[threshold_str] == without_threshold_str] |
|||
fig, axes = plt.subplots(2, 3) |
|||
ax_col_id = 0 |
|||
ax_row_id = -1 |
|||
for timespan in [hour_timespan_str,min_timespan_str]: |
|||
ax_row_id +=1 |
|||
for model in [model_type_lstm, model_type_bilstm, model_type_gru]: |
|||
with_sub = with_threshold[(with_threshold[timespan_str] == timespan) & (with_threshold[model_type_str] == model)] |
|||
without_sub = without_threshold[(without_threshold[timespan_str] == timespan) & (without_threshold[model_type_str] == model)] |
|||
with_sub = with_sub.sort_values(sequence_length_str) |
|||
without_sub = without_sub.sort_values(sequence_length_str) |
|||
ax = axes[ax_row_id, ax_col_id] |
|||
ax.set_title(model+' '+timespan) |
|||
ax.plot(with_sub[sequence_length_str], with_sub[f1_string], label=with_threshold_str) |
|||
ax.plot(without_sub[sequence_length_str], without_sub[f1_string], label=without_threshold_str) |
|||
ax.legend() |
|||
ax_col_id +=1 |
|||
ax_col_id %= 3 |
|||
fig.tight_layout() |
|||
fig.savefig(figure_path+'v2_results.svg') |
|||
# Fazit: keine eindeutig besseren Versionen erkennbar |
|||
|
|||
|
|||
def test(model_type): |
|||
sequence_length = 20 |
|||
data_filename = os.listdir(dataset_path)[0] |
|||
timespan_id = hour_timespan_str |
|||
threshold_id = with_threshold_str |
|||
|
|||
file_path = os.path.join(dataset_path, data_filename) |
|||
df = load_dataset(file_path) |
|||
df = remove_covid_data(df) |
|||
results = pd.DataFrame() |
|||
|
|||
for percentage in [33,66,100]: |
|||
print('Percentage:', percentage) |
|||
tr,val,te = split_data_by_userdata_percentage(df, percentages=(80,10,10),sample=percentage) |
|||
tr = reduce_columns(tr, data_filename) |
|||
val = reduce_columns(val, data_filename) |
|||
te = reduce_columns(te, data_filename) |
|||
|
|||
user_data_train = prepare_user_data(tr) |
|||
user_data_val = prepare_user_data(val) |
|||
|
|||
best_model = train_models_v2(user_data_train, user_data_val, |
|||
sequence_length=sequence_length, |
|||
model_type=model_type) |
|||
|
|||
results = pd.concat([results, |
|||
evaluate_model_on_test_data(model=best_model, |
|||
test_df=te, |
|||
sequence_length=sequence_length, |
|||
time_span_id=timespan_id, |
|||
threshold_id=threshold_id, |
|||
model_type=model_type, |
|||
split_id=data_split_str)], |
|||
ignore_index=True) |
|||
print(results) |
|||
|
|||
def manual_tuning(model_type): |
|||
# load dataset |
|||
sequence_length = 20 |
|||
data_filename = 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx' |
|||
timespan_id = min_timespan_str |
|||
threshold_id = with_threshold_str |
|||
|
|||
file_path = os.path.join(dataset_path, data_filename) |
|||
df = load_dataset(file_path) |
|||
df = remove_covid_data(df) |
|||
|
|||
tr, val, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=100) |
|||
tr = reduce_columns(tr, data_filename) |
|||
val = reduce_columns(val, data_filename) |
|||
te = reduce_columns(te, data_filename) |
|||
|
|||
user_data_train = prepare_user_data(tr) |
|||
user_data_val = prepare_user_data(val) |
|||
|
|||
# fit and evaluate model |
|||
# config |
|||
repeats = 3 |
|||
n_batch = 1024 |
|||
n_epochs = 500 |
|||
n_neurons = 16 |
|||
l_rate = 1e-4 |
|||
reg = L1L2(l1=0.0, l2=0.0) |
|||
|
|||
history_list = list() |
|||
# run diagnostic tests |
|||
for i in range(repeats): |
|||
history = train_one_model(user_data_train, user_data_val, n_batch, n_epochs, |
|||
n_neurons, l_rate, reg, |
|||
sequence_length=sequence_length, |
|||
model_type=model_type) |
|||
history_list.append(history) |
|||
for metric in ['p', 'r', 'f1']: |
|||
for history in history_list: |
|||
plt.plot(history['train_'+metric], color='blue') |
|||
plt.plot(history['test_'+metric], color='orange') |
|||
plt.savefig(figure_path+metric+'_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+ |
|||
str(n_batch)+'_l'+str(l_rate)+'_diagnostic.png') |
|||
plt.clf() |
|||
print('Done') |
|||
|
|||
|
|||
def upsampling(df): |
|||
max_user_data = df[user_str].value_counts().max() |
|||
for user in df[user_str].unique(): |
|||
user_data = df[df[user_str]==user] |
|||
user_count = user_data.shape[0] |
|||
times = max_user_data / user_count |
|||
before_comma = math.floor(times) |
|||
after_comma = times % 1 |
|||
after_comma_data = user_data.sample(frac=after_comma) |
|||
for i in range(1, before_comma): |
|||
df = pd.concat([df, user_data], ignore_index=True) |
|||
df = pd.concat([df, after_comma_data], ignore_index=True) |
|||
return df |
|||
|
|||
|
|||
def manual_tuning_v3(model_type): |
|||
# TODO: hrs/min |
|||
sequence_length = 1 |
|||
|
|||
tr, val, te = get_prepared_data_v3(dataset_hrs_path) |
|||
|
|||
# fit and evaluate model |
|||
# config |
|||
repeats = 3 |
|||
n_batch = 1024 |
|||
n_epochs = 10 |
|||
n_neurons = 256 |
|||
n_neurons2 = 512 |
|||
n_neurons3 = 512 |
|||
n_neurons4 = 128 |
|||
l_rate = 1e-2 |
|||
d1 = 256 |
|||
reg1 = L1L2(l1=0.0, l2=0.001) |
|||
r1 = '0001' |
|||
reg2 = L1L2(l1=0.0, l2=0.1) |
|||
r2 = '01' |
|||
|
|||
history_list = list() |
|||
# run diagnostic tests |
|||
for i in range(repeats): |
|||
history = train_one_model(tr, val, n_batch, n_epochs, |
|||
n_neurons,n_neurons2, n_neurons3, n_neurons4, l_rate, d1, r1, reg1, r2, reg2, |
|||
sequence_length=sequence_length, |
|||
model_type=model_type) |
|||
history_list.append(history) |
|||
for metric in ['acc', 'p', 'r', 'f1']: |
|||
for history in history_list: |
|||
plt.plot(history['train_'+metric], color='blue') |
|||
plt.plot(history['test_'+metric], color='orange') |
|||
plt.savefig(figure_path+'v3/'+metric+get_save_id(n_epochs, n_neurons, n_neurons2, n_neurons3,n_neurons4, n_batch, l_rate, d1, r1, r2) |
|||
+'.png') |
|||
plt.clf() |
|||
print('Done') |
|||
|
|||
|
|||
|
|||
def calculate_baselines(): |
|||
file_combinations = [(hour_timespan_str, with_threshold_str,'ALL32USERS1HR_WITHTHRESHOLD.xlsx'), |
|||
(min_timespan_str, with_threshold_str, 'ALL32USERS15MIN_WITHTHRESHOLD.xlsx'), |
|||
(min_timespan_str, without_threshold_str, 'ALLUSERS32_15MIN_WITHOUTTHREHOLD.xlsx'), |
|||
(hour_timespan_str, without_threshold_str, 'ALLUSERS_32_1HR_WITHOUT_THRESHOLD.xlsx'), |
|||
] |
|||
baseline_res = pd.DataFrame() |
|||
for timespan_id, threshold_id, filename in file_combinations: |
|||
file_path = os.path.join(dataset_path, filename) |
|||
df = load_dataset(file_path) |
|||
df = remove_covid_data(df) |
|||
|
|||
_, _, te = split_data_by_userdata_percentage(df, percentages=(80, 10, 10), sample=20) |
|||
te = reduce_columns(te, filename) |
|||
user_data_te = prepare_user_data(te) |
|||
for sequence_length in range(5,30, 5): |
|||
x, y = prepare_data_for_model(user_data=user_data_te, sequence_length=sequence_length) |
|||
|
|||
for strategy in ['most_frequent', 'stratified', 'uniform']: |
|||
cls = DummyClassifier(strategy=strategy) |
|||
cls.fit(x,y) |
|||
y_pred = cls.predict(x) |
|||
acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred) |
|||
baseline_res = pd.concat([baseline_res, |
|||
DataFrame({ 'strategy':[strategy], threshold_str:[threshold_id], |
|||
timespan_str:[timespan_id], sequence_length_str:[sequence_length], |
|||
accuracy_str:[acc],precision_str:[p],recall_str:[r], |
|||
f1_string:f1})], ignore_index=True) |
|||
baseline_res.to_json('baseline_results.json') |
|||
print('Done') |
|||
|
|||
def get_prepared_data_v3(filename, sample=100, print_unique=False): |
|||
df = pd.read_json(filename) |
|||
df = remove_covid_data(df) |
|||
|
|||
# remove users which are a complete subset of another user (but keep one) |
|||
users_to_remove = [] |
|||
for user_a in df[user_str].unique(): |
|||
for user_b in df[user_str].unique(): |
|||
if user_a != user_b: |
|||
data = pd.concat([df[df[user_str]==user_a], df[df[user_str]==user_b]]) |
|||
columns = data.columns.tolist() |
|||
columns.remove(user_str) |
|||
|
|||
no_dup = data.drop_duplicates(columns, keep=False) |
|||
if len(no_dup[no_dup[user_str]==user_a]) == 0: |
|||
if print_unique: |
|||
print(user_a, 'is subset of',user_b) |
|||
if user_b not in users_to_remove: |
|||
users_to_remove.append(user_a) |
|||
df = df[~df[user_str].isin(users_to_remove)] |
|||
|
|||
# bin steps per hour TODO: adjust for minutes |
|||
for hour in ['Hour_'+str(i) for i in range(24)]: |
|||
hour_data = df[hour] |
|||
# smaller 1000 - round to 10 |
|||
a = ((hour_data[hour_data<1000]/10).round()*10) |
|||
# between 1000 and 10000 - round to next 100 |
|||
b = ((hour_data[(hour_data>=1000)& (hour_data<10000)]/100).round()*100) |
|||
# higher or equal 10000 - one class |
|||
c = hour_data[hour_data > 10000] |
|||
c = pd.Series(data={ind:10000 for ind in c.index}, index=c.index) |
|||
new = pd.concat([a, b, c]).sort_index().astype(int) |
|||
df[hour] = new |
|||
|
|||
# remove users with too little data (optional) |
|||
#value_counts = df[user_str].value_counts() |
|||
#df = df[df[user_str].isin(value_counts[value_counts>200].index)] |
|||
min_datapoints = 500 # 500 leads to at least 75 datapoints in the valid set |
|||
users_to_remove = set() |
|||
cols = df.columns.tolist() |
|||
cols.remove(user_str) |
|||
reduced = df.drop_duplicates(subset=cols, keep=False) |
|||
for user_id in df[user_str].unique(): |
|||
subset = df[df[user_str] == user_id] |
|||
reduced_subset = reduced[reduced[user_str] == user_id] |
|||
if print_unique: |
|||
print(user_id, len(subset), len(reduced_subset)) |
|||
if len(reduced_subset) < min_datapoints: |
|||
users_to_remove.add(user_id) |
|||
if print_unique: |
|||
print('removing', user_id) |
|||
|
|||
df = df[~df[user_str].isin(users_to_remove)] |
|||
|
|||
tr, val, te = split_data_by_userdata_percentage(df, percentages=(70, 15, 15), sample=sample) |
|||
tr = reduce_columns_v3(tr) |
|||
val = reduce_columns_v3(val) |
|||
te = reduce_columns_v3(te) |
|||
|
|||
|
|||
|
|||
print('Train: Users', len(tr[user_str].unique()), 'mean num datapoins:', tr[user_str].value_counts().mean()) |
|||
print('Valid: Users', len(val[user_str].unique()), 'mean num datapoins:', val[user_str].value_counts().mean()) |
|||
print('Test: Users', len(te[user_str].unique()), 'mean num datapoins:', te[user_str].value_counts().mean()) |
|||
|
|||
tr, val, te = add_features(tr), add_features(val), add_features(te) |
|||
|
|||
scaler = MinMaxScaler() |
|||
scaler.fit(tr.drop(columns=[user_str])) |
|||
|
|||
return scale_dataset(scaler, tr), scale_dataset(scaler, val), scale_dataset(scaler, te) |
|||
|
|||
|
|||
def scale_dataset(scaler, df): |
|||
y = df[user_str] |
|||
x_scaled = scaler.transform(df.drop(columns=[user_str])) |
|||
x_scaled = pd.DataFrame(x_scaled) |
|||
x_scaled.columns = df.drop(columns=[user_str]).columns |
|||
|
|||
df_scaled = pd.concat([x_scaled, pd.DataFrame(y.reset_index()[user_str])], axis=1) |
|||
# df_scaled.columns = df.columns |
|||
return prepare_user_data(df_scaled) |
|||
|
|||
|
|||
def calculate_baselines_v3(): |
|||
file_combinations = [(hour_timespan_str, dataset_hrs_path), |
|||
# (min_timespan_str, dataset_min_path), # TODO: dataset bining not ready for minutes |
|||
] |
|||
baseline_res = pd.DataFrame() |
|||
for timespan_id, filename in file_combinations: |
|||
_, _, te = get_prepared_data_v3(filename) |
|||
for sequence_length in range(1,30,5): |
|||
x, y = prepare_data_for_model(user_data=te, sequence_length=sequence_length) |
|||
|
|||
for strategy in ['most_frequent', 'stratified', 'uniform']: |
|||
cls = DummyClassifier(strategy=strategy) |
|||
cls.fit(x,y) |
|||
y_pred = cls.predict(x) |
|||
acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred) |
|||
baseline_res = pd.concat([baseline_res, |
|||
DataFrame({ 'strategy':[strategy], |
|||
timespan_str:[timespan_id], sequence_length_str:[sequence_length], |
|||
accuracy_str:[acc],precision_str:[p],recall_str:[r], |
|||
f1_string:f1})], ignore_index=True) |
|||
baseline_res.to_json('baseline_results_v3.json') |
|||
print('Done') |
|||
|
|||
|
|||
def hypertune_basic_algorithms(): |
|||
# TODO: hrs/min |
|||
# iterate over sequence lengths |
|||
sequence_length = 7 |
|||
|
|||
tr, val, te = get_prepared_data_v3(dataset_hrs_path) |
|||
|
|||
x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) |
|||
x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) |
|||
|
|||
random_state = 17 |
|||
results = pd.DataFrame() |
|||
for tag, clf, grid in [ |
|||
('GradientBoosting', GradientBoostingClassifier(random_state=random_state), |
|||
{'loss': ['log_loss', 'exponential'], |
|||
'learning_rate': [0.1, 0.5, 1.0,2.0, 5.0], |
|||
'n_estimators': [10, 50, 100, 150, 200], |
|||
'subsample': [0.1, 0.5, 1.0], |
|||
'criterion': ['friedman_mse', 'squared_error'], |
|||
'min_samples_split': [2, 10, 100], |
|||
'min_samples_leaf': [1, 5, 10], |
|||
'min_weight_fraction_leaf': [0.0, 0.1, 0.5], |
|||
'max_depth': [None, 2, 10, 100], |
|||
'min_impurity_decrease': [0.0, 0.1, 0.5], |
|||
'max_features': ['sqrt', 'log2', None, 10, 20], |
|||
'max_leaf_nodes': [None, 1, 5, 10], |
|||
}), |
|||
('Bernoulli', BernoulliNB(), {'fit_prior': [True, False], |
|||
'binarize': [0.0, 0.1, 0.25, 0.5, 0.75], |
|||
'force_alpha': [True, False], |
|||
'alpha':[0.0, 0.25, 0.5, 0.75, 1.0]}), |
|||
('extra trees', ExtraTreesClassifier(random_state=random_state, n_jobs=1), |
|||
{'n_estimators': [10, 50, 100, 150, 200], |
|||
'criterion': ['gini', 'entropy', 'log_loss'], |
|||
'max_depth': [None, 2, 10, 100], |
|||
'min_samples_split': [2, 10, 100], |
|||
'min_samples_leaf': [1, 5, 10], |
|||
'min_weight_fraction_leaf': [0.0, 0.1, 0.5], |
|||
'max_features': ['sqrt', 'log2', None, 10, 20], |
|||
'max_leaf_nodes': [None, 1, 5, 10], |
|||
'min_impurity_decrease': [0.0, 0.1, 0.5], |
|||
'bootstrap': [True, False], |
|||
'class_weight': [None, 'balanced', 'balanced_subsample'], |
|||
'max_samples': [None, 0.1, 0.2, 0.3]} |
|||
), |
|||
('random forest', RandomForestClassifier(random_state=random_state, n_jobs=1), |
|||
{'n_estimators':[10, 50, 100, 150, 200], |
|||
'criterion':['gini', 'entropy', 'log_loss'], |
|||
'max_depth':[None, 2, 10,100], |
|||
'min_samples_split': [2,10,100], |
|||
'min_samples_leaf':[1,5,10], |
|||
'min_weight_fraction_leaf':[0.0,0.1, 0.5], |
|||
'max_features':['sqrt', 'log2', None, 10, 20], |
|||
'max_leaf_nodes':[None, 1, 5, 10], |
|||
'min_impurity_decrease':[0.0, 0.1, 0.5], |
|||
'bootstrap':[True, False], |
|||
'class_weight':[None, 'balanced', 'balanced_subsample'], |
|||
'max_samples':[None,0.1, 0.2, 0.3]}) |
|||
]: |
|||
grid_search = GridSearchCV( |
|||
estimator=clf, param_grid=grid, scoring='f1_weighted', cv=5, n_jobs=1) |
|||
grid_search.fit(x_tr, y_tr) |
|||
|
|||
best_model = grid_search.best_estimator_ |
|||
y_pred = best_model.predict(x_val) |
|||
acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) |
|||
results = pd.concat([results, DataFrame({ 'params': str(grid_search.best_params_), |
|||
'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) |
|||
results.to_json('basic_ht_results.json') |
|||
print('Done') |
|||
|
|||
def test_basic_algorithms(): |
|||
# TODO: hrs/min |
|||
# TODO: iterate over sequence lengths |
|||
sequence_length = 21 |
|||
|
|||
tr, val, te = get_prepared_data_v3(dataset_hrs_path) |
|||
|
|||
x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) |
|||
x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) |
|||
|
|||
random_state = 17 |
|||
results = pd.DataFrame() |
|||
for tag, clf in [ |
|||
# ('Label Propagation', LabelPropagation()), |
|||
# ('Label Spreading', LabelSpreading()), |
|||
# ('VBGMM', GaussianMixture(random_state=random_state)), |
|||
# ('linear discrimenant analysis', LinearDiscriminantAnalysis()), |
|||
# ('discriminent analysis', QuadraticDiscriminantAnalysis()), |
|||
# ('oneclassSVM', OneClassSVM()), |
|||
# ('mlp', MLPClassifier(random_state=random_state)), |
|||
# ('Perceptron', Perceptron(random_state=random_state)), |
|||
# ('SVC', SVC(random_state=random_state)), |
|||
#('logisticRegression', LogisticRegression(random_state=random_state)), |
|||
#('logisticRegressionCV', LogisticRegressionCV(random_state=random_state)), |
|||
#('multinomialNB', MultinomialNB()), |
|||
#('nearestCentroid', NearestCentroid()), |
|||
#('linearSVC', LinearSVC(random_state=random_state)), |
|||
#('ada boost', AdaBoostClassifier(random_state=random_state)), |
|||
#('GradientBoosting', GradientBoostingClassifier(random_state=random_state)), |
|||
#('Bernoulli', BernoulliNB()), |
|||
#('claibrated', CalibratedClassifierCV()), |
|||
#('naive Bayes', GaussianNB()), |
|||
#('sgd', SGDClassifier(random_state=random_state)), |
|||
#('ridgeCV', RidgeClassifierCV()), |
|||
# ('ridge', RidgeClassifier(random_state=random_state)), |
|||
# ('passiveAggressive', PassiveAggressiveClassifier(random_state=random_state)), |
|||
# ('knn', KNeighborsClassifier()), |
|||
# ('bagging', BaggingClassifier(random_state=random_state)), |
|||
# ('decision tree', DecisionTreeClassifier(random_state=random_state)), |
|||
# ('extra tree', ExtraTreeClassifier(random_state=random_state)), |
|||
# ('extra trees', ExtraTreesClassifier(random_state=random_state)), |
|||
('random forest', RandomForestClassifier(random_state=random_state)) |
|||
]: |
|||
clf.fit(x_tr, y_tr) |
|||
y_pred = clf.predict(x_val) |
|||
acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) |
|||
results = pd.concat([results, DataFrame({ 'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) |
|||
print('Done') |
|||
|
|||
|
|||
def add_features(df): |
|||
# indicator weekend |
|||
df['weekend'] = df[day_of_week_str + '_Saturday']+df[day_of_week_str + '_Sunday'] |
|||
# sum of steps per day |
|||
df['day_total'] = sum([df['Hour_'+str(i)] for i in range(23)]) |
|||
# sum of steps morning, afternoon, evening, night |
|||
df['morning_total'] = sum([df['Hour_' + str(i)] for i in range(6,13)]) |
|||
df['afternoon_total'] = sum([df['Hour_' + str(i)] for i in range(13,19)]) |
|||
df['evening_total'] = sum([df['Hour_' + str(i)] for i in range(19,23)]) |
|||
df['night_total'] = sum([df['Hour_' + str(i)] for i in [23,0,1,2,3,4,5]]) |
|||
return df |
|||
|
|||
|
|||
def feature_engineering(): |
|||
sequence_length = 1 |
|||
|
|||
tr, val, te = get_prepared_data_v3(dataset_hrs_path, print_unique=True) |
|||
|
|||
x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) |
|||
x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) |
|||
|
|||
random_state = 17 |
|||
clf=RandomForestClassifier(random_state=random_state) |
|||
clf.fit(x_tr, y_tr) |
|||
y_pred = clf.predict(x_val) |
|||
acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) |
|||
cf = confusion_matrix(y_pred=y_pred, y_true=y_val) |
|||
# TODO: welche funktionieren schlecht? warum? |
|||
# TODO: auf minutes umändern |
|||
|
|||
print('Done') |
|||
|
|||
|
|||
def test_sequence_length_on_approach(clf = RandomForestClassifier(random_state=17)): |
|||
tr, val, te = get_prepared_data_v3(dataset_hrs_path) |
|||
|
|||
results_train = pd.DataFrame() |
|||
results_valid = pd.DataFrame() |
|||
for sequence_length in range(1, 60, 5): |
|||
x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) |
|||
x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) |
|||
clf.fit(x_tr, y_tr) |
|||
acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=clf.predict(x_val)) |
|||
results_valid = pd.concat([results_valid, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) |
|||
acc, p, r, f1 = eval_metrics(y_true=y_tr, y_pred=clf.predict(x_tr)) |
|||
results_train = pd.concat([results_train, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) |
|||
|
|||
fig = plt.figure() |
|||
|
|||
for frame in [results_train, results_valid]: |
|||
plt.plot(frame[sequence_length_str], frame[f1_string]) |
|||
|
|||
plt.show() |
|||
print('') |
|||
|
|||
def manual_tuning_v4(model_type): |
|||
# TODO: hrs/min |
|||
tr, val, te = get_prepared_data_v3(dataset_hrs_path) |
|||
n_epochs= 20 |
|||
n_neurons = 1024 |
|||
results_train = pd.DataFrame() |
|||
results_valid = pd.DataFrame() |
|||
for sequence_length in range(1, 50, 5): |
|||
train_data = prepare_data_for_model(user_data=tr, sequence_length=sequence_length) |
|||
val_data = prepare_data_for_model(user_data=val, sequence_length=sequence_length) |
|||
|
|||
# fit and evaluate model |
|||
history_list = list() |
|||
repeats = 3 |
|||
# run diagnostic tests |
|||
for i in range(repeats): |
|||
history = train_one_model_v2(train_data, val_data, 1024, n_epochs, |
|||
n_neurons, sequence_length=sequence_length, |
|||
model_type=model_type) |
|||
history_list.append(history) |
|||
results = pd.concat([history.tail(1) for history in history_list]).mean() |
|||
results_train = pd.concat([results_train, |
|||
DataFrame({sequence_length_str:[sequence_length], |
|||
accuracy_str:[results['train_acc']], |
|||
precision_str:[results['train_p']], |
|||
recall_str:[results['train_r']], |
|||
f1_string:[results['train_f1']]})], ignore_index=True) |
|||
results_valid = pd.concat([results_valid, |
|||
DataFrame({sequence_length_str:[sequence_length], |
|||
accuracy_str:[results['test_acc']], |
|||
precision_str:[results['test_p']], |
|||
recall_str:[results['test_r']], |
|||
f1_string:[results['test_f1']]})], ignore_index=True) |
|||
|
|||
fig = plt.figure() |
|||
for frame in [results_train, results_valid]: |
|||
plt.plot(frame[sequence_length_str], frame[f1_string]) |
|||
|
|||
plt.show() |
|||
print('Done') |
|||
|
|||
|
|||
if __name__ == "__main__": |
|||
# Ordner erstellen, die benötigt werden |
|||
create_dir('results/') |
|||
create_dir(figure_path) |
|||
pd.options.mode.copy_on_write = True |
|||
|
|||
main_two_v1() |
|||
visualise_results_v1() |
|||
#test(model_type=model_type_gru) |
|||
# main_two_v2(model_type=model_type_gru) |
|||
#visualise_results_v2() |
|||
#manual_tuning(model_type=model_type_lstm) |
|||
#calculate_baselines() |
|||
|
|||
#### Ab hier aktuell (21.01.2026) |
|||
#calculate_baselines() |
|||
# manual_tuning_v3(model_type=model_type_lstm) |
|||
#test_basic_algorithms() |
|||
# test_basic_algorithm_on_sequence_lengths() |
|||
manual_tuning_v4(model_type=model_type_lstm) |
|||
#feature_engineering() |
|||
#hypertune_basic_algorithms() |
|||
print('Done') |
|||
@ -0,0 +1,466 @@ |
|||
import random |
|||
|
|||
import keras_tuner |
|||
import numpy as np |
|||
import pandas as pd |
|||
import shutil |
|||
|
|||
from keras import Input |
|||
from keras.src.losses import SparseCategoricalCrossentropy |
|||
from keras.src.metrics import F1Score, Precision, Recall, Accuracy, SparseCategoricalAccuracy |
|||
from pandas import ExcelWriter, DataFrame |
|||
from tensorflow.keras.models import Sequential |
|||
from tensorflow.keras.layers import LSTM, Dense, Dropout, Bidirectional,GRU |
|||
from tensorflow.keras.optimizers import Adam |
|||
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping |
|||
from keras_tuner import RandomSearch |
|||
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix |
|||
|
|||
epochs = 5#50 |
|||
model_type_gru = 'GRU' |
|||
model_type_lstm = 'LSTM' |
|||
model_type_bilstm = 'BiLSTM' |
|||
|
|||
|
|||
# === Data functions === |
|||
def load_dataset(file_path): |
|||
return pd.read_excel(file_path) |
|||
|
|||
def filter_data(df, scenario, ALLUSERS32_15MIN_WITHOUTREHOLD): |
|||
filtered = pd.DataFrame() |
|||
for year, months in scenario: |
|||
filtered = pd.concat([filtered, df[(df['Year'] == year) & (df['Month'].isin(months))]]) |
|||
|
|||
if ALLUSERS32_15MIN_WITHOUTREHOLD: |
|||
return filtered.drop(columns=['Month', 'Year', 'date', 'DayOfWeek']) |
|||
else: |
|||
return filtered.drop(columns=['Month', 'Year', 'date']) |
|||
|
|||
def filter_test_data(df, scenario): |
|||
data_parts = [] |
|||
for year, months in scenario: |
|||
part = df[(df['Year'] == year) & (df['Month'].isin(months))] |
|||
data_parts.append(part) |
|||
return pd.concat(data_parts, ignore_index=True) |
|||
|
|||
def prepare_user_data(df): |
|||
#df_sorted = df.sort_values(by='user').reset_index(drop=True) |
|||
users = df['user'].unique() |
|||
return {user: df[df['user'] == user] for user in users} |
|||
|
|||
def make_sequences(data, sequence_length): |
|||
x, y = [], [] |
|||
features = data.drop('user', axis=1).values |
|||
labels = data['user'].values |
|||
for i in range(len(features) - sequence_length+1): # with overlap on days |
|||
# for i in range(0, len(features) - sequence_length + 1, sequence_length): # without overlap on days |
|||
x.append(features[i:i + sequence_length]) |
|||
y.append(labels[i + sequence_length-1]) |
|||
return x, y |
|||
|
|||
def prepare_data_for_basic_algorithm(user_data, sequence_length): |
|||
combined = pd.DataFrame() |
|||
for user, data in user_data.items(): |
|||
x_new, y_new = make_sequences(data, sequence_length) |
|||
if len(x_new)>0: |
|||
var = [[pd.DataFrame(a[s]) for s in range(sequence_length)] for a in x_new] |
|||
df_var = pd.concat([pd.concat(seq_list).T for seq_list in var]) |
|||
df_var['user'] = user |
|||
combined = pd.concat([combined, df_var], ignore_index=True) |
|||
return combined.drop(columns=['user']), combined['user'] |
|||
|
|||
def prepare_data_for_model(user_data, sequence_length, print_counts=False): |
|||
x, y = [], [] |
|||
combined = pd.DataFrame() |
|||
for user, data in user_data.items(): |
|||
x_new, y_new = make_sequences(data, sequence_length) |
|||
x = x + x_new |
|||
y = y + y_new |
|||
if print_counts and len(x_new)>0: |
|||
var = [[pd.DataFrame(a[s])for s in range(sequence_length)] for a in x_new ] |
|||
df_var = pd.concat([pd.concat(seq_list).T for seq_list in var]) |
|||
df_var['user'] = user |
|||
combined = pd.concat([combined, df_var], ignore_index=True) |
|||
if print_counts: |
|||
combined_ohne = combined.drop('user', axis=1) |
|||
print('Alle', len(combined)) |
|||
print('Unique mit user', len(combined.drop_duplicates())) |
|||
print('Unique ohne user', len(combined_ohne.drop_duplicates())) |
|||
print('Unique') |
|||
print(combined.drop_duplicates()['user'].value_counts()) |
|||
print('Alle') |
|||
print(combined['user'].value_counts()) |
|||
random.Random(17).shuffle(x) |
|||
random.Random(17).shuffle(y) |
|||
x = np.array(x) |
|||
y = np.array(y) |
|||
return x,y |
|||
|
|||
# === Training & Validation === |
|||
def train_models(user_data, user_data_val, sequence_lengths, tuner_dir="./working/tuner", model_type=model_type_lstm): |
|||
best_models = {} |
|||
early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True) |
|||
lr_scheduler = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, verbose=1) |
|||
users = list(user_data.keys()) |
|||
|
|||
shutil.rmtree(tuner_dir, ignore_errors=True) |
|||
|
|||
for sequence_length in sequence_lengths: |
|||
print(f"\n=== Training for Sequence Length: {sequence_length} ===") |
|||
X, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length) |
|||
X_val, y_val = prepare_data_for_model(user_data=user_data_val, sequence_length=sequence_length) |
|||
|
|||
if X.shape[0] == 0 or X_val.shape[0] == 0: |
|||
print(f"⚠️ Skipped sequence length {sequence_length} due to insufficient data.") |
|||
continue |
|||
|
|||
n_features = X.shape[2] |
|||
|
|||
def build_model(hp): |
|||
model = Sequential() |
|||
if model_type==model_type_bilstm: |
|||
model.add(Bidirectional(LSTM(units=hp.Int('units', 32, 256, step=2), |
|||
input_shape=(sequence_length, n_features)))) |
|||
if model_type==model_type_lstm: |
|||
model.add(LSTM(units=hp.Int('units', 32, 256, step=2), |
|||
input_shape=(sequence_length, n_features))) |
|||
if model_type==model_type_gru: |
|||
model.add(GRU(units=hp.Int('units', 32, 256, step=2), |
|||
input_shape=(sequence_length, n_features))) |
|||
model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.5, step=0.1))) |
|||
model.add(Dense(len(users), activation='softmax')) |
|||
model.compile( |
|||
optimizer=Adam(learning_rate=hp.Choice('learning_rate', [1e-2, 1e-3, 1e-4])), |
|||
loss='sparse_categorical_crossentropy', |
|||
metrics=['accuracy'] |
|||
) |
|||
return model |
|||
|
|||
tuner = RandomSearch( |
|||
build_model, |
|||
objective='val_loss', |
|||
max_trials=30, |
|||
executions_per_trial=2, |
|||
directory=tuner_dir, |
|||
project_name=f'lstm_seq_{sequence_length}' |
|||
) |
|||
|
|||
tuner.search(X, y, epochs=epochs, validation_data=(X_val, y_val), |
|||
callbacks=[early_stopping, lr_scheduler], verbose=0) |
|||
|
|||
best_hps = tuner.get_best_hyperparameters(1)[0] |
|||
best_model = tuner.hypermodel.build(best_hps) |
|||
best_model.fit(X, y, epochs=epochs, validation_data=(X_val, y_val), |
|||
callbacks=[early_stopping, lr_scheduler], verbose=0) |
|||
|
|||
best_models[sequence_length] = { |
|||
'model': best_model, |
|||
'best_hyperparameters': { |
|||
'units': best_hps.get('units'), |
|||
'dropout_rate': best_hps.get('dropout_rate'), |
|||
'learning_rate': best_hps.get('learning_rate') |
|||
} |
|||
} |
|||
|
|||
return best_models |
|||
|
|||
# === Training & Validation === |
|||
def train_models_v2(user_data, user_data_val, sequence_length, model_type): |
|||
tuner_dir = "./working/tuner/"+model_type |
|||
#val_metric = 'val_f1' |
|||
val_metric = 'val_precision' |
|||
|
|||
early_stopping = EarlyStopping(monitor=val_metric, patience=3, restore_best_weights=True) |
|||
lr_scheduler = ReduceLROnPlateau(monitor=val_metric, factor=0.5, patience=2) |
|||
|
|||
shutil.rmtree(tuner_dir, ignore_errors=True) |
|||
|
|||
x, y = prepare_data_for_model(user_data=user_data, sequence_length=sequence_length) |
|||
x_val, y_val = prepare_data_for_model(user_data=user_data_val, sequence_length=sequence_length) |
|||
|
|||
n_features = x.shape[2] |
|||
users = list(user_data.keys()) |
|||
|
|||
#y_val = np.array(y_val).reshape(-1, 1) |
|||
#y = np.array(y).reshape(-1, 1) |
|||
|
|||
def build_model(hp): |
|||
units_hp = hp.Int('units', 2, 8, step=2, sampling="log") |
|||
# units_hp = hp.Int('units', 2, 256, step=2, sampling="log") |
|||
|
|||
model = Sequential() |
|||
model.add(Input((sequence_length, n_features))) |
|||
if model_type==model_type_bilstm: |
|||
model.add(Bidirectional(LSTM(units=units_hp))) |
|||
if model_type==model_type_lstm: |
|||
model.add(LSTM(units=units_hp)) |
|||
if model_type==model_type_gru: |
|||
model.add(GRU(units=units_hp)) |
|||
model.add(Dropout(hp.Float('dropout_rate', 0.1, 0.2, step=0.1))) |
|||
model.add(Dense(len(users), activation='softmax')) |
|||
model.compile( |
|||
optimizer=Adam(learning_rate=hp.Choice('learning_rate', [1e-5])), |
|||
loss='sparse_categorical_crossentropy', |
|||
metrics=[#F1Score(name='f1', average='weighted'), |
|||
Precision(), #Recall(), Accuracy() |
|||
] |
|||
) |
|||
return model |
|||
|
|||
tuner = RandomSearch( |
|||
build_model, |
|||
objective=keras_tuner.Objective(val_metric, direction="max"), |
|||
max_trials=120, |
|||
directory=tuner_dir, |
|||
) |
|||
|
|||
tuner.search(x, y, epochs=epochs, validation_data=(x_val, y_val), |
|||
callbacks=[early_stopping, lr_scheduler]) |
|||
return tuner.get_best_models(num_models=1)[0] |
|||
|
|||
|
|||
def train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons,n_neurons2,n_neurons3,n_neurons4, l_rate, d1, r1, reg1, r2, reg2, sequence_length, model_type): |
|||
x, y = prepare_data_for_model(user_data=train_data, sequence_length=sequence_length) |
|||
n_features = x.shape[2] |
|||
users = list(train_data.keys()) |
|||
|
|||
# prepare model |
|||
def build_model(): |
|||
model = Sequential() |
|||
model.add(Input(shape=(sequence_length, n_features), batch_size=n_batch)) |
|||
if model_type == model_type_bilstm: |
|||
model.add(Bidirectional(LSTM(n_neurons))) |
|||
if model_type == model_type_lstm: |
|||
# model.add(LSTM(n_neurons, kernel_regularizer=reg1, return_sequences=True)) |
|||
model.add(LSTM(n_neurons)) |
|||
# model.add(LSTM(n_neurons2)) |
|||
if model_type == model_type_gru: |
|||
model.add(GRU(n_neurons)) |
|||
#model.add(Dense(n_neurons, activation='relu')) |
|||
#model.add(Dropout(d1)) |
|||
model.add(Dense(len(users), activation='softmax')) |
|||
model.compile( |
|||
optimizer=Adam(learning_rate=l_rate), |
|||
loss=SparseCategoricalCrossentropy(), |
|||
metrics=[SparseCategoricalAccuracy()], |
|||
) |
|||
return model |
|||
|
|||
model = build_model() |
|||
|
|||
# fit model |
|||
train_acc, test_acc, train_p, test_p, train_r, test_r, train_f1, test_f1 = list(), list(),list(), list(),list(), list(),list(), list() |
|||
for i in range(n_epochs): |
|||
model.fit(x, y, batch_size=n_batch, epochs=1, verbose=0, shuffle=False) |
|||
# evaluate model on train data |
|||
acc, p, r, f1 = evaluate(model, train_data, sequence_length, n_batch) |
|||
train_acc.append(acc) |
|||
train_p.append(p) |
|||
train_r.append(r) |
|||
train_f1.append(f1) |
|||
# evaluate model on test data |
|||
savename = 'cf_matrix_'+get_save_id(n_epochs, n_neurons, n_neurons2,n_neurons3, n_neurons4, n_batch, l_rate,d1,r1, r2)+'.json' |
|||
acc, p, r, f1 = evaluate(model, val_data, sequence_length, n_batch, save_name=savename) |
|||
test_acc.append(acc) |
|||
test_p.append(p) |
|||
test_r.append(r) |
|||
test_f1.append(f1) |
|||
|
|||
history = DataFrame() |
|||
history['train_acc'], history['test_acc'] = train_acc, test_acc |
|||
history['train_p'], history['test_p'] = train_p, test_p |
|||
history['train_r'], history['test_r'] = train_r, test_r |
|||
history['train_f1'], history['test_f1'] = train_f1, test_f1 |
|||
return history |
|||
|
|||
|
|||
def train_one_model_v2(train_data, val_data, n_batch, n_epochs, n_neurons, sequence_length, model_type): |
|||
x, y = train_data |
|||
x_v, y_v = val_data |
|||
users = list(set(y)) |
|||
|
|||
# renumber users |
|||
user_map = {users[i]:i for i in range(len(users))} |
|||
y = np.array([user_map[x] for x in y]) |
|||
y_v = np.array([user_map[x] for x in y_v]) |
|||
n_features = x.shape[2] |
|||
user_num = len(users) |
|||
|
|||
# prepare model |
|||
def build_model(): |
|||
model = Sequential() |
|||
model.add(Input(shape=(sequence_length, n_features), batch_size=n_batch)) |
|||
if model_type == model_type_bilstm: |
|||
model.add(Bidirectional(LSTM(n_neurons))) |
|||
if model_type == model_type_lstm: |
|||
model.add(LSTM(n_neurons)) |
|||
if model_type == model_type_gru: |
|||
model.add(GRU(n_neurons)) |
|||
#model.add(Dense(n_neurons, activation='relu')) |
|||
#model.add(Dropout(d1)) |
|||
model.add(Dense(user_num, activation='softmax')) |
|||
model.compile( |
|||
optimizer=Adam(), |
|||
loss=SparseCategoricalCrossentropy(), |
|||
metrics=[SparseCategoricalAccuracy()], |
|||
) |
|||
return model |
|||
|
|||
model = build_model() |
|||
|
|||
# fit model |
|||
train_acc, test_acc, train_p, test_p, train_r, test_r, train_f1, test_f1 = list(), list(),list(), list(),list(), list(),list(), list() |
|||
for i in range(n_epochs): |
|||
model.fit(x, y, batch_size=n_batch, epochs=1, verbose=0, shuffle=False) |
|||
# evaluate model on train data |
|||
acc, p, r, f1 = evaluate_v2(model, (x,y), sequence_length, n_batch) |
|||
train_acc.append(acc) |
|||
train_p.append(p) |
|||
train_r.append(r) |
|||
train_f1.append(f1) |
|||
# evaluate model on test data |
|||
savename = 'cf_matrix_'+get_save_id(n_epochs, n_neurons, '', '', '', n_batch, '','','','')+'.json' |
|||
acc, p, r, f1 = evaluate_v2(model, (x_v, y_v), sequence_length, n_batch, save_name=savename) |
|||
test_acc.append(acc) |
|||
test_p.append(p) |
|||
test_r.append(r) |
|||
test_f1.append(f1) |
|||
|
|||
history = DataFrame() |
|||
history['train_acc'], history['test_acc'] = train_acc, test_acc |
|||
history['train_p'], history['test_p'] = train_p, test_p |
|||
history['train_r'], history['test_r'] = train_r, test_r |
|||
history['train_f1'], history['test_f1'] = train_f1, test_f1 |
|||
return history |
|||
|
|||
def get_save_id(n_epochs, n_neurons, n_neurons2,n_neurons3,n_neurons4, n_batch, l_rate, d1,r1, r2): |
|||
return '_e'+str(n_epochs)+'_n'+str(n_neurons)+'_b'+ str(n_batch) |
|||
#'x'+str(n_neurons3)+'x'+str(n_neurons4) |
|||
#+'_l'+str(l_rate)+'_r'+str(r1)+'xx'+str(r2) |
|||
|
|||
|
|||
def evaluate(model, df, sequence_length, batch_size, save_name=None): |
|||
x, y = prepare_data_for_model(user_data=df, sequence_length=sequence_length) |
|||
x = np.array(x) |
|||
y_true = np.array(y) |
|||
|
|||
y_pred = model.predict(x, verbose=0, batch_size=batch_size) |
|||
y_pred_classes = np.argmax(y_pred, axis=1) |
|||
cf_matrix = pd.DataFrame(confusion_matrix(y_true, y_pred_classes)) |
|||
if save_name is not None: |
|||
cf_matrix.to_json('results/'+save_name) |
|||
true_counts = pd.DataFrame(y).value_counts() |
|||
print('Top true occurrences', true_counts[:6]) |
|||
predicted_counts = pd.DataFrame(y_pred_classes).value_counts() |
|||
print('Top predicted occurrences', predicted_counts[:6]) |
|||
|
|||
return eval_metrics(y_true=y_true, y_pred=y_pred_classes) |
|||
|
|||
def evaluate_v2(model, data, sequence_length, batch_size, save_name=None): |
|||
x, y_true = data |
|||
y_pred = model.predict(x, verbose=0, batch_size=batch_size) |
|||
y_pred_classes = np.argmax(y_pred, axis=1) |
|||
cf_matrix = pd.DataFrame(confusion_matrix(y_true, y_pred_classes)) |
|||
if save_name is not None: |
|||
cf_matrix.to_json('results/'+save_name) |
|||
true_counts = pd.DataFrame(y_true).value_counts() |
|||
print('Top true occurrences', true_counts[:6]) |
|||
predicted_counts = pd.DataFrame(y_pred_classes).value_counts() |
|||
print('Top predicted occurrences', predicted_counts[:6]) |
|||
|
|||
return eval_metrics(y_true=y_true, y_pred=y_pred_classes) |
|||
|
|||
|
|||
def eval_metrics(y_true, y_pred): |
|||
f1 = f1_score(y_true=y_true, y_pred=y_pred, average='weighted') |
|||
p = precision_score(y_true=y_true, y_pred=y_pred, average='weighted') |
|||
r = recall_score(y_true=y_true, y_pred=y_pred, average='weighted') |
|||
acc = accuracy_score(y_true=y_true, y_pred=y_pred) |
|||
return acc, p, r, f1 |
|||
|
|||
# === Evaluation === |
|||
def evaluate_models(best_models, df_test, sequence_lengths, output_excel_path, ALLUSERS32_15MIN_WITHOUTTHREHOLD): |
|||
print("\n🧪 Evaluating on Test Data...") |
|||
with ExcelWriter(output_excel_path) as writer: |
|||
for sequence_length in sequence_lengths: |
|||
if sequence_length not in best_models: |
|||
continue |
|||
evaluate_model_on_test_data(best_models[sequence_length]['model'], df_test.copy(), |
|||
sequence_length, writer, ALLUSERS32_15MIN_WITHOUTTHREHOLD) |
|||
|
|||
def evaluate_model_on_test_data(model, test_df, sequence_length, excel_writer, ALLUSERS32_15MIN_WITHOUTTHREHOLD): |
|||
if(ALLUSERS32_15MIN_WITHOUTTHREHOLD): |
|||
test_df = test_df.drop(columns=['Month', 'Year', 'date', 'DayOfWeek']) |
|||
else: |
|||
test_df = test_df.drop(columns=['Month', 'Year', 'date']) |
|||
|
|||
test_df = test_df.sort_values(by='user').reset_index(drop=True) |
|||
|
|||
users = test_df['user'].unique() |
|||
results = [] |
|||
accuracy_above_50 = 0 |
|||
|
|||
for user in users: |
|||
user_df = test_df[test_df['user'] == user] |
|||
X, y_true = [], [] |
|||
user_features = user_df.drop(columns=['user']).values |
|||
user_labels = user_df['user'].values |
|||
|
|||
if len(user_df) <= sequence_length: |
|||
print(f"Skipping User {user} (not enough data for sequence length {sequence_length})") |
|||
continue |
|||
|
|||
for i in range(len(user_df) - sequence_length): |
|||
seq_x = user_features[i:i + sequence_length] |
|||
seq_y = user_labels[i + sequence_length] |
|||
X.append(seq_x) |
|||
y_true.append(seq_y) |
|||
|
|||
X = np.array(X) |
|||
y_true = np.array(y_true) |
|||
|
|||
if len(X) == 0: |
|||
continue |
|||
|
|||
y_pred = model.predict(X, verbose=0) |
|||
y_pred_classes = np.argmax(y_pred, axis=1) |
|||
|
|||
# counts which class was predicted how often |
|||
unique_pred, counts_pred = np.unique(y_pred_classes, return_counts=True) |
|||
label_counts_pred = dict(zip(unique_pred, counts_pred)) |
|||
|
|||
# counts which class should have been predicted how often (only one class for the user) |
|||
unique_true, counts_true = np.unique(y_true, return_counts=True) |
|||
label_counts_true = dict(zip(unique_true, counts_true)) |
|||
|
|||
# the fraction of correctly classified samples |
|||
acc = accuracy_score(y_true, y_pred_classes) |
|||
if acc > 0.5: |
|||
accuracy_above_50 += 1 |
|||
|
|||
results.append({ |
|||
'User': user, |
|||
'Accuracy (%)': acc * 100, |
|||
'Predicted Class Distribution': str(label_counts_pred), |
|||
'Actual Class Distribution': str(label_counts_true) |
|||
}) |
|||
|
|||
print(f"\n=== User {user} ===") |
|||
print(f"✅ Accuracy: {acc * 100:.2f}%") |
|||
print("📊 Predicted Class Distribution:", label_counts_pred) |
|||
print("📌 Actual Class Distribution: ", label_counts_true) |
|||
|
|||
final_accuracy_percent = (accuracy_above_50 / 32) * 100 |
|||
print(f"\n🟩 Final Evaluation Summary for Sequence Length {sequence_length}:") |
|||
print(f"Users with >50% Accuracy: {accuracy_above_50} / 32") |
|||
print(f"✅ Final Success Rate: {final_accuracy_percent:.2f}%") |
|||
|
|||
results.append({ |
|||
'User': 'TOTAL', |
|||
'Accuracy (%)': '', |
|||
'Predicted Class Distribution': f'Users >50% Acc: {accuracy_above_50}/32', |
|||
'Actual Class Distribution': f'Success Rate: {final_accuracy_percent:.2f}%' |
|||
}) |
|||
|
|||
df_results = pd.DataFrame(results) |
|||
df_results.to_excel(excel_writer, sheet_name=f"SeqLen_{sequence_length}", index=False) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue