import os import numpy as np import pandas as pd from matplotlib import pyplot as plt from pandas import DataFrame from sklearn.dummy import DummyClassifier from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier from sklearn.model_selection import GridSearchCV from sklearn.naive_bayes import BernoulliNB from sklearn.preprocessing import MinMaxScaler from pipeline import ( prepare_data_for_neural_model, model_type_gru, model_type_lstm, model_type_bilstm, eval_metrics, prepare_data_for_basic_algorithm, train_one_model, ) year_str = 'Year' month_str = 'Month' day_str = 'Day' date_str = 'Date' time_str = 'Time' day_of_week_str = 'DayOfWeek' user_str = 'user' split_str = 'split type' data_split_str = 'data percentages' month_split_str = 'month percentages' timespan_str = 'time used' hour_timespan_str = '1HR' min_timespan_str = '15MIN' sequence_length_str = 'sequence length' accuracy_str = 'accuracy' precision_str = 'precision' recall_str = 'recall' f1_string = 'f1 score' model_type_str = 'model type' week_column_names = ['DayOfWeek_' + day for day in ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday' ]] figure_path = 'figures/' # === Configurable Parameters === dataset_path = './Datasets/' dataset_hrs_path = './Datasets/hours.json' dataset_min_path = './Datasets/minutes.json' def create_dir(path): """ Creates a directory if it doesn't exist yet. :param path: The path to the directory """ if not os.path.exists(path): os.makedirs(path) def remove_covid_data(df): """ Removes covid data from dataframe because the steps from these times will most likely differ from before :param df: Dataframe with the data :return: the data without covid time data """ df = df[~(df[year_str]>=2020)] return df def split_data_by_userdata_percentage(df, percentages, sample=100): """ Splits data by userdata percentages. Each users data will be split according to the given percentages along the time axis :param df: Data with all users data :param percentages: triple with percentages :param sample: overall percentage if less data should be used. Use only for testing. Has an error!! :return: the split data """ train_p, valid_p, test_p = percentages tr, va, te = pd.DataFrame(), pd.DataFrame(), pd.DataFrame() for user_id in df[user_str].unique(): # !! following sample creates gaps in data if sample smaller 100 user_data = df[df[user_str]==user_id].sample(frac=sample/ 100).sort_values([date_str]) # have to sort for time shift u_tr, u_va, u_te = np.split(user_data, [int((train_p/100)*len(user_data)), int(((train_p+valid_p)/100)*len(user_data))]) tr = pd.concat([tr, u_tr], ignore_index=True) va = pd.concat([va, u_va], ignore_index=True) te = pd.concat([te, u_te], ignore_index=True) return tr, va, te def reduce_columns(df): """ Removes unnecessary columns from dataframe. :param df: Dataframe with the data :return: dataframe without unnecessary columns """ return df.drop(columns=[month_str, year_str, date_str]) def filter_and_preprocess_data(filename, sample=100, print_unique=False): """ Preprocesses data. Removes users with too little data or which subsume another, bins the step data and creates train, valid, test splits :param filename: Name of the file for loading :param sample: percentage of the sample in case less data is wanted (e.g., for testing) :param print_unique: To print the number of unique users :return: train, valid, test splits as dicts from user_id to dataframe """ df = pd.read_json(filename) df = remove_covid_data(df) # remove users which are a complete subset of another user (but keep one) users_to_remove = [] for user_a in df[user_str].unique(): for user_b in df[user_str].unique(): if user_a != user_b: data = pd.concat([df[df[user_str]==user_a], df[df[user_str]==user_b]]) columns = data.columns.tolist() columns.remove(user_str) no_dup = data.drop_duplicates(columns, keep=False) if len(no_dup[no_dup[user_str]==user_a]) == 0: if print_unique: print(user_a, 'is subset of',user_b) if user_b not in users_to_remove: users_to_remove.append(user_a) df = df[~df[user_str].isin(users_to_remove)] # bin steps per hour TODO: adjust for minutes for hour in ['Hour_'+str(i) for i in range(24)]: hour_data = df[hour] # smaller 1000 - round to 10 a = ((hour_data[hour_data<1000]/10).round()*10) # between 1000 and 10000 - round to next 100 b = ((hour_data[(hour_data>=1000)& (hour_data<10000)]/100).round()*100) # higher or equal 10000 - one class c = hour_data[hour_data > 10000] c = pd.Series(data={ind:10000 for ind in c.index}, index=c.index) new = pd.concat([a, b, c]).sort_index().astype(int) df[hour] = new # remove users with too little data min_datapoints = 500 # 500 leads to at least 75 datapoints in the valid set users_to_remove = set() cols = df.columns.tolist() cols.remove(user_str) reduced = df.drop_duplicates(subset=cols, keep=False) for user_id in df[user_str].unique(): subset = df[df[user_str] == user_id] reduced_subset = reduced[reduced[user_str] == user_id] if print_unique: print(user_id, len(subset), len(reduced_subset)) if len(reduced_subset) < min_datapoints: users_to_remove.add(user_id) if print_unique: print('removing', user_id) df = df[~df[user_str].isin(users_to_remove)] tr, val, te = split_data_by_userdata_percentage(df, percentages=(70, 15, 15), sample=sample) tr = reduce_columns(tr) val = reduce_columns(val) te = reduce_columns(te) if print_unique: print('Train: Users', len(tr[user_str].unique()), 'mean num datapoins:', tr[user_str].value_counts().mean()) print('Valid: Users', len(val[user_str].unique()), 'mean num datapoins:', val[user_str].value_counts().mean()) print('Test: Users', len(te[user_str].unique()), 'mean num datapoins:', te[user_str].value_counts().mean()) tr, val, te = add_features(tr), add_features(val), add_features(te) scaler = MinMaxScaler() scaler.fit(tr.drop(columns=[user_str])) return scale_dataset(scaler, tr), scale_dataset(scaler, val), scale_dataset(scaler, te) def scale_dataset(scaler, df): """ Data scaling function :param scaler: The scaler object :param df: data to scale :return: the scaled data """ y = df[user_str] x_scaled = scaler.transform(df.drop(columns=[user_str])) x_scaled = pd.DataFrame(x_scaled) x_scaled.columns = df.drop(columns=[user_str]).columns df_scaled = pd.concat([x_scaled, pd.DataFrame(y.reset_index()[user_str])], axis=1) return convert_to_user_dict(df_scaled) def convert_to_user_dict(df): """ Converts the dataframe to a dict of dataframes with the key the user id :param df: Complete dataframe :return: the dict of dataframes """ users = df[user_str].unique() return {user: df[df[user_str] == user] for user in users} def calculate_baselines(): """ Calculates very simple baselines for the scenario and saves them :return the calculated baselines """ file_combinations = [(hour_timespan_str, dataset_hrs_path), # (min_timespan_str, dataset_min_path), # TODO: dataset binning not ready for minutes yet, rerun this method when that works ] str_result_filename = 'baseline_results.json' if os.path.exists(str_result_filename): baseline_res = pd.read_json(str_result_filename) else: baseline_res = pd.DataFrame() for timespan_id, filename in file_combinations: _, _, te = filter_and_preprocess_data(filename) for sequence_length in range(1,30,5): x, y = prepare_data_for_neural_model(user_data=te, sequence_length=sequence_length) for strategy in ['most_frequent', 'stratified', 'uniform']: cls = DummyClassifier(strategy=strategy) cls.fit(x,y) y_pred = cls.predict(x) acc, p, r, f1 = eval_metrics(y_true=y, y_pred=y_pred) baseline_res = pd.concat([baseline_res, DataFrame({ 'strategy':[strategy], timespan_str:[timespan_id], sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r], f1_string:f1})], ignore_index=True) baseline_res.to_json(str_result_filename) return baseline_res def hypertune_basic_algorithms(): """ Function can be used to hypertune basic sklearn algorithms. Takes very long. """ # TODO: mnake it run for minutes, iterate over sequence lengths sequence_length = 7 tr, val, te = filter_and_preprocess_data(dataset_hrs_path) x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) random_state = 17 results = pd.DataFrame() for tag, clf, grid in [ ('GradientBoosting', GradientBoostingClassifier(random_state=random_state), {'loss': ['log_loss', 'exponential'], 'learning_rate': [0.1, 0.5, 1.0,2.0, 5.0], 'n_estimators': [10, 50, 100, 150, 200], 'subsample': [0.1, 0.5, 1.0], 'criterion': ['friedman_mse', 'squared_error'], 'min_samples_split': [2, 10, 100], 'min_samples_leaf': [1, 5, 10], 'min_weight_fraction_leaf': [0.0, 0.1, 0.5], 'max_depth': [None, 2, 10, 100], 'min_impurity_decrease': [0.0, 0.1, 0.5], 'max_features': ['sqrt', 'log2', None, 10, 20], 'max_leaf_nodes': [None, 1, 5, 10], }), ('Bernoulli', BernoulliNB(), {'fit_prior': [True, False], 'binarize': [0.0, 0.1, 0.25, 0.5, 0.75], 'force_alpha': [True, False], 'alpha':[0.0, 0.25, 0.5, 0.75, 1.0]}), ('extra trees', ExtraTreesClassifier(random_state=random_state, n_jobs=1), {'n_estimators': [10, 50, 100, 150, 200], 'criterion': ['gini', 'entropy', 'log_loss'], 'max_depth': [None, 2, 10, 100], 'min_samples_split': [2, 10, 100], 'min_samples_leaf': [1, 5, 10], 'min_weight_fraction_leaf': [0.0, 0.1, 0.5], 'max_features': ['sqrt', 'log2', None, 10, 20], 'max_leaf_nodes': [None, 1, 5, 10], 'min_impurity_decrease': [0.0, 0.1, 0.5], 'bootstrap': [True, False], 'class_weight': [None, 'balanced', 'balanced_subsample'], 'max_samples': [None, 0.1, 0.2, 0.3]} ), ('random forest', RandomForestClassifier(random_state=random_state, n_jobs=1), {'n_estimators':[10, 50, 100, 150, 200], 'criterion':['gini', 'entropy', 'log_loss'], 'max_depth':[None, 2, 10,100], 'min_samples_split': [2,10,100], 'min_samples_leaf':[1,5,10], 'min_weight_fraction_leaf':[0.0,0.1, 0.5], 'max_features':['sqrt', 'log2', None, 10, 20], 'max_leaf_nodes':[None, 1, 5, 10], 'min_impurity_decrease':[0.0, 0.1, 0.5], 'bootstrap':[True, False], 'class_weight':[None, 'balanced', 'balanced_subsample'], 'max_samples':[None,0.1, 0.2, 0.3]}) ]: grid_search = GridSearchCV( estimator=clf, param_grid=grid, scoring='f1_weighted', cv=5, n_jobs=1) grid_search.fit(x_tr, y_tr) best_model = grid_search.best_estimator_ y_pred = best_model.predict(x_val) acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) results = pd.concat([results, DataFrame({ 'params': str(grid_search.best_params_), 'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) results.to_json('basic_ht_results.json') print('Done') def test_basic_algorithms(): """ Method for testing basic algorithms from the sklearn library. Tested more, but those 4 were the best :return: The calculated values """ # TODO: also check for minutes # TODO: iterate over sequence lengths sequence_length = 21 tr, val, te = filter_and_preprocess_data(dataset_hrs_path) x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) random_state = 17 results = pd.DataFrame() for tag, clf in [ ('Bernoulli', BernoulliNB()), ('GradientBoosting', GradientBoostingClassifier(random_state=random_state)), ('extra trees', ExtraTreesClassifier(random_state=random_state)), ('random forest', RandomForestClassifier(random_state=random_state)) ]: clf.fit(x_tr, y_tr) y_pred = clf.predict(x_val) acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=y_pred) results = pd.concat([results, DataFrame({ 'tag':tag,accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) return results def add_features(df): """ Functiion for adding additional features to the dataframe :param df: dataframe :return: dataframe with features added """ # indicator weekend df['weekend'] = df[day_of_week_str + '_Saturday']+df[day_of_week_str + '_Sunday'] # sum of steps per day df['day_total'] = sum([df['Hour_'+str(i)] for i in range(23)]) # sum of steps morning, afternoon, evening, night df['morning_total'] = sum([df['Hour_' + str(i)] for i in range(6,13)]) df['afternoon_total'] = sum([df['Hour_' + str(i)] for i in range(13,19)]) df['evening_total'] = sum([df['Hour_' + str(i)] for i in range(19,23)]) df['night_total'] = sum([df['Hour_' + str(i)] for i in [23,0,1,2,3,4,5]]) return df def test_basic_algorithm_on_sequence_lengths(clf = RandomForestClassifier(random_state=17)): """ Runs a basic algorith from scikit learn on different sequence lengths. Plots an image for the results. :param clf: the sklearn classifier """ tr, val, te = filter_and_preprocess_data(dataset_hrs_path) results_train = pd.DataFrame() results_valid = pd.DataFrame() # iterate over sequence lengths for sequence_length in range(1, 60, 5): x_tr, y_tr = prepare_data_for_basic_algorithm(user_data=tr, sequence_length=sequence_length) x_val, y_val = prepare_data_for_basic_algorithm(user_data=val, sequence_length=sequence_length) clf.fit(x_tr, y_tr) acc, p, r, f1 = eval_metrics(y_true=y_val, y_pred=clf.predict(x_val)) results_valid = pd.concat([results_valid, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) acc, p, r, f1 = eval_metrics(y_true=y_tr, y_pred=clf.predict(x_tr)) results_train = pd.concat([results_train, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[acc],precision_str:[p],recall_str:[r],f1_string:f1})], ignore_index=True) fig = plt.figure() for frame in [results_train, results_valid]: plt.plot(frame[sequence_length_str], frame[f1_string]) plt.show() print('') def tune_neural_network(model_type): """ Tunes a neural model for different sequence lengths. Plots the results :param model_type: either lstm, gru or bilstm, use set strings """ # TODO: Also do this for minute data tr, val, te = filter_and_preprocess_data(dataset_hrs_path) n_epochs= 20 n_neurons = 1024 n_batch = 1024 results_train = pd.DataFrame() results_valid = pd.DataFrame() # iterate over sequence lengths for sequence_length in range(1, 50, 5): train_data = prepare_data_for_neural_model(user_data=tr, sequence_length=sequence_length) val_data = prepare_data_for_neural_model(user_data=val, sequence_length=sequence_length) # fit and evaluate model history_list = list() repeats = 3 # run diagnostic tests for i in range(repeats): history = train_one_model(train_data, val_data, n_batch, n_epochs, n_neurons, sequence_length=sequence_length, model_type=model_type) history_list.append(history) results = pd.concat([history.tail(1) for history in history_list]).mean() results_train = pd.concat([results_train, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[results['train_acc']], precision_str:[results['train_p']], recall_str:[results['train_r']], f1_string:[results['train_f1']]})], ignore_index=True) results_valid = pd.concat([results_valid, DataFrame({sequence_length_str:[sequence_length], accuracy_str:[results['test_acc']], precision_str:[results['test_p']], recall_str:[results['test_r']], f1_string:[results['test_f1']]})], ignore_index=True) fig = plt.figure() for frame in [results_train, results_valid]: plt.plot(frame[sequence_length_str], frame[f1_string]) plt.show() print('Done') if __name__ == "__main__": # create needed directories create_dir('results/') create_dir(figure_path) pd.options.mode.copy_on_write = True baselines = calculate_baselines() # use basic algorithms from scikit learn test_basic_algorithms() hypertune_basic_algorithms() test_basic_algorithm_on_sequence_lengths() # tune a neural network tune_neural_network(model_type=model_type_lstm) print('Done')