|
|
import os
import pandas as pd
from main import month_str, year_str, time_str, date_str, day_of_week_str, user_str, dataset_min_path, dataset_hrs_path, \ week_column_names
def process_file_one_hour(file_path, user_label): # Load the dataset df = pd.read_csv(file_path, delimiter=';', low_memory=False)
# Filter for iPhone devices iphone_df = df[df['device'].str.contains('iPhone', na=False)] # Treat NaN as False
# Convert startDate to datetime iphone_df['startDate'] = pd.to_datetime(iphone_df['startDate'], format='%Y-%m-%d %H:%M:%S %z')
# Extract date and hour hour_str = 'hour' iphone_df[hour_str] = iphone_df['startDate'].dt.hour iphone_df[date_str] = iphone_df['startDate'].dt.date iphone_df[year_str] = iphone_df['startDate'].dt.year iphone_df[month_str] = iphone_df['startDate'].dt.month
# Group by date and hour, then sum the values hourly_sum = iphone_df.groupby([date_str, hour_str, year_str, month_str])['value'].sum().reset_index()
# Pivot the data to get one row per day with 24 columns for each hour pivot_table = hourly_sum.pivot(index=[date_str, year_str, month_str], columns=hour_str, values='value').fillna(0)
pivot_table = pivot_table.astype(int) # float because of the filled nas
# Rename columns to reflect hours pivot_table.columns = [f'Hour_{i}' for i in pivot_table.columns] all_hours = ['Hour_'+ str(i) for i in range(24)] for hours in all_hours: if hours not in pivot_table.columns: pivot_table[hours] = 0
# Reset index pivot_table.reset_index(inplace=True)
# Add day of the week, month, and year columns pivot_table[day_of_week_str] = pd.to_datetime(pivot_table[date_str]).dt.day_name()
# One-hot encode the 'DayOfWeek' column pivot_table = pd.concat([pivot_table, pd.get_dummies(pivot_table[day_of_week_str], prefix=day_of_week_str, dtype=int)], axis=1) for week_day_col in week_column_names: if week_day_col not in pivot_table.columns: pivot_table[week_day_col] = 0
# Add 'user' column with the specified user label pivot_table[user_str] = user_label
# Step 13: Drop the 'DayOfWeek' column pivot_table.drop(columns=[day_of_week_str], inplace=True)
return pivot_table
def process_file_15_min(file_path, user_label): interval_str = '15min_interval'
# Load the dataset df = pd.read_csv(file_path, delimiter=';', low_memory=False)
# TODO: evtl. nicht nur iPhone date nutzen # Filter for iPhone devices iphone_df = df[df['device'].str.contains('iPhone', na=False)]
# Convert startDate to datetime iphone_df['startDate'] = pd.to_datetime(iphone_df['startDate'], format='%Y-%m-%d %H:%M:%S %z')
# Round down the startDate to the nearest 15-minute interval iphone_df[interval_str] = iphone_df['startDate'].dt.floor('15min')
# Extract date, time, year, and month for 15-minute intervals iphone_df[date_str] = iphone_df[interval_str].dt.date iphone_df[time_str] = iphone_df[interval_str].dt.time iphone_df[year_str] = iphone_df[interval_str].dt.year iphone_df[month_str] = iphone_df[interval_str].dt.month
# Group by date, time, year, and month, then sum the values interval_sum = iphone_df.groupby([date_str, time_str, year_str, month_str])['value'].sum().reset_index()
# Create a full range of 15-minute intervals (00:00:00 to 23:45:00) full_time_range = pd.date_range('00:00', '23:45', freq='15min').time
# Pivot the data to get one row per day with columns for each 15-minute interval pivot_table = interval_sum.pivot(index=[date_str, year_str, month_str], columns=time_str, values='value').fillna(0) pivot_table = pivot_table.astype(int) # float because of the filled nas
# Reindex to include all possible 15-minute intervals pivot_table = pivot_table.reindex(columns=full_time_range, fill_value=0)
# Rename columns to reflect 15-minute intervals pivot_table.columns = [f'{str(col)}' for col in pivot_table.columns]
# Reset index to have 'date', 'Year', and 'Month' as columns instead of index pivot_table.reset_index(inplace=True)
# Add day of the week pivot_table[day_of_week_str] = pd.to_datetime(pivot_table[date_str]).dt.day_name()
# One-hot encode the 'DayOfWeek' column pivot_table = pd.concat( [pivot_table, pd.get_dummies(pivot_table[day_of_week_str], prefix=day_of_week_str, dtype=int)], axis=1) for week_day_col in week_column_names: if week_day_col not in pivot_table.columns: pivot_table[week_day_col] = 0
# Add a user column with the specified user label pivot_table[user_str] = user_label
pivot_table.drop(columns=[day_of_week_str], inplace=True)
return pivot_table
if __name__ == "__main__": pd.options.mode.copy_on_write = True # Generate file paths files = (['Europe/Europe/'+file for file in os.listdir('Europe/Europe/')] + ['Rest_of_the_World/'+file for file in os.listdir('Rest_of_the_World')])
# Generate user labels based on file index user_labels = list(range(len(files)))
for save_name, process_func in [(dataset_hrs_path, process_file_one_hour), (dataset_min_path, process_file_15_min)]: # Process each file with its corresponding user label and concatenate the results processed_dfs = [process_func(file_path, user_label) for file_path, user_label in zip(files, user_labels)]
combined_df = pd.concat(processed_dfs, ignore_index=True)
# Save the combined DataFrame to a new json file combined_df.to_json(save_name, index=False)
print('Done')
|
