1.6 MiB
1.6 MiB
In [2]:
%pip install seabornDefaulting to user installation because normal site-packages is not writeable Requirement already satisfied: seaborn in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (0.13.2) Requirement already satisfied: numpy!=1.24.0,>=1.20 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from seaborn) (2.4.4) Requirement already satisfied: pandas>=1.2 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from seaborn) (3.0.2) Requirement already satisfied: matplotlib!=3.6.1,>=3.4 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from seaborn) (3.10.8) Requirement already satisfied: contourpy>=1.0.1 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.3.3) Requirement already satisfied: cycler>=0.10 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (0.12.1) Requirement already satisfied: fonttools>=4.22.0 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (4.62.1) Requirement already satisfied: kiwisolver>=1.3.1 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (1.5.0) Requirement already satisfied: packaging>=20.0 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (26.1) Requirement already satisfied: pillow>=8 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (12.2.0) Requirement already satisfied: pyparsing>=3 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (3.3.2) Requirement already satisfied: python-dateutil>=2.7 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from matplotlib!=3.6.1,>=3.4->seaborn) (2.9.0.post0) Requirement already satisfied: tzdata in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from pandas>=1.2->seaborn) (2026.2) Requirement already satisfied: six>=1.5 in C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.4->seaborn) (1.17.0) Note: you may need to restart the kernel to use updated packages.
[notice] A new release of pip is available: 26.1.2 -> 26.2.1 [notice] To update, run: python.exe -m pip install --upgrade pip
In [3]:
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_theme(style="whitegrid")
# The already-cleaned dataset your main.py uses
df = pd.read_json('Datasets/hours.json')
print(df.shape)
df.head()Out [3]:
(49798, 35)
| Date | Year | Month | Hour_0 | Hour_1 | Hour_2 | Hour_3 | Hour_4 | Hour_5 | Hour_6 | ... | Hour_22 | Hour_23 | DayOfWeek_Friday | DayOfWeek_Monday | DayOfWeek_Saturday | DayOfWeek_Sunday | DayOfWeek_Thursday | DayOfWeek_Tuesday | DayOfWeek_Wednesday | user | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2015-09-16 | 2015 | 9 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 38 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 |
| 1 | 2015-09-17 | 2015 | 9 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 103 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 |
| 2 | 2015-09-18 | 2015 | 9 | 0 | 0 | 0 | 0 | 0 | 0 | 180 | ... | 782 | 48 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 3 | 2015-09-19 | 2015 | 9 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 359 | 562 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 |
| 4 | 2015-09-20 | 2015 | 9 | 165 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 |
5 rows × 35 columns
In [4]:
# How many missing values per column?
missing_counts = df.isnull().sum().sort_values(ascending=False)
print(missing_counts[missing_counts > 0])
# Visualize where the gaps are
plt.figure(figsize=(14, 6))
sns.heatmap(df.isnull(), cbar=False, cmap='viridis')
plt.title('Missing values map (yellow = missing)')
plt.show()Series([], dtype: int64)
In [5]:
user_counts = df['user'].value_counts().sort_index()
plt.figure(figsize=(14, 5))
user_counts.plot(kind='bar')
plt.title('Number of day-records per user')
plt.xlabel('User ID')
plt.ylabel('Number of days of data')
plt.show()
print('Fewest days:', user_counts.min(), '| Most days:', user_counts.max())
print('Average days per user:', user_counts.mean().round(1))
Fewest days: 3 | Most days: 2102 Average days per user: 976.4
In [6]:
hour_cols = [c for c in df.columns if c.startswith('Hour_')]
plt.figure(figsize=(16, 6))
sns.boxplot(data=df[hour_cols])
plt.title('Step count distribution by hour of day (outliers = dots)')
plt.xticks(rotation=45)
plt.show()
In [7]:
# Check if group info exists in the processed data
print(df.columns.tolist())['Date', 'Year', 'Month', 'Hour_0', 'Hour_1', 'Hour_2', 'Hour_3', 'Hour_4', 'Hour_5', 'Hour_6', 'Hour_7', 'Hour_8', 'Hour_9', 'Hour_10', 'Hour_11', 'Hour_12', 'Hour_13', 'Hour_14', 'Hour_15', 'Hour_16', 'Hour_17', 'Hour_18', 'Hour_19', 'Hour_20', 'Hour_21', 'Hour_22', 'Hour_23', 'DayOfWeek_Friday', 'DayOfWeek_Monday', 'DayOfWeek_Saturday', 'DayOfWeek_Sunday', 'DayOfWeek_Thursday', 'DayOfWeek_Tuesday', 'DayOfWeek_Wednesday', 'user']
In [8]:
plt.figure(figsize=(14, 12))
sns.heatmap(df[hour_cols].corr(), cmap='coolwarm', center=0)
plt.title('Correlation between hours of the day')
plt.show()
In [ ]:
In [9]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
df = pd.read_json('Datasets/hours.json')
hour_cols = [c for c in df.columns if c.startswith('Hour_')]
# Missingness correlation matrix — do certain hours go missing TOGETHER?
missing_matrix = df[hour_cols].isnull().astype(int)
plt.figure(figsize=(12, 10))
sns.heatmap(missing_matrix.corr(), cmap='coolwarm', center=0, annot=False)
plt.title('Do missing hours cluster together? (missingness correlation)')
plt.show()
# Is missingness related to WHICH user it is? (MAR test)
missing_by_user = missing_matrix.groupby(df['user']).mean().mean(axis=1)
print("Missingness rate varies by user — std dev:", missing_by_user.std().round(4))
missing_by_user.sort_values(ascending=False).head(10)Out [9]:
C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages\seaborn\matrix.py:202: RuntimeWarning: All-NaN slice encountered vmin = np.nanmin(calc_data) C:\Users\PMLS\AppData\Roaming\Python\Python314\site-packages\seaborn\matrix.py:207: RuntimeWarning: All-NaN slice encountered vmax = np.nanmax(calc_data)
Missingness rate varies by user — std dev: 0.0
user 0 0.0 1 0.0 2 0.0 3 0.0 4 0.0 5 0.0 6 0.0 7 0.0 8 0.0 9 0.0 dtype: float64
In [10]:
quality = pd.DataFrame({
'n_days': df.groupby('user').size(),
'missing_pct': missing_matrix.groupby(df['user']).mean().mean(axis=1) * 100,
'zero_days_pct': (df.groupby('user')[hour_cols].sum(axis=1).groupby(df['user']).apply(lambda x: (x == 0).mean()) * 100) if False else None
})
# zero_days_pct done separately to avoid nested groupby issue:
daily_total = df[hour_cols].sum(axis=1)
zero_flag = (daily_total == 0)
quality['zero_days_pct'] = zero_flag.groupby(df['user']).mean() * 100
quality['quality_score'] = 100 - quality['missing_pct'] - quality['zero_days_pct']
quality.sort_values('quality_score').head(10)Out [10]:
| n_days | missing_pct | zero_days_pct | quality_score | |
|---|---|---|---|---|
| user | ||||
| 0 | 2070 | 0.0 | 0.0 | 100.0 |
| 1 | 2102 | 0.0 | 0.0 | 100.0 |
| 2 | 49 | 0.0 | 0.0 | 100.0 |
| 3 | 1005 | 0.0 | 0.0 | 100.0 |
| 4 | 510 | 0.0 | 0.0 | 100.0 |
| 5 | 1025 | 0.0 | 0.0 | 100.0 |
| 6 | 1713 | 0.0 | 0.0 | 100.0 |
| 7 | 874 | 0.0 | 0.0 | 100.0 |
| 8 | 954 | 0.0 | 0.0 | 100.0 |
| 9 | 3 | 0.0 | 0.0 | 100.0 |
In [11]:
user_counts = df['user'].value_counts()
# Shannon entropy — measures how "balanced" the class distribution is
# Max entropy = perfectly balanced. Big gap = severe imbalance.
probs = user_counts / user_counts.sum()
entropy = -np.sum(probs * np.log2(probs))
max_entropy = np.log2(len(user_counts))
print(f"Entropy: {entropy:.3f} bits (max possible: {max_entropy:.3f} bits)")
print(f"Balance ratio: {entropy/max_entropy:.2%} (100% = perfectly balanced)")
# Imbalance ratio: biggest class vs smallest class
print(f"Imbalance ratio (max/min): {user_counts.max()/user_counts.min():.1f}x")
# Gini coefficient of the class distribution (borrowed from economics — inequality measure)
sorted_counts = np.sort(user_counts.values)
n = len(sorted_counts)
cum = np.cumsum(sorted_counts)
gini = (2 * np.sum((np.arange(1, n+1)) * sorted_counts) - (n+1) * cum[-1]) / (n * cum[-1])
print(f"Gini coefficient of class sizes: {gini:.3f} (0=equal, 1=max inequality)")Entropy: 5.433 bits (max possible: 5.672 bits) Balance ratio: 95.79% (100% = perfectly balanced) Imbalance ratio (max/min): 700.7x Gini coefficient of class sizes: 0.310 (0=equal, 1=max inequality)
In [19]:
from scipy.stats import f_oneway, kruskal
# For each hour, test: does step count differ significantly ACROSS users?
# (This tells you which features are actually discriminative — worth feeding the model)
anova_results = []
for h in hour_cols:
groups = [df[df['user']==u][h].dropna() for u in df['user'].unique()]
groups = [g for g in groups if len(g) > 1]
f_stat, p_val = f_oneway(*groups)
anova_results.append({'hour': h, 'f_stat': f_stat, 'p_value': p_val})
anova_df = pd.DataFrame(anova_results).sort_values('f_stat', ascending=False)
print(anova_df.head(10)) # Top 10 most discriminative hours
# Show the most discriminative hours
top_hours = anova_df.head(10).sort_values('f_stat', ascending=True)
plt.figure(figsize=(10, 6))
plt.barh(
top_hours['hour'],
top_hours['f_stat']
)
plt.xlabel('ANOVA F-statistic')
plt.ylabel('Hour')
plt.title('Most discriminative hours between users')
plt.show()
print("Top 10 discriminative hours:")
print(anova_df.head(10)[['hour', 'f_stat', 'p_value']])hour f_stat p_value 5 Hour_5 72.617907 0.0 6 Hour_6 72.241065 0.0 8 Hour_8 61.546298 0.0 20 Hour_20 55.639084 0.0 1 Hour_1 47.432846 0.0 21 Hour_21 44.906896 0.0 19 Hour_19 42.048390 0.0 12 Hour_12 40.109184 0.0 4 Hour_4 37.500643 0.0 10 Hour_10 37.159183 0.0
Top 10 discriminative hours:
hour f_stat p_value
5 Hour_5 72.617907 0.0
6 Hour_6 72.241065 0.0
8 Hour_8 61.546298 0.0
20 Hour_20 55.639084 0.0
1 Hour_1 47.432846 0.0
21 Hour_21 44.906896 0.0
19 Hour_19 42.048390 0.0
12 Hour_12 40.109184 0.0
4 Hour_4 37.500643 0.0
10 Hour_10 37.159183 0.0
In [13]:
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.preprocessing import StandardScaler
# Use a subset of users for a readable plot
top_users = df['user'].value_counts().head(10).index
subset = df[df['user'].isin(top_users)].dropna(subset=hour_cols)
X = StandardScaler().fit_transform(subset[hour_cols])
y = subset['user']
# PCA — linear separability
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X)
print(f"Variance explained by 2 PCs: {pca.explained_variance_ratio_.sum():.1%}")
plt.figure(figsize=(10, 8))
sns.scatterplot(x=X_pca[:,0], y=X_pca[:,1], hue=y.astype(str), palette='tab10', alpha=0.6)
plt.title('PCA: Are users linearly separable in step-pattern space?')
plt.show()
# t-SNE — non-linear separability (closer to what a neural net "sees")
tsne = TSNE(n_components=2, perplexity=30, random_state=17)
X_tsne = tsne.fit_transform(X)
plt.figure(figsize=(10, 8))
sns.scatterplot(x=X_tsne[:,0], y=X_tsne[:,1], hue=y.astype(str), palette='tab10', alpha=0.6)
plt.title('t-SNE: Non-linear separability of users')
plt.show()Variance explained by 2 PCs: 18.0%
In [14]:
from pandas.plotting import autocorrelation_plot
# Pick one well-populated user, look at their daily total step pattern over time
sample_user = quality.sort_values('n_days', ascending=False).index[0]
user_df = df[df['user'] == sample_user].copy()
daily_total = user_df[hour_cols].sum(axis=1)
plt.figure(figsize=(12, 5))
autocorrelation_plot(daily_total)
plt.title(f'Autocorrelation of daily steps — User {sample_user}')
plt.show()
# Day-of-week effect — is there a weekly rhythm?
if 'weekend' in df.columns:
plt.figure(figsize=(8, 5))
sns.boxplot(x=df['weekend'], y=df[hour_cols].sum(axis=1))
plt.title('Total daily steps: weekday vs weekend')
plt.show()
In [15]:
from sklearn.ensemble import IsolationForest
iso = IsolationForest(contamination=0.02, random_state=17)
X_clean = df[hour_cols].dropna()
outlier_flags = iso.fit_predict(X_clean)
n_outliers = (outlier_flags == -1).sum()
print(f"Isolation Forest flagged {n_outliers} outlier day-records ({n_outliers/len(X_clean):.2%})")
# Which users do these outliers cluster in?
outlier_df = df.loc[X_clean.index].copy()
outlier_df['is_outlier'] = outlier_flags == -1
outlier_df.groupby('user')['is_outlier'].mean().sort_values(ascending=False).head(10)Out [15]:
Isolation Forest flagged 996 outlier day-records (2.00%)
user 42 0.088847 39 0.077572 34 0.070248 41 0.058824 48 0.050360 35 0.045455 37 0.045020 23 0.040493 15 0.038270 38 0.037796 Name: is_outlier, dtype: float64
In [16]:
# How does sequence length affect the number of users
# who have enough raw data?
sequence_lengths = [1, 3, 7, 14, 21, 30, 50]
results = []
for seq_len in sequence_lengths:
usable_users = quality[quality['n_days'] >= seq_len]
results.append({
'sequence_length': seq_len,
'usable_users': len(usable_users),
'percentage_of_users': len(usable_users) / len(quality) * 100
})
seq_df = pd.DataFrame(results)
print(seq_df)
plt.figure(figsize=(10, 5))
plt.plot(
seq_df['sequence_length'],
seq_df['percentage_of_users'],
marker='o'
)
plt.xlabel('Sequence length (days)')
plt.ylabel('% of users with enough data')
plt.title('Effect of sequence length on usable users')
plt.ylim(0, 105)
plt.grid(True)
plt.show()sequence_length usable_users percentage_of_users 0 1 51 100.000000 1 3 51 100.000000 2 7 50 98.039216 3 14 50 98.039216 4 21 50 98.039216 5 30 50 98.039216 6 50 49 96.078431
In [24]:
import sys, types
# Stub out keras/tensorflow so importing main.py doesn't require
# installing the full TensorFlow library — we only need filter_and_preprocess_data,
# which has nothing to do with neural networks.
keras_mod = types.ModuleType('keras')
keras_mod.Input = object
sys.modules['keras'] = keras_mod
sys.modules['keras.src'] = types.ModuleType('keras.src')
losses_mod = types.ModuleType('keras.src.losses')
losses_mod.SparseCategoricalCrossentropy = object
sys.modules['keras.src.losses'] = losses_mod
metrics_mod = types.ModuleType('keras.src.metrics')
metrics_mod.SparseCategoricalAccuracy = object
sys.modules['keras.src.metrics'] = metrics_mod
sys.modules['tensorflow'] = types.ModuleType('tensorflow')
sys.modules['tensorflow.keras'] = types.ModuleType('tensorflow.keras')
tfkm = types.ModuleType('tensorflow.keras.models')
tfkm.Sequential = object
sys.modules['tensorflow.keras.models'] = tfkm
tfkl = types.ModuleType('tensorflow.keras.layers')
tfkl.LSTM = object
tfkl.Dense = object
tfkl.Bidirectional = object
tfkl.GRU = object
sys.modules['tensorflow.keras.layers'] = tfkl
tfko = types.ModuleType('tensorflow.keras.optimizers')
tfko.Adam = object
sys.modules['tensorflow.keras.optimizers'] = tfko
print("TensorFlow/Keras stubbed — main.py can now be imported without it")TensorFlow/Keras stubbed — main.py can now be imported without it
In [26]:
%pip install "pandas==2.1.4"Defaulting to user installation because normal site-packages is not writeable
Collecting pandas==2.1.4
Downloading pandas-2.1.4.tar.gz (4.3 MB)
---------------------------------------- 0.0/4.3 MB ? eta -:--:--
--------- ------------------------------ 1.0/4.3 MB 11.8 MB/s eta 0:00:01
------------------------------- -------- 3.4/4.3 MB 11.6 MB/s eta 0:00:01
---------------------------------------- 4.3/4.3 MB 10.9 MB/s 0:00:00
Installing build dependencies: started
Installing build dependencies: still running...
Installing build dependencies: still running...
Installing build dependencies: still running...
Installing build dependencies: still running...
Installing build dependencies: finished with status 'done'
Getting requirements to build wheel: started
Getting requirements to build wheel: finished with status 'done'
Preparing metadata (pyproject.toml): started
Preparing metadata (pyproject.toml): finished with status 'error'
Note: you may need to restart the kernel to use updated packages.
error: subprocess-exited-with-error
× Preparing metadata (pyproject.toml) did not run successfully.
│ exit code: 1
╰─> [12 lines of output]
+ meson setup C:\Users\PMLS\AppData\Local\Temp\pip-install-27cw5jl9\pandas_7977c75aef884d0582454f16c9211d88 C:\Users\PMLS\AppData\Local\Temp\pip-install-27cw5jl9\pandas_7977c75aef884d0582454f16c9211d88\.mesonpy-zwd24j7z\build -Dbuildtype=release -Db_ndebug=if-release -Db_vscrt=md --vsenv --native-file=C:\Users\PMLS\AppData\Local\Temp\pip-install-27cw5jl9\pandas_7977c75aef884d0582454f16c9211d88\.mesonpy-zwd24j7z\build\meson-python-native-file.ini
The Meson build system
Version: 1.2.1
Source dir: C:\Users\PMLS\AppData\Local\Temp\pip-install-27cw5jl9\pandas_7977c75aef884d0582454f16c9211d88
Build dir: C:\Users\PMLS\AppData\Local\Temp\pip-install-27cw5jl9\pandas_7977c75aef884d0582454f16c9211d88\.mesonpy-zwd24j7z\build
Build type: native build
Project name: pandas
Project version: 2.1.4
..\..\meson.build:2:0: ERROR: Could not find C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe
A full log can be found at C:\Users\PMLS\AppData\Local\Temp\pip-install-27cw5jl9\pandas_7977c75aef884d0582454f16c9211d88\.mesonpy-zwd24j7z\build\meson-logs\meson-log.txt
[end of output]
note: This error originates from a subprocess, and is likely not a problem with pip.
[notice] A new release of pip is available: 26.1.2 -> 26.2.1
[notice] To update, run: python.exe -m pip install --upgrade pip
error: metadata-generation-failed
× Encountered error while generating package metadata.
╰─> pandas
note: This is an issue with the package mentioned above, not pip.
hint: See above for details.
In [28]:
# ============================================================
# Check how many users remain after the project's filtering
# ============================================================
# Start with the original EDA dataframe
project_df = df.copy()
print("Users in original dataset:", project_df['user'].nunique())
# ------------------------------------------------------------
# 1. Remove COVID-period data
# The project removes data from 2020 onwards.
# ------------------------------------------------------------
project_df = project_df[project_df['Year'] < 2020]
print("Users after removing COVID-period data:",
project_df['user'].nunique())
# ------------------------------------------------------------
# 2. Apply the same hourly binning used by the project
# ------------------------------------------------------------
for hour in hour_cols:
# Below 1000: round to nearest 10
mask_low = project_df[hour] < 1000
project_df.loc[mask_low, hour] = (
(project_df.loc[mask_low, hour] / 10).round() * 10
)
# 1000–9999: round to nearest 100
mask_mid = (
(project_df[hour] >= 1000) &
(project_df[hour] < 10000)
)
project_df.loc[mask_mid, hour] = (
(project_df.loc[mask_mid, hour] / 100).round() * 100
)
# Above 10000: cap at 10000
project_df.loc[project_df[hour] > 10000, hour] = 10000
# ------------------------------------------------------------
# 3. Remove duplicate rows
# The project uses the remaining unique rows to count
# usable datapoints for each user.
# ------------------------------------------------------------
cols_without_user = [
col for col in project_df.columns
if col != 'user'
]
reduced = project_df.drop_duplicates(
subset=cols_without_user,
keep=False
)
# ------------------------------------------------------------
# 4. Count usable datapoints for every user
# ------------------------------------------------------------
user_counts = reduced.groupby('user').size()
# Project threshold = 500 usable datapoints
users_kept = user_counts[user_counts >= 500]
users_removed = user_counts[user_counts < 500]
print("\nUsers kept by project:", len(users_kept))
print("Users removed by project:", len(users_removed))
print("\nUsers removed because they have fewer than 500 usable datapoints:")
print(users_removed.sort_index())Users in original dataset: 51 Users after removing COVID-period data: 48 Users kept by project: 32 Users removed by project: 14 Users removed because they have fewer than 500 usable datapoints: user 3 459 7 334 9 3 21 104 23 406 25 116 27 113 28 364 31 290 32 189 34 86 36 349 39 436 50 104 dtype: int64
In [29]:
# ============================================================
# User × Hour heatmap
# ============================================================
# Calculate average activity for each user at each hour
user_hour_mean = df.groupby('user')[hour_cols].mean()
plt.figure(figsize=(16, 12))
sns.heatmap(
user_hour_mean,
cmap='viridis',
linewidths=0.2
)
plt.title('Average Hourly Activity Pattern per User')
plt.xlabel('Hour of Day')
plt.ylabel('User ID')
plt.show()
In [ ]: