Strava/export/src/export/Main.java

package export;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import javax.net.ssl.HttpsURLConnection;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;

public class Main
{
	public static final String baseUrl = "https://www.strava.com/api/v3/";
	public static final String tokenUrl = "https://www.strava.com/api/v3/oauth/token";
	private static final int retryTimes = 100;
	private static final int httpCodeLimitReached = 429;
	private static int athleteId = -1;
	private static JSONParser parser = new JSONParser();
	private static String testRequest;
	private static File errorFile;
	// TODO: enduco needs to insert the correct request limits here
	private static final int requestLimit15Minutes = 100 / 3;
	private static final int requestLimitDay = 1000 / 3;

	private static int simpleActivityId=0;
	private static int dailyRequestCount = 0;
	private static int fifteenMinuteRequestCount = 0;
	private static long firstRequestTimeInCurrent15MinInterval = 0;
	private static String accessToken = "";
	/**
	 * a is client_id, b is client_secret, c is refresh_token
	 */
	private static Triplet refreshInfo;

	private static void writeLog(String text, InputStream errorStream)
	{
		SimpleDateFormat format = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
		
		if (errorFile == null)
		{
			errorFile = new File("log_" + System.currentTimeMillis() + ".txt");
		}

		try (BufferedWriter bout = new BufferedWriter(new FileWriter(errorFile, true)))
		{
			bout.write(format.format(new Date(System.currentTimeMillis()))+": "+text);
			bout.newLine();
			if (errorStream!= null)
			{
				BufferedReader bin = new BufferedReader(new InputStreamReader(errorStream));

				String errorLine = bin.readLine();
				while (errorLine != null)
				{
					bout.write(" **** "+errorLine);
					bout.newLine();
					errorLine = bin.readLine();
				}
				
				bin.close();
			}
		}
		catch (IOException e)
		{
		}
	}
	
	private static void writeLog(String text)
	{
		writeLog(text, null);
	}
	
	

	/**
	 * For testing: ensures the test requests are used
	 * 
	 * @param testrequest Request to be used
	 */
	static void setTest(String testrequest)
	{
		testRequest = testrequest;
	}

	/**
	 * Saves the data of one athlete into a temp file and returns it.
	 * 
	 * @return created temp file
	 */
	@SuppressWarnings("unchecked")
	static File oneAthlete()
	{
		// get Activities and general information
		Map<String, JSONObject> activities;
		JSONObject athleteInfo;
		try
		{
			activities = getActivities();
			athleteInfo = saveGeneralInformation();
		}
		catch (NoAccessException e1)
		{
			writeLog("Athlete " + athleteId + ": Access expired and no new token possible");
			return null; // no data at all. Stop right away
		}
		if (athleteInfo == null) // error getting General Information
		{
			athleteInfo = new JSONObject();
		}
		

		// for each activity: save streams
		JSONArray allActivities = new JSONArray();
		simpleActivityId = 0;
		for (String id : activities.keySet())
		{
			JSONObject data;
			try
			{
				data = addStreams(id, activities.get(id));
			}
			catch (NoAccessException e)
			{
				writeLog("Athlete " + athleteId + ": Access expired and no new token possible");
				break; //stop the loop and save what you got up to there
			}
			data.put("activity_id", simpleActivityId);
			allActivities.add(data);
			simpleActivityId++;
		}

		athleteInfo.put("activities", allActivities);

		try
		{
			File temp = File.createTempFile("Athlete_" + athleteId, ".json");
			temp.deleteOnExit();
			BufferedWriter bw = new BufferedWriter(new FileWriter(temp));
			bw.write(athleteInfo.toString());
			bw.close();
			return temp;
		}
		catch (IOException e)
		{
			writeLog("Athlete " + athleteId + ": Error writing temp file: " + e.toString());
		}
		return null;
	}

	/**
	 * Adds the streams to the given activity
	 * 
	 * @param id   Strava id of the activity
	 * @param data general information of the activity
	 * @return The data with the added streams
	 * @throws NoAccessException If the access token expired and no new one could be acquired.
	 */
	@SuppressWarnings("unchecked")
	static JSONObject addStreams(String id, JSONObject data) throws NoAccessException
	{
		String requestUrlExtension = "activities/" + id + "/streams?"
				+ "keys=[time,distance,latlng,altitude,velocity_smooth,heartrate,"
				+ "cadence,watts,temp,moving,grade_smooth]&key_by_type=true";

		String json = makeGetRequestWithRetry(requestUrlExtension);

		if (json == null || json.isEmpty() || json.isBlank() || json.equals(""))
		{
			return data;
		}
		Object obj;
		try
		{
			obj = parser.parse(json);
			JSONObject listOfStreams = (JSONObject) obj;
			for (Object key : listOfStreams.keySet())
			{
				JSONObject oneStream = (JSONObject) listOfStreams.get(key);
				data.put("stream_" + key.toString(), oneStream);
			}
		}
		catch (ParseException | NumberFormatException e)
		{
			writeLog("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString());
		}
		return data;
	}

	/**
	 * Gathers all activities of a user, extracts the general information and the
	 * ids.
	 * 
	 * @return A Map with key = Strava Id of an activity and value = JSONObject with
	 *         the general information of the activity
	 * @throws NoAccessException If the access token expired and no new one could be acquired.
	 */
	@SuppressWarnings("unchecked")
	static Map<String, JSONObject> getActivities() throws NoAccessException
	{
		Map<String, JSONObject> result = new HashMap<>();
		/*
		 * Possible values = AlpineSki, BackcountrySki, Canoeing, Crossfit, EBikeRide,
		 * Elliptical, Golf, Handcycle, Hike, IceSkate, InlineSkate, Kayaking, Kitesurf,
		 * NordicSki, Ride, RockClimbing, RollerSki, Rowing, Run, Sail, Skateboard,
		 * Snowboard, Snowshoe, Soccer, StairStepper, StandUpPaddling, Surfing, Swim,
		 * Velomobile, VirtualRide, VirtualRun, Walk, WeightTraining, Wheelchair,
		 * Windsurf, Workout, Yoga
		 */
		String type = "type";
		String timezone = "timezone";
		String start = "start_date_local"; // Start date measured in users time zone
		String id = "id";

		int pageIndex = 1;
		while (true)
		{
			String requestExtension = "athlete/activities?per_page=100&page=" + pageIndex;
			String json = makeGetRequestWithRetry(requestExtension);
			if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) 
			// don't know where the last page is...
			{
				break;
			}

			Object obj;
			try
			{
				obj = parser.parse(json);
				JSONArray listOfActivites = (JSONArray) obj;
				for (int i = 0; i < listOfActivites.size(); i++)
				{
					JSONObject oneActivity = (JSONObject) listOfActivites.get(i);

					JSONObject toSave = new JSONObject();
					toSave.put(type, oneActivity.get(type));
					toSave.put(timezone, oneActivity.get(timezone));
					toSave.put(start, oneActivity.get(start));
					toSave.put("athlete_id", athleteId);

					Object idObj = oneActivity.get(id);
					if (idObj != null)
					{
						result.put(oneActivity.get(id).toString(), toSave);
					}
				}
			}
			catch (ParseException | NumberFormatException e)
			{
				writeLog("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString());
			}
			pageIndex++;
		}
		writeLog("Athlete " + athleteId + ": Found "+result.size()+"Activities");
		return result;
	}

	/**
	 * In the case that the daily request limit has been reached, this method waits
	 * for the next day
	 */
	static void checkRequestLimit()
	{
		if (dailyRequestCount > requestLimitDay) // daily request limit reached, wait until the next day
		{
			//Write info what you did today
			writeLog("Daily requests done. Current status: Athlete "+athleteId+", Activity "+simpleActivityId);
			Calendar tomorrow = new GregorianCalendar();
			tomorrow.setTimeInMillis(System.currentTimeMillis());
			tomorrow.set(Calendar.DAY_OF_YEAR, tomorrow.get(Calendar.DAY_OF_YEAR) + 1);
			if (tomorrow.get(Calendar.DAY_OF_YEAR) == 1) // reached a new year ...
			{
				tomorrow.set(Calendar.YEAR, 2022);
			}
			tomorrow.set(Calendar.HOUR_OF_DAY, 0);
			tomorrow.set(Calendar.MINUTE, 0);
			try
			{
				TimeUnit.MILLISECONDS.sleep(tomorrow.getTimeInMillis() - System.currentTimeMillis());
			}
			catch (InterruptedException e1)
			{
			}
			dailyRequestCount=0;
		}
		
		//check fifteen Minute limit
		if (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval > 15*60*1000 )
		{
			//more than 15 Minutes have passed, so we have a new 15 minute limit
			fifteenMinuteRequestCount =0;
		}
		else if(fifteenMinuteRequestCount >= requestLimit15Minutes)	
		{
			//we are still in the old interval and reached the limit
			long sleeptime = 15*60*1000 - (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval);
			//sleep for the rest of the interval and start in a new interval
			try
			{
				TimeUnit.MILLISECONDS.sleep(sleeptime);
				fifteenMinuteRequestCount=0;
			}
			catch (InterruptedException e)
			{
			}
		}	
	}

	/**
	 * Method is used to find the next request window: It tries the same request
	 * again after 5 minutes. After a set number of times (retryTimes) it stops if
	 * it still wasn't successful.
	 * 
	 * @param urlExtension UrlExtension for the request
	 * @return Data as a String or {@code null} if there is no data
	 * @throws NoAccessException If the access token expired and no new one could be acquired.
	 */
	static String makeGetRequestWithRetry(String urlExtension) throws NoAccessException
	{
		checkRequestLimit();
		String json = null;
		int count = 0;
		do
		{
			try
			{
				json = makeOneGetRequest(urlExtension);
			}
			catch (ResponseCodeWrongException e)
			{
				count = handleWrongResponseCode(count, e);
				if (count == -1)
				{
					return null;
				}
			}

		} while (json == null);
		return json;
	}
	
	/**
	 * Method processed the wrong response code and adjusts the retry times as needed: 
	 * When the retry limit is reached, -1 is returned.
	 * When the access limit is reached, the retry count is increased.
	 * When an unknown error appears, the retry count is set to max to allow only one more retry.
	 * After that, the method sleeps for 5 Minutes.
	 * @param count Retry count
	 * @param e Exception containing the information
	 * @return the new retry count or -1 if there are no more tries left
	 * @throws NoAccessException if no AccessToken could be requested
	 */
	static int handleWrongResponseCode(int count, ResponseCodeWrongException e) throws NoAccessException
	{
		// tried enough times, so stop now
		if (count >= retryTimes)
		{
			writeLog(
					"Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode());
			return -1;
		}
		if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
		{	//token might have expired
			if(!getAccessToken())	//token doesn't work anymore and we can't get a new one
			{
				throw new NoAccessException();
			}
		}
		else if (e.getResponseCode() == httpCodeLimitReached)
		{// request limit is reached, try again later
			count++;
		}
		else // some other error: try only one other time!
		{
			count = retryTimes;
		}
		// Sleep for 5 minutes and try to get the next 15 min request window
		try
		{
			TimeUnit.MINUTES.sleep(5);
		}
		catch (InterruptedException e1)
		{
		}
		return count;
	}

	/**
	 * Extracts an athletes general information.
	 * 
	 * @return extracted data or null if there was an error
	 * @throws NoAccessException If the access token expired and no new one could be acquired.
	 */
	@SuppressWarnings("unchecked")
	static JSONObject saveGeneralInformation() throws NoAccessException
	{
		String sex = "sex"; // Possible values = M, F
		String country = "country";
		String date_pref = "date_preference";
		String meas_pref = "measurement_preference"; // Possible values = feet, meters
		String weight = "weight";

		String json = makeGetRequestWithRetry("athlete");

		JSONObject toSave = new JSONObject();
		try
		{
			Object obj = parser.parse(json);
			JSONObject data = (JSONObject) obj;
			toSave.put(sex, data.get(sex));
			toSave.put(country, data.get(country));
			toSave.put(date_pref, data.get(date_pref));
			toSave.put(meas_pref, data.get(meas_pref));
			toSave.put(weight, data.get(weight));
			toSave.put("id", athleteId);

			return toSave;
		}
		catch (ParseException e)
		{
			writeLog("Athlete " + athleteId + ": Error parsing general information.");
			return null;
		}
		catch (NullPointerException e)
		{
			writeLog("Athlete " + athleteId + ": No general information found.");
			return null;
		}
	}

	/**
	 * Zip a list of files in one .zip file.
	 * 
	 * @param files HasMap of <intended Filename, File> which should be zipped
	 * @param count COunt or id to create distinct files each time
	 * @throws IOException If there was an error zipping
	 */
	static void zipFiles(Map<String, File> files, int count) throws IOException
	{
		FileOutputStream fos = new FileOutputStream("data(" + count + ").zip");
		ZipOutputStream zipOut = new ZipOutputStream(fos);
		for (String key : files.keySet())
		{
			File fileToZip = files.get(key);
			FileInputStream fis = new FileInputStream(fileToZip);
			ZipEntry zipEntry = new ZipEntry(key);
			zipOut.putNextEntry(zipEntry);

			byte[] bytes = new byte[1024];
			int length;
			while ((length = fis.read(bytes)) >= 0)
			{
				zipOut.write(bytes, 0, length);
			}
			fis.close();
		}
		zipOut.close();
		fos.close();
	}

	/**
	 * Handles one GET request to the API
	 * 
	 * @param requestUrlExtension Extension for the baseUrl (without '/')
	 * @return The response as a String, an empty String in case of error.
	 * @throws ResponseCodeWrongException If there was an http error
	 */
	static String makeOneGetRequest(String requestUrlExtension) throws ResponseCodeWrongException
	{
		if (testRequest != null)
		{
			String varTestRequest = testRequest.replaceAll("\\:\\s*([0-9]{15,})\\,", ":\"$1\",");
			testRequest = null;
			return varTestRequest;
		}
		HttpsURLConnection connection = null;
		try
		{
			// Create connection
			URL url = new URL(baseUrl + requestUrlExtension);
			connection = (HttpsURLConnection) url.openConnection();
			connection.setRequestMethod("GET");
			connection.setRequestProperty("Authorization", "Bearer " + accessToken);

			return getResponse(connection);
		}
		catch (IOException e)
		{
			writeLog("Athlete: " + athleteId + " Error while handling GET request: " + e.toString());
		}
		return "";
	}

	/**
	 * Reads the response from the connection and returns it as a String
	 * 
	 * @param connection Connection to the site
	 * @return Response as a String
	 * @throws IOException                in case of error with the stream
	 * @throws ResponseCodeWrongException if no data was read because of http
	 *                                    problems
	 */
	static String getResponse(HttpsURLConnection connection) throws IOException, ResponseCodeWrongException
	{
		StringBuilder result = new StringBuilder();
		int responseCode = connection.getResponseCode();
		
		if( fifteenMinuteRequestCount == 0) //first request in a 15 Min Interval
		{
			firstRequestTimeInCurrent15MinInterval = System.currentTimeMillis();			
		}
		dailyRequestCount++;
		fifteenMinuteRequestCount++;
		
		if (responseCode != HttpURLConnection.HTTP_OK)
		{
			// excluded error messages appearing on missing streams and reached rate limit
			if (responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode != httpCodeLimitReached)
			{
				writeLog("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream());
			}
			throw new ResponseCodeWrongException(responseCode);
		}

		try (Reader reader = new BufferedReader(
				new InputStreamReader(connection.getInputStream(), Charset.forName(StandardCharsets.UTF_8.name()))))
		{
			int c = reader.read();
			while (c != -1)
			{
				result.append((char) c);
				c = reader.read();
			}
		}

		// Numbers too long for int or long are turned into Strings
		return result.toString().replaceAll("\\:\\s*([0-9]{15,})\\,", ":\"$1\",");
	}

	/**
	 * Used for generating the accessToken
	 * 
	 * @return The response as a String, an empty String in case of error.
	 * @throws ResponseCodeWrongException If there was an http error
	 */
	static String makeOnePostRequest() throws ResponseCodeWrongException
	{
		HttpsURLConnection connection = null;
		try
		{
			// Create connection
			URL url = new URL(tokenUrl);
			connection = (HttpsURLConnection) url.openConnection();
			connection.setRequestMethod("POST");
			connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");

			Map<String, String> params = new LinkedHashMap<>();
			params.put("client_id", refreshInfo.getA());
			params.put("client_secret", refreshInfo.getB());
			params.put("grant_type", "refresh_token");
			params.put("refresh_token", refreshInfo.getC());

			StringBuilder postData = new StringBuilder();
			for (String key : params.keySet())
			{
				if (postData.length() != 0)
				{
					postData.append('&');
				}
				postData.append(URLEncoder.encode(key, "UTF-8"));
				postData.append('=');
				postData.append(URLEncoder.encode(params.get(key), "UTF-8"));
			}
			byte[] postDataBytes = postData.toString().getBytes("UTF-8");

			connection.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
			connection.setDoOutput(true);
			connection.getOutputStream().write(postDataBytes);

			return getResponse(connection);
		}
		catch (IOException e)
		{
			writeLog("Athlete: " + athleteId + "Error while handling POST request: " + e.toString());
		}
		return "";
	}

	/**
	 * Sends the old refresh token to strava and retrieves a new one and an access
	 * token
	 * 
	 * @param refreshInfo Refresh data with a the cliend_id, b the client_secret and
	 *                    c the refresh_token
	 * @return {@code true} if everything went right or {@code false} if there was
	 *         an error
	 */
	static boolean getAccessToken()
	{
		checkRequestLimit();
		String json = null;
		int count = 0;
		do
		{
			try
			{
				json = makeOnePostRequest();
			}
			catch (ResponseCodeWrongException e)
			{
				try
				{
					count = handleWrongResponseCode(count, e);
				}
				catch (NoAccessException e2)
				{
					return false;
				}
				if (count == -1)
				{
					return false;
				}
			}

		} while (json == null);
		try
		{
			Object obj = parser.parse(json);
			JSONObject data = (JSONObject) obj;
			accessToken = data.get("access_token").toString();
			refreshInfo.setC(data.get("refresh_token").toString());
			return true;
		}
		catch (ParseException e)
		{
			writeLog("Athlete " + athleteId + ": Error parsing refresh info.");
		}
		return false;
	}

	public static void main(String[] args)
	{
		// TODO: tokens need to be added by Enduco

		// Triplet a is client_id, b is client_secret, c is refresh_token
		List<Triplet> refreshTokens = new ArrayList<>();
		// don't know if you need it but here you will find the new refresh tokens after
		// the requests
		List<Triplet> newRefreshTokens = new ArrayList<>();

		Map<String, File> allFiles = new HashMap<>();
		int zipcount = 1;
		for (Triplet oneUser : refreshTokens)
		{
			refreshInfo = oneUser;
			athleteId++;
			if (!getAccessToken())
			{
				writeLog("Couldn't get new access token for client " + athleteId);
				continue;
			}

			File athlete = oneAthlete();
			if (athlete != null)
			{
				allFiles.put("Athlete_" + athleteId + ".json", athlete);
			}
			newRefreshTokens.add(refreshInfo);

			// pack zip-files of 10 athletes
			if (allFiles.size() >= 10)
			{
				try
				{
					zipFiles(allFiles, zipcount);
					zipcount++;
					allFiles = new HashMap<>();
				}
				catch (IOException e)
				{
					writeLog("Files coulnd't be zipped");
				}
			}
		}
		// zip the rest
		try
		{
			zipFiles(allFiles, zipcount);
		}
		catch (IOException e)
		{
			writeLog("Files coulnd't be zipped");
		}
	}
}