Strava/export/src/export/Main.java

package export;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import javax.net.ssl.HttpsURLConnection;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;

public class Main
{
	public static final String baseUrl = "https://www.strava.com/api/v3/";
	public static final String tokenUrl = "https://www.strava.com/api/v3/oauth/token";
	private static final int retryTimes = 100;
	private static final int httpCodeLimitReached = 429;
	private static int athleteId = -1;
	private static JSONParser parser = new JSONParser();
	private static String testRequest;
	private static File errorFile;
	//TODO: enduco needs to insert the correct request limits here
	private static final int requestLimit15Minutes = 100;
	private static final int requestLimitDay = 1000 / 3;

	private static int dailyRequestCount =0;
	private static int waitTimeMil = 60000*3*15/requestLimit15Minutes;
	private static long lastRequestTimeInMillis=0;

	private static void writeError(String text)
	{
		if (errorFile == null)
		{
			errorFile = new File("error_" + System.currentTimeMillis() + ".txt");
		}

		try (BufferedWriter bout = new BufferedWriter(new FileWriter(errorFile, true)))
		{
			bout.write(text);
			bout.newLine();
		}
		catch (IOException e)
		{
		}
	}

	/**
	 * For testing: ensures the test requests are used
	 * 
	 * @param testrequest Request to be used
	 */
	static void setTest(String testrequest)
	{
		testRequest = testrequest;
	}

	/**
	 * Saves the data of one athlete into a temp file and returns it.
	 * 
	 * @param token Identifier / authorization token of the athlete
	 * @return created temp file
	 */
	@SuppressWarnings("unchecked")
	static File oneAthlete(String token)
	{
		// get Activities
		Map<String, JSONObject> activities = getActivities(token);

		// for each activity: save streams
		JSONArray allActivities = new JSONArray();
		int simpleActivityId = 0;
		for (String id : activities.keySet())
		{
			JSONObject data = addStreams(id, activities.get(id), token);
			data.put("activity_id", simpleActivityId);
			allActivities.add(data);
			simpleActivityId++;
		}

		// get general information
		JSONObject athleteInfo = saveGeneralInformation(token);
		if(athleteInfo==null)	//error getting General Information
		{
			athleteInfo = new JSONObject();
		}
		athleteInfo.put("activities", allActivities);

		try
		{
			File temp = File.createTempFile("Athlete_" + athleteId, ".json");
			temp.deleteOnExit();
			BufferedWriter bw = new BufferedWriter(new FileWriter(temp));
			bw.write(athleteInfo.toString());
			bw.close();
			return temp;
		}
		catch (IOException e)
		{
			writeError("Athlete " + athleteId + ": Error writing temp file: " + e.toString());
		}
		return null;
	}

	/**
	 * Adds the streams to the given activity
	 * 
	 * @param id    Strava id of the activity
	 * @param data  general information of the activity
	 * @param token Identifier / authorization token of the athlete
	 * @return The data with the added streams
	 */
	@SuppressWarnings("unchecked")
	static JSONObject addStreams(String id, JSONObject data, String token)
	{
		String requestUrlExtension = "activities/" + id + "/streams?"
				+ "keys=[time,distance,latlng,altitude,velocity_smooth,heartrate,"
				+ "cadence,watts,temp,moving,grade_smooth]&key_by_type=true";
		
		String json = makeGetRequestWithRetry(requestUrlExtension, token);;

		if (json==null ||json.isEmpty()||json.isBlank()||json.equals(""))
		{
			return data;
		}
		Object obj;
		try
		{
			obj = parser.parse(json);
			JSONObject listOfStreams = (JSONObject) obj;
			for (Object key : listOfStreams.keySet())
			{
				JSONObject oneStream = (JSONObject) listOfStreams.get(key);
				data.put("stream_" + key.toString(), oneStream);
			}
		}
		catch (ParseException | NumberFormatException e)
		{
			writeError("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString());
		}
		return data;
	}

	/**
	 * Gathers all activities of a user, extracts the general information and the
	 * ids.
	 * 
	 * @param token Identifier / authorization token of the athlete
	 * @return A Map with key = Strava Id of an activity and value = JSONObject with
	 *         the general information of the activity
	 */
	@SuppressWarnings("unchecked")
	static Map<String, JSONObject> getActivities(String token)
	{
		Map<String, JSONObject> result = new HashMap<>();
		/*
		 * Possible values = AlpineSki, BackcountrySki, Canoeing, Crossfit, EBikeRide,
		 * Elliptical, Golf, Handcycle, Hike, IceSkate, InlineSkate, Kayaking, Kitesurf,
		 * NordicSki, Ride, RockClimbing, RollerSki, Rowing, Run, Sail, Skateboard,
		 * Snowboard, Snowshoe, Soccer, StairStepper, StandUpPaddling, Surfing, Swim,
		 * Velomobile, VirtualRide, VirtualRun, Walk, WeightTraining, Wheelchair,
		 * Windsurf, Workout, Yoga
		 */
		String type = "type";
		String timezone = "timezone";
		String start = "start_date_local"; // Start date measured in users time zone
		String id = "id";

		int pageIndex = 1;
		while (true)
		{							 
			//TODO: this 'per_page' value could perhaps be the root of all evil
			String requestExtension = "athlete/activities?per_page=100&page=" + pageIndex;
			String json = makeGetRequestWithRetry(requestExtension, token);
			if (json==null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know where the last page is...
			{
				break;
			}

			Object obj;
			try
			{
				obj = parser.parse(json);
				JSONArray listOfActivites = (JSONArray) obj;
				for (int i = 0; i < listOfActivites.size(); i++)
				{
					JSONObject oneActivity = (JSONObject) listOfActivites.get(i);

					JSONObject toSave = new JSONObject();
					toSave.put(type, oneActivity.get(type));
					toSave.put(timezone, oneActivity.get(timezone));
					toSave.put(start, oneActivity.get(start));
					toSave.put("athlete_id", athleteId);

					Object idObj = oneActivity.get(id);
					if (idObj != null)
					{
						result.put(oneActivity.get(id).toString(), toSave);
					}
				}
			}
			catch (ParseException | NumberFormatException e)
			{
				writeError("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString());
			}
			pageIndex++;
		}
		return result;
	}

	/**
	 * In the case that the daily request limit has been reached, this method waits for the next day
	 */
	static void checkRequestLimit()
	{
		if(dailyRequestCount>requestLimitDay) //daily request limit reached, wait until the next day
		{
			Calendar tomorrow = new GregorianCalendar();
			tomorrow.setTimeInMillis(System.currentTimeMillis());
			tomorrow.set(Calendar.DAY_OF_YEAR, tomorrow.get(Calendar.DAY_OF_YEAR)+1);
			if(tomorrow.get(Calendar.DAY_OF_YEAR)==1) //reached a new year ...
			{
				tomorrow.set(Calendar.YEAR, 2022);
			}
			tomorrow.set(Calendar.HOUR_OF_DAY, 0);
			tomorrow.set(Calendar.MINUTE, 0);
			try
			{
				TimeUnit.MILLISECONDS.sleep(tomorrow.getTimeInMillis()-System.currentTimeMillis());
			}
			catch (InterruptedException e1)
			{
			}
		}		
	}
	
	/**
	 * Method is used to find the next request window: It tries the same request again after 5 minutes.
	 * After a set number of times (retryTimes) it stops if it still wasn't successful.
	 * @param urlExtension UrlExtension for the request
	 * @param token Token for the request
	 * @return Data as a String or  {@code null} if there is no data
	 */
	static String makeGetRequestWithRetry(String urlExtension, String token)
	{
		checkRequestLimit();
		String json=null;
		int count=0;
		do
		{
			try
			{
				json = makeOneGetRequest(urlExtension, token);
				dailyRequestCount++;
			}
			catch (ResponseCodeWrongException e)
			{
				//tried enough times, so stop now
				if(count >= retryTimes)
				{
					writeError("Athlete: "+athleteId+" Retry limit reached. Last error code: "+e.getResponseCode());
					return null;
				}
				//request limit is reached, try again later
				if (e.getResponseCode()==httpCodeLimitReached)
				{
					count++;					
				}
				else //some other error: try only one other time!
				{
					count=retryTimes;
				}
				//Sleep for 5 minutes and try to get the next 15 min request window
				try
				{
					TimeUnit.MINUTES.sleep(5);
				}
				catch (InterruptedException e1)
				{
				}
			}
			
		}while(json==null);
		return json;
	}
	
	/**
	 * Extracts an athletes general information.
	 * 
	 * @param token Identifier / authorization token of the athlete
	 * @return extracted data or null if there was an error
	 */
	@SuppressWarnings("unchecked")
	static JSONObject saveGeneralInformation(String token)
	{
		String sex = "sex"; // Possible values = M, F
		String country = "country";
		String date_pref = "date_preference";
		String meas_pref = "measurement_preference"; // Possible values = feet, meters
		String weight = "weight";

		
		String json = makeGetRequestWithRetry("athlete", token);

		JSONObject toSave = new JSONObject();
		try
		{
			Object obj = parser.parse(json);
			JSONObject data = (JSONObject) obj;
			toSave.put(sex, data.get(sex));
			toSave.put(country, data.get(country));
			toSave.put(date_pref, data.get(date_pref));
			toSave.put(meas_pref, data.get(meas_pref));
			toSave.put(weight, data.get(weight));
			toSave.put("id", athleteId);

			return toSave;
		}
		catch (ParseException e)
		{
			writeError("Athlete " + athleteId + ": Error parsing general information.");
			return null;
		}
		catch (NullPointerException e)
		{
			writeError("Athlete " + athleteId + ": No general information found.");
			return null;			
		}
	}

	/**
	 * Zip a list of files in one .zip file.
	 * 
	 * @param files HasMap of <intended Filename, File> which should be zipped
	 * @param count	COunt or id to create distinct files each time
	 * @throws IOException If there was an error zipping
	 */
	static void zipFiles(Map<String, File> files, int count) throws IOException
	{
		FileOutputStream fos = new FileOutputStream("data("+count+").zip");
		ZipOutputStream zipOut = new ZipOutputStream(fos);
		for (String key : files.keySet())
		{
			File fileToZip = files.get(key);
			FileInputStream fis = new FileInputStream(fileToZip);
			ZipEntry zipEntry = new ZipEntry(key);
			zipOut.putNextEntry(zipEntry);

			byte[] bytes = new byte[1024];
			int length;
			while ((length = fis.read(bytes)) >= 0)
			{
				zipOut.write(bytes, 0, length);
			}
			fis.close();
		}
		zipOut.close();
		fos.close();
	}

	/**
	 * Handles one GET request to the API
	 * 
	 * @param requestUrlExtension Extension for the baseUrl (without '/')
	 * @param token               Identification / authorization token of the
	 *                            athlete
	 * @return The response as a String, an empty String in case of error.
	 * @throws ResponseCodeWrongException If there was an http error
	 */
	static String makeOneGetRequest(String requestUrlExtension, String token) throws ResponseCodeWrongException
	{
		if (testRequest != null)
		{
			String varTestRequest = testRequest.replaceAll("\\:\\s*([0-9]{15,})\\,", ":\"$1\",");
			testRequest = null;
			return varTestRequest;
		}
		HttpsURLConnection connection = null;
		try
		{
			long timeSinceLastRequest =System.currentTimeMillis()-lastRequestTimeInMillis; 
			if( timeSinceLastRequest<waitTimeMil)
			{
				try
				{
					TimeUnit.MILLISECONDS.sleep(waitTimeMil-timeSinceLastRequest);
				}
				catch (InterruptedException e)
				{
				}
			};
			lastRequestTimeInMillis=System.currentTimeMillis();
			// Create connection
			URL url = new URL(baseUrl + requestUrlExtension);
			connection = (HttpsURLConnection) url.openConnection();
			connection.setRequestMethod("GET");
			connection.setRequestProperty("Authorization", "Bearer " + token);

			return getResponse(connection);
		}
		catch (IOException e)
		{
			writeError("Athlete: "+athleteId+" Error while handling GET request: " + e.toString());
		}
		return "";
	}

	/**
	 * Reads the response from the connection and returns it as a String
	 * 
	 * @param connection Connection to the site
	 * @return Response as a String
	 * @throws IOException in case of error with the stream
	 * @throws ResponseCodeWrongException if no data was read because of http problems
	 */
	static String getResponse(HttpsURLConnection connection) throws IOException, ResponseCodeWrongException
	{
		StringBuilder result = new StringBuilder();
		int responseCode = connection.getResponseCode();
		if (responseCode != HttpURLConnection.HTTP_OK)
		{
			//excluded error messages appearing on missing streams and reached rate limit
			if(responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode!= httpCodeLimitReached)
			{
				writeError("Athlete: "+athleteId+" Wrong response code: " + responseCode);				
			}
			throw new ResponseCodeWrongException(responseCode);
		}

		try (Reader reader = new BufferedReader(
				new InputStreamReader(connection.getInputStream(), Charset.forName(StandardCharsets.UTF_8.name()))))
		{
			int c = reader.read();
			while (c != -1)
			{
				result.append((char) c);
				c = reader.read();
			}
		}

		// Numbers too long for int or long are turned into Strings
		return result.toString().replaceAll("\\:\\s*([0-9]{15,})\\,", ":\"$1\",");
	}

	/**
	 * Used for generating the accessToken
	 * 
	 * @param data Triplet containing: a the client_id, b the client_secret, c the
	 *             refresh_token
	 * @return The response as a String, an empty String in case of error.
	 * @throws ResponseCodeWrongException If there was an http error
	 */
	static String makeOnePostRequest(Triplet data) throws ResponseCodeWrongException
	{
		HttpsURLConnection connection = null;
		try
		{
			// Create connection
			URL url = new URL(tokenUrl);
			connection = (HttpsURLConnection) url.openConnection();
			connection.setRequestMethod("POST");
			connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
			
			Map<String, String> params = new LinkedHashMap<>();
			params.put("client_id", data.getA());
			params.put("client_secret", data.getB());
			params.put("grant_type", "refresh_token");
			params.put("refresh_token", data.getC());

			StringBuilder postData = new StringBuilder();
			for (String key : params.keySet())
			{
				if (postData.length() != 0)
				{
					postData.append('&');
				}
				postData.append(URLEncoder.encode(key, "UTF-8"));
				postData.append('=');
				postData.append(URLEncoder.encode(params.get(key), "UTF-8"));
			}
			byte[] postDataBytes = postData.toString().getBytes("UTF-8");

			connection.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
			connection.setDoOutput(true);
			connection.getOutputStream().write(postDataBytes);

			return getResponse(connection);
		}
		catch (IOException e)
		{
			writeError("Athlete: "+athleteId+"Error while handling POST request: " + e.toString());
		}
		return "";
	}

	/**
	 * Sends the old refresh token to strava and retrieves a new one and an access token
	 * @param refreshInfo Refresh data with a the cliend_id, b the client_secret and c the refresh_token
	 * @return A tuple with a the access_token and b the new refresh_token or {@code null} if there was an error
	 */
	static Tuple getAccessToken(Triplet refreshInfo)
	{
		checkRequestLimit();
		String json=null;
		int count =0;
		do
		{
			try
			{
				json = makeOnePostRequest(refreshInfo);
				dailyRequestCount++;
			}
			catch (ResponseCodeWrongException e)
			{
				//tried enough times, so stop now
				if(count >= retryTimes)
				{
					return null;
				}
				//request limit is reached, try again later
				if (e.getResponseCode()==httpCodeLimitReached)
				{
					count++;					
				}
				else //some other error: try only one other time!
				{
					count=retryTimes;
				}
				//Sleep for 5 minutes and try to get the next 15 min request window
				try
				{
					TimeUnit.MINUTES.sleep(5);
				}
				catch (InterruptedException e1)
				{
				}
			}
			
		}while(json==null);
		try
		{
			Object obj = parser.parse(json);
			JSONObject data = (JSONObject) obj;
			return new Tuple(data.get("access_token").toString(), data.get("refresh_token").toString());
		}
		catch (ParseException e)
		{
			writeError("Athlete " + athleteId + ": Error parsing general information.");
		}
		return null;
	}

	public static void main(String[] args)
	{
		// TODO: tokens need to be added by Enduco

		// Triplet a is client_id, b is client_secret, c is refresh_token
		List<Triplet> refreshTokens = new ArrayList<>();
		// don't know if you need it but here you will find the new refresh tokens after
		// the requests
		List<Triplet> newRefreshTokens = new ArrayList<>();

		Map<String, File> allFiles = new HashMap<>();
		int zipcount =1;
		for (Triplet oneUser : refreshTokens)
		{
			athleteId++;
			// a is access_token and b is new refresh_token
			Tuple withAccessToken = getAccessToken(oneUser);
			if (withAccessToken == null)
			{
				writeError("Couldn't get new access token for client "+athleteId);	
				continue;
			}
			newRefreshTokens.add(new Triplet(oneUser.getA(), oneUser.getB(), withAccessToken.getB()));

			File athlete = oneAthlete(withAccessToken.getA());
			if (athlete != null)
			{
				allFiles.put("Athlete_" + athleteId + ".json", athlete);
			}
			//pack zip-files of 10 athletes
			if(allFiles.size()>=10)
			{
				try
				{
					zipFiles(allFiles, zipcount);
					zipcount++;
					allFiles=new HashMap<>();
				}
				catch (IOException e)
				{
					writeError("Files coulnd't be zipped");
				}				
			}
		}
		//zip the rest
		try
		{
			zipFiles(allFiles, zipcount);
		}
		catch (IOException e)
		{
			writeError("Files coulnd't be zipped");
		}
	}
}