package export; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import javax.net.ssl.HttpsURLConnection; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; public class Main { public static final String baseUrl = "https://www.strava.com/api/v3/"; public static final String tokenUrl = "https://www.strava.com/api/v3/oauth/token"; private static final int retryTimes = 100; private static final int httpCodeLimitReached = 429; private static int athleteId = -1; private static JSONParser parser = new JSONParser(); private static String testRequest; private static File errorFile; // TODO: enduco needs to insert the correct request limits here private static final int requestLimit15Minutes = 100 / 3; private static final int requestLimitDay = 1000 / 3; private static int simpleActivityId=0; private static int dailyRequestCount = 0; private static int fifteenMinuteRequestCount = 0; private static long firstRequestTimeInCurrent15MinInterval = 0; private static String accessToken = ""; /** * a is client_id, b is client_secret, c is refresh_token */ private static Triplet refreshInfo; private static void writeLog(String text, InputStream errorStream) { SimpleDateFormat format = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss"); if (errorFile == null) { errorFile = new File("log_" + System.currentTimeMillis() + ".txt"); } try (BufferedWriter bout = new BufferedWriter(new FileWriter(errorFile, true))) { bout.write(format.format(new Date(System.currentTimeMillis()))+": "+text); bout.newLine(); if (errorStream!= null) { BufferedReader bin = new BufferedReader(new InputStreamReader(errorStream)); String errorLine = bin.readLine(); while (errorLine != null) { bout.write(" **** "+errorLine); bout.newLine(); errorLine = bin.readLine(); } bin.close(); } } catch (IOException e) { } } private static void writeLog(String text) { writeLog(text, null); } /** * For testing: ensures the test requests are used * * @param testrequest Request to be used */ static void setTest(String testrequest) { testRequest = testrequest; } /** * Saves the data of one athlete into a temp file and returns it. * * @return created temp file */ @SuppressWarnings("unchecked") static File oneAthlete() { // get Activities and general information Map activities; JSONObject athleteInfo; try { activities = getActivities(); athleteInfo = saveGeneralInformation(); } catch (NoAccessException e1) { writeLog("Athlete " + athleteId + ": Access expired and no new token possible"); return null; // no data at all. Stop right away } if (athleteInfo == null) // error getting General Information { athleteInfo = new JSONObject(); } // for each activity: save streams JSONArray allActivities = new JSONArray(); simpleActivityId = 0; for (String id : activities.keySet()) { JSONObject data; try { data = addStreams(id, activities.get(id)); } catch (NoAccessException e) { writeLog("Athlete " + athleteId + ": Access expired and no new token possible"); break; //stop the loop and save what you got up to there } data.put("activity_id", simpleActivityId); allActivities.add(data); simpleActivityId++; } athleteInfo.put("activities", allActivities); try { File temp = File.createTempFile("Athlete_" + athleteId, ".json"); temp.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new FileWriter(temp)); bw.write(athleteInfo.toString()); bw.close(); return temp; } catch (IOException e) { writeLog("Athlete " + athleteId + ": Error writing temp file: " + e.toString()); } return null; } /** * Adds the streams to the given activity * * @param id Strava id of the activity * @param data general information of the activity * @return The data with the added streams * @throws NoAccessException If the access token expired and no new one could be acquired. */ @SuppressWarnings("unchecked") static JSONObject addStreams(String id, JSONObject data) throws NoAccessException { String requestUrlExtension = "activities/" + id + "/streams?" + "keys=[time,distance,latlng,altitude,velocity_smooth,heartrate," + "cadence,watts,temp,moving,grade_smooth]&key_by_type=true"; String json = makeGetRequestWithRetry(requestUrlExtension); if (json == null || json.isEmpty() || json.isBlank() || json.equals("")) { return data; } Object obj; try { obj = parser.parse(json); JSONObject listOfStreams = (JSONObject) obj; for (Object key : listOfStreams.keySet()) { JSONObject oneStream = (JSONObject) listOfStreams.get(key); data.put("stream_" + key.toString(), oneStream); } } catch (ParseException | NumberFormatException e) { writeLog("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString()); } return data; } /** * Gathers all activities of a user, extracts the general information and the * ids. * * @return A Map with key = Strava Id of an activity and value = JSONObject with * the general information of the activity * @throws NoAccessException If the access token expired and no new one could be acquired. */ @SuppressWarnings("unchecked") static Map getActivities() throws NoAccessException { Map result = new HashMap<>(); /* * Possible values = AlpineSki, BackcountrySki, Canoeing, Crossfit, EBikeRide, * Elliptical, Golf, Handcycle, Hike, IceSkate, InlineSkate, Kayaking, Kitesurf, * NordicSki, Ride, RockClimbing, RollerSki, Rowing, Run, Sail, Skateboard, * Snowboard, Snowshoe, Soccer, StairStepper, StandUpPaddling, Surfing, Swim, * Velomobile, VirtualRide, VirtualRun, Walk, WeightTraining, Wheelchair, * Windsurf, Workout, Yoga */ String type = "type"; String timezone = "timezone"; String start = "start_date_local"; // Start date measured in users time zone String id = "id"; int pageIndex = 1; while (true) { String requestExtension = "athlete/activities?per_page=100&page=" + pageIndex; String json = makeGetRequestWithRetry(requestExtension); if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know where the last page is... { break; } Object obj; try { obj = parser.parse(json); JSONArray listOfActivites = (JSONArray) obj; for (int i = 0; i < listOfActivites.size(); i++) { JSONObject oneActivity = (JSONObject) listOfActivites.get(i); JSONObject toSave = new JSONObject(); toSave.put(type, oneActivity.get(type)); toSave.put(timezone, oneActivity.get(timezone)); toSave.put(start, oneActivity.get(start)); toSave.put("athlete_id", athleteId); Object idObj = oneActivity.get(id); if (idObj != null) { result.put(oneActivity.get(id).toString(), toSave); } } } catch (ParseException | NumberFormatException e) { writeLog("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString()); } pageIndex++; } writeLog("Athlete " + athleteId + ": Found "+result.size()+"Activities"); return result; } /** * In the case that the daily request limit has been reached, this method waits * for the next day */ static void checkRequestLimit() { if (dailyRequestCount > requestLimitDay) // daily request limit reached, wait until the next day { //Write info what you did today writeLog("Daily requests done. Current status: Athlete "+athleteId+", Activity "+simpleActivityId); Calendar tomorrow = new GregorianCalendar(); tomorrow.setTimeInMillis(System.currentTimeMillis()); tomorrow.set(Calendar.DAY_OF_YEAR, tomorrow.get(Calendar.DAY_OF_YEAR) + 1); if (tomorrow.get(Calendar.DAY_OF_YEAR) == 1) // reached a new year ... { tomorrow.set(Calendar.YEAR, 2022); } tomorrow.set(Calendar.HOUR_OF_DAY, 0); tomorrow.set(Calendar.MINUTE, 0); try { TimeUnit.MILLISECONDS.sleep(tomorrow.getTimeInMillis() - System.currentTimeMillis()); } catch (InterruptedException e1) { } dailyRequestCount=0; } //check fifteen Minute limit if (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval > 15*60*1000 ) { //more than 15 Minutes have passed, so we have a new 15 minute limit fifteenMinuteRequestCount =0; } else if(fifteenMinuteRequestCount >= requestLimit15Minutes) { //we are still in the old interval and reached the limit long sleeptime = 15*60*1000 - (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval); //sleep for the rest of the interval and start in a new interval try { TimeUnit.MILLISECONDS.sleep(sleeptime); fifteenMinuteRequestCount=0; } catch (InterruptedException e) { } } } /** * Method is used to find the next request window: It tries the same request * again after 5 minutes. After a set number of times (retryTimes) it stops if * it still wasn't successful. * * @param urlExtension UrlExtension for the request * @return Data as a String or {@code null} if there is no data * @throws NoAccessException If the access token expired and no new one could be acquired. */ static String makeGetRequestWithRetry(String urlExtension) throws NoAccessException { checkRequestLimit(); String json = null; int count = 0; do { try { json = makeOneGetRequest(urlExtension); } catch (ResponseCodeWrongException e) { count = handleWrongResponseCode(count, e); if (count == -1) { return null; } } } while (json == null); return json; } /** * Method processed the wrong response code and adjusts the retry times as needed: * When the retry limit is reached, -1 is returned. * When the access limit is reached, the retry count is increased. * When an unknown error appears, the retry count is set to max to allow only one more retry. * After that, the method sleeps for 5 Minutes. * @param count Retry count * @param e Exception containing the information * @return the new retry count or -1 if there are no more tries left * @throws NoAccessException if no AccessToken could be requested */ static int handleWrongResponseCode(int count, ResponseCodeWrongException e) throws NoAccessException { // tried enough times, so stop now if (count >= retryTimes) { writeLog( "Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode()); return -1; } if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED) { //token might have expired if(!getAccessToken()) //token doesn't work anymore and we can't get a new one { throw new NoAccessException(); } } else if (e.getResponseCode() == httpCodeLimitReached) {// request limit is reached, try again later count++; } else // some other error: try only one other time! { count = retryTimes; } // Sleep for 5 minutes and try to get the next 15 min request window try { TimeUnit.MINUTES.sleep(5); } catch (InterruptedException e1) { } return count; } /** * Extracts an athletes general information. * * @return extracted data or null if there was an error * @throws NoAccessException If the access token expired and no new one could be acquired. */ @SuppressWarnings("unchecked") static JSONObject saveGeneralInformation() throws NoAccessException { String sex = "sex"; // Possible values = M, F String country = "country"; String date_pref = "date_preference"; String meas_pref = "measurement_preference"; // Possible values = feet, meters String weight = "weight"; String json = makeGetRequestWithRetry("athlete"); JSONObject toSave = new JSONObject(); try { Object obj = parser.parse(json); JSONObject data = (JSONObject) obj; toSave.put(sex, data.get(sex)); toSave.put(country, data.get(country)); toSave.put(date_pref, data.get(date_pref)); toSave.put(meas_pref, data.get(meas_pref)); toSave.put(weight, data.get(weight)); toSave.put("id", athleteId); return toSave; } catch (ParseException e) { writeLog("Athlete " + athleteId + ": Error parsing general information."); return null; } catch (NullPointerException e) { writeLog("Athlete " + athleteId + ": No general information found."); return null; } } /** * Zip a list of files in one .zip file. * * @param files HasMap of which should be zipped * @param count COunt or id to create distinct files each time * @throws IOException If there was an error zipping */ static void zipFiles(Map files, int count) throws IOException { FileOutputStream fos = new FileOutputStream("data(" + count + ").zip"); ZipOutputStream zipOut = new ZipOutputStream(fos); for (String key : files.keySet()) { File fileToZip = files.get(key); FileInputStream fis = new FileInputStream(fileToZip); ZipEntry zipEntry = new ZipEntry(key); zipOut.putNextEntry(zipEntry); byte[] bytes = new byte[1024]; int length; while ((length = fis.read(bytes)) >= 0) { zipOut.write(bytes, 0, length); } fis.close(); } zipOut.close(); fos.close(); } /** * Handles one GET request to the API * * @param requestUrlExtension Extension for the baseUrl (without '/') * @return The response as a String, an empty String in case of error. * @throws ResponseCodeWrongException If there was an http error */ static String makeOneGetRequest(String requestUrlExtension) throws ResponseCodeWrongException { if (testRequest != null) { String varTestRequest = testRequest.replaceAll("\\:\\s*([0-9]{15,})\\,", ":\"$1\","); testRequest = null; return varTestRequest; } HttpsURLConnection connection = null; try { // Create connection URL url = new URL(baseUrl + requestUrlExtension); connection = (HttpsURLConnection) url.openConnection(); connection.setRequestMethod("GET"); connection.setRequestProperty("Authorization", "Bearer " + accessToken); return getResponse(connection); } catch (IOException e) { writeLog("Athlete: " + athleteId + " Error while handling GET request: " + e.toString()); } return ""; } /** * Reads the response from the connection and returns it as a String * * @param connection Connection to the site * @return Response as a String * @throws IOException in case of error with the stream * @throws ResponseCodeWrongException if no data was read because of http * problems */ static String getResponse(HttpsURLConnection connection) throws IOException, ResponseCodeWrongException { StringBuilder result = new StringBuilder(); int responseCode = connection.getResponseCode(); if( fifteenMinuteRequestCount == 0) //first request in a 15 Min Interval { firstRequestTimeInCurrent15MinInterval = System.currentTimeMillis(); } dailyRequestCount++; fifteenMinuteRequestCount++; if (responseCode != HttpURLConnection.HTTP_OK) { // excluded error messages appearing on missing streams and reached rate limit if (responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode != httpCodeLimitReached) { writeLog("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream()); } throw new ResponseCodeWrongException(responseCode); } try (Reader reader = new BufferedReader( new InputStreamReader(connection.getInputStream(), Charset.forName(StandardCharsets.UTF_8.name())))) { int c = reader.read(); while (c != -1) { result.append((char) c); c = reader.read(); } } // Numbers too long for int or long are turned into Strings return result.toString().replaceAll("\\:\\s*([0-9]{15,})\\,", ":\"$1\","); } /** * Used for generating the accessToken * * @return The response as a String, an empty String in case of error. * @throws ResponseCodeWrongException If there was an http error */ static String makeOnePostRequest() throws ResponseCodeWrongException { HttpsURLConnection connection = null; try { // Create connection URL url = new URL(tokenUrl); connection = (HttpsURLConnection) url.openConnection(); connection.setRequestMethod("POST"); connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); Map params = new LinkedHashMap<>(); params.put("client_id", refreshInfo.getA()); params.put("client_secret", refreshInfo.getB()); params.put("grant_type", "refresh_token"); params.put("refresh_token", refreshInfo.getC()); StringBuilder postData = new StringBuilder(); for (String key : params.keySet()) { if (postData.length() != 0) { postData.append('&'); } postData.append(URLEncoder.encode(key, "UTF-8")); postData.append('='); postData.append(URLEncoder.encode(params.get(key), "UTF-8")); } byte[] postDataBytes = postData.toString().getBytes("UTF-8"); connection.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length)); connection.setDoOutput(true); connection.getOutputStream().write(postDataBytes); return getResponse(connection); } catch (IOException e) { writeLog("Athlete: " + athleteId + "Error while handling POST request: " + e.toString()); } return ""; } /** * Sends the old refresh token to strava and retrieves a new one and an access * token * * @param refreshInfo Refresh data with a the cliend_id, b the client_secret and * c the refresh_token * @return {@code true} if everything went right or {@code false} if there was * an error */ static boolean getAccessToken() { checkRequestLimit(); String json = null; int count = 0; do { try { json = makeOnePostRequest(); } catch (ResponseCodeWrongException e) { try { count = handleWrongResponseCode(count, e); } catch (NoAccessException e2) { return false; } if (count == -1) { return false; } } } while (json == null); try { Object obj = parser.parse(json); JSONObject data = (JSONObject) obj; accessToken = data.get("access_token").toString(); refreshInfo.setC(data.get("refresh_token").toString()); return true; } catch (ParseException e) { writeLog("Athlete " + athleteId + ": Error parsing refresh info."); } return false; } public static void main(String[] args) { // TODO: tokens need to be added by Enduco // Triplet a is client_id, b is client_secret, c is refresh_token List refreshTokens = new ArrayList<>(); // don't know if you need it but here you will find the new refresh tokens after // the requests List newRefreshTokens = new ArrayList<>(); Map allFiles = new HashMap<>(); int zipcount = 1; for (Triplet oneUser : refreshTokens) { refreshInfo = oneUser; athleteId++; if (!getAccessToken()) { writeLog("Couldn't get new access token for client " + athleteId); continue; } File athlete = oneAthlete(); if (athlete != null) { allFiles.put("Athlete_" + athleteId + ".json", athlete); } newRefreshTokens.add(refreshInfo); // pack zip-files of 10 athletes if (allFiles.size() >= 10) { try { zipFiles(allFiles, zipcount); zipcount++; allFiles = new HashMap<>(); } catch (IOException e) { writeLog("Files coulnd't be zipped"); } } } // zip the rest try { zipFiles(allFiles, zipcount); } catch (IOException e) { writeLog("Files coulnd't be zipped"); } } }