From e582892b9fb1d480cf3668f306763df032b11f4b Mon Sep 17 00:00:00 2001 From: Bianca Steffes Date: Wed, 13 Oct 2021 16:04:28 +0200 Subject: [PATCH] Code now retries to send a request when there was a wrong response code (mostly 429, which is request limit reached). If there was another error, there will only be one retry --- export/src/export/Main.java | 160 ++++++++++++------ .../export/ResponseCodeWrongException.java | 24 +++ export/src/export/Triplet.java | 17 ++ export/src/export/Tuple.java | 23 +++ 4 files changed, 171 insertions(+), 53 deletions(-) create mode 100644 export/src/export/ResponseCodeWrongException.java create mode 100644 export/src/export/Triplet.java create mode 100644 export/src/export/Tuple.java diff --git a/export/src/export/Main.java b/export/src/export/Main.java index cd47ba8..569926b 100644 --- a/export/src/export/Main.java +++ b/export/src/export/Main.java @@ -34,6 +34,8 @@ public class Main { public static final String baseUrl = "https://www.strava.com/api/v3/"; public static final String tokenUrl = "https://www.strava.com/api/v3/oauth/token"; + private static final int retryTimes = 20; + private static final int httpCodeLimitReached = 429; private static int athleteId = 0; private static JSONParser parser = new JSONParser(); private static String testRequest; @@ -95,6 +97,10 @@ public class Main // get general information JSONObject athleteInfo = saveGeneralInformation(token); + if(athleteInfo==null) //error getting General Information + { + athleteInfo = new JSONObject(); + } athleteInfo.put("activities", allActivities); try @@ -127,9 +133,10 @@ public class Main String requestUrlExtension = "activities/" + id + "/streams?" + "keys=[time,distance,latlng,altitude,velocity_smooth,heartrate," + "cadence,watts,temp,moving,grade_smooth]&key_by_type=true"; - String json = makeOneGetRequest(requestUrlExtension, token); + + String json = makeGetRequestWithRetry(requestUrlExtension, token);; - if (json.isEmpty()||json.isBlank()||json.equals("")) + if (json==null ||json.isEmpty()||json.isBlank()||json.equals("")) { return data; } @@ -179,9 +186,9 @@ public class Main int pageIndex = 1; while (true) { - String requestExtension = "athlete/activities?page=" + pageIndex; - String json = makeOneGetRequest(requestExtension, token); - if (json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know where the last page is... + String requestExtension = "athlete/activities?per_page=500&page=" + pageIndex; + String json = makeGetRequestWithRetry(requestExtension, token); + if (json==null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know where the last page is... { break; } @@ -217,6 +224,53 @@ public class Main return result; } + /** + * Method is used to find the next request window: It tries the same request again after 5 minutes. + * After a set number of times (retryTimes) it stops if it still wasn't successful. + * @param urlExtension UrlExtension for the request + * @param token Token for the request + * @return Data as a String or {@code null} if there is no data + */ + static String makeGetRequestWithRetry(String urlExtension, String token) + { + String json=null; + int count=0; + do + { + try + { + json = makeOneGetRequest(urlExtension, token); + } + catch (ResponseCodeWrongException e) + { + //tried enough times, so stop now + if(count >= retryTimes) + { + return null; + } + //request limit is reached, try again later + if (e.getResponseCode()==httpCodeLimitReached) + { + count++; + } + else //some other error: try only one other time! + { + count=retryTimes; + } + //Sleep for 5 minutes and try to get the next 15 min request window + try + { + TimeUnit.MINUTES.sleep(5); + } + catch (InterruptedException e1) + { + } + } + + }while(json==null); + return json; + } + /** * Extracts an athletes general information. * @@ -232,7 +286,8 @@ public class Main String meas_pref = "measurement_preference"; // Possible values = feet, meters String weight = "weight"; - String json = makeOneGetRequest("athlete", token); + + String json = makeGetRequestWithRetry("athlete", token); JSONObject toSave = new JSONObject(); try @@ -291,8 +346,9 @@ public class Main * @param token Identification / authorization token of the * athlete * @return The response as a String, an empty String in case of error. + * @throws ResponseCodeWrongException If there was an http error */ - static String makeOneGetRequest(String requestUrlExtension, String token) + static String makeOneGetRequest(String requestUrlExtension, String token) throws ResponseCodeWrongException { if (testRequest != null) { @@ -335,20 +391,21 @@ public class Main * * @param connection Connection to the site * @return Response as a String - * @throws IOException in case of error + * @throws IOException in case of error with the stream + * @throws ResponseCodeWrongException if no data was read because of http problems */ - static String getResponse(HttpsURLConnection connection) throws IOException + static String getResponse(HttpsURLConnection connection) throws IOException, ResponseCodeWrongException { StringBuilder result = new StringBuilder(); int responseCode = connection.getResponseCode(); - if (responseCode != HttpURLConnection.HTTP_OK ) + if (responseCode != HttpURLConnection.HTTP_OK) { - //excluded error messages appearing on missing streams - if(responseCode != HttpURLConnection.HTTP_NOT_FOUND) + //excluded error messages appearing on missing streams and reached rate limit + if(responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode!= httpCodeLimitReached) { writeError("Wrong response code: " + responseCode); } - return ""; + throw new ResponseCodeWrongException(responseCode); } try (Reader reader = new BufferedReader( @@ -372,8 +429,9 @@ public class Main * @param data Triplet containing: a the client_id, b the client_secret, c the * refresh_token * @return The response as a String, an empty String in case of error. + * @throws ResponseCodeWrongException If there was an http error */ - static String makeOnePostRequest(Triplet data) + static String makeOnePostRequest(Triplet data) throws ResponseCodeWrongException { HttpsURLConnection connection = null; try @@ -423,7 +481,41 @@ public class Main */ static Tuple getAccessToken(Triplet refreshInfo) { - String json = makeOnePostRequest(refreshInfo); + String json=null; + int count =0; + do + { + try + { + json = makeOnePostRequest(refreshInfo); + } + catch (ResponseCodeWrongException e) + { + //tried enough times, so stop now + if(count >= retryTimes) + { + return null; + } + //request limit is reached, try again later + if (e.getResponseCode()==httpCodeLimitReached) + { + count++; + } + else //some other error: try only one other time! + { + count=retryTimes; + } + //Sleep for 5 minutes and try to get the next 15 min request window + try + { + TimeUnit.MINUTES.sleep(5); + } + catch (InterruptedException e1) + { + } + } + + }while(json==null); try { Object obj = parser.parse(json); @@ -475,42 +567,4 @@ public class Main writeError("Files coulnd't be zipped"); } } -} - -class Tuple -{ - private String a; - private String b; - - public Tuple(String a, String b) - { - this.a = a; - this.b = b; - } - - public String getA() - { - return a; - } - - public String getB() - { - return b; - } -} - -class Triplet extends Tuple -{ - private String c; - - public Triplet(String a, String b, String c) - { - super(a, b); - this.c = c; - } - - public String getC() - { - return c; - } } \ No newline at end of file diff --git a/export/src/export/ResponseCodeWrongException.java b/export/src/export/ResponseCodeWrongException.java new file mode 100644 index 0000000..eb2c493 --- /dev/null +++ b/export/src/export/ResponseCodeWrongException.java @@ -0,0 +1,24 @@ +package export; + +public class ResponseCodeWrongException extends Exception +{ + private static final long serialVersionUID = -1593242809354614946L; + private int responseCode; + + public ResponseCodeWrongException(int responseCode) + { + this.responseCode=responseCode; + } + + public int getResponseCode() + { + return responseCode; + } + + public void setResponseCode(int responseCode) + { + this.responseCode = responseCode; + } + + +} diff --git a/export/src/export/Triplet.java b/export/src/export/Triplet.java new file mode 100644 index 0000000..7bc837b --- /dev/null +++ b/export/src/export/Triplet.java @@ -0,0 +1,17 @@ +package export; + +public class Triplet extends Tuple +{ + private String c; + + public Triplet(String a, String b, String c) + { + super(a, b); + this.c = c; + } + + public String getC() + { + return c; + } +} diff --git a/export/src/export/Tuple.java b/export/src/export/Tuple.java new file mode 100644 index 0000000..bdbf87b --- /dev/null +++ b/export/src/export/Tuple.java @@ -0,0 +1,23 @@ +package export; + +public class Tuple +{ + private String a; + private String b; + + public Tuple(String a, String b) + { + this.a = a; + this.b = b; + } + + public String getA() + { + return a; + } + + public String getB() + { + return b; + } +}