Browse Source

Code now retries to send a request when there was a wrong response code (mostly 429, which is request limit reached). If there was another error, there will only be one retry

master
Bianca Steffes 2 years ago
parent
commit
e582892b9f
  1. 160
      export/src/export/Main.java
  2. 24
      export/src/export/ResponseCodeWrongException.java
  3. 17
      export/src/export/Triplet.java
  4. 23
      export/src/export/Tuple.java

160
export/src/export/Main.java

@ -34,6 +34,8 @@ public class Main
{
public static final String baseUrl = "https://www.strava.com/api/v3/";
public static final String tokenUrl = "https://www.strava.com/api/v3/oauth/token";
private static final int retryTimes = 20;
private static final int httpCodeLimitReached = 429;
private static int athleteId = 0;
private static JSONParser parser = new JSONParser();
private static String testRequest;
@ -95,6 +97,10 @@ public class Main
// get general information
JSONObject athleteInfo = saveGeneralInformation(token);
if(athleteInfo==null) //error getting General Information
{
athleteInfo = new JSONObject();
}
athleteInfo.put("activities", allActivities);
try
@ -127,9 +133,10 @@ public class Main
String requestUrlExtension = "activities/" + id + "/streams?"
+ "keys=[time,distance,latlng,altitude,velocity_smooth,heartrate,"
+ "cadence,watts,temp,moving,grade_smooth]&key_by_type=true";
String json = makeOneGetRequest(requestUrlExtension, token);
String json = makeGetRequestWithRetry(requestUrlExtension, token);;
if (json.isEmpty()||json.isBlank()||json.equals(""))
if (json==null ||json.isEmpty()||json.isBlank()||json.equals(""))
{
return data;
}
@ -179,9 +186,9 @@ public class Main
int pageIndex = 1;
while (true)
{
String requestExtension = "athlete/activities?page=" + pageIndex;
String json = makeOneGetRequest(requestExtension, token);
if (json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know where the last page is...
String requestExtension = "athlete/activities?per_page=500&page=" + pageIndex;
String json = makeGetRequestWithRetry(requestExtension, token);
if (json==null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know where the last page is...
{
break;
}
@ -217,6 +224,53 @@ public class Main
return result;
}
/**
* Method is used to find the next request window: It tries the same request again after 5 minutes.
* After a set number of times (retryTimes) it stops if it still wasn't successful.
* @param urlExtension UrlExtension for the request
* @param token Token for the request
* @return Data as a String or {@code null} if there is no data
*/
static String makeGetRequestWithRetry(String urlExtension, String token)
{
String json=null;
int count=0;
do
{
try
{
json = makeOneGetRequest(urlExtension, token);
}
catch (ResponseCodeWrongException e)
{
//tried enough times, so stop now
if(count >= retryTimes)
{
return null;
}
//request limit is reached, try again later
if (e.getResponseCode()==httpCodeLimitReached)
{
count++;
}
else //some other error: try only one other time!
{
count=retryTimes;
}
//Sleep for 5 minutes and try to get the next 15 min request window
try
{
TimeUnit.MINUTES.sleep(5);
}
catch (InterruptedException e1)
{
}
}
}while(json==null);
return json;
}
/**
* Extracts an athletes general information.
*
@ -232,7 +286,8 @@ public class Main
String meas_pref = "measurement_preference"; // Possible values = feet, meters
String weight = "weight";
String json = makeOneGetRequest("athlete", token);
String json = makeGetRequestWithRetry("athlete", token);
JSONObject toSave = new JSONObject();
try
@ -291,8 +346,9 @@ public class Main
* @param token Identification / authorization token of the
* athlete
* @return The response as a String, an empty String in case of error.
* @throws ResponseCodeWrongException If there was an http error
*/
static String makeOneGetRequest(String requestUrlExtension, String token)
static String makeOneGetRequest(String requestUrlExtension, String token) throws ResponseCodeWrongException
{
if (testRequest != null)
{
@ -335,20 +391,21 @@ public class Main
*
* @param connection Connection to the site
* @return Response as a String
* @throws IOException in case of error
* @throws IOException in case of error with the stream
* @throws ResponseCodeWrongException if no data was read because of http problems
*/
static String getResponse(HttpsURLConnection connection) throws IOException
static String getResponse(HttpsURLConnection connection) throws IOException, ResponseCodeWrongException
{
StringBuilder result = new StringBuilder();
int responseCode = connection.getResponseCode();
if (responseCode != HttpURLConnection.HTTP_OK )
if (responseCode != HttpURLConnection.HTTP_OK)
{
//excluded error messages appearing on missing streams
if(responseCode != HttpURLConnection.HTTP_NOT_FOUND)
//excluded error messages appearing on missing streams and reached rate limit
if(responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode!= httpCodeLimitReached)
{
writeError("Wrong response code: " + responseCode);
}
return "";
throw new ResponseCodeWrongException(responseCode);
}
try (Reader reader = new BufferedReader(
@ -372,8 +429,9 @@ public class Main
* @param data Triplet containing: a the client_id, b the client_secret, c the
* refresh_token
* @return The response as a String, an empty String in case of error.
* @throws ResponseCodeWrongException If there was an http error
*/
static String makeOnePostRequest(Triplet data)
static String makeOnePostRequest(Triplet data) throws ResponseCodeWrongException
{
HttpsURLConnection connection = null;
try
@ -423,7 +481,41 @@ public class Main
*/
static Tuple getAccessToken(Triplet refreshInfo)
{
String json = makeOnePostRequest(refreshInfo);
String json=null;
int count =0;
do
{
try
{
json = makeOnePostRequest(refreshInfo);
}
catch (ResponseCodeWrongException e)
{
//tried enough times, so stop now
if(count >= retryTimes)
{
return null;
}
//request limit is reached, try again later
if (e.getResponseCode()==httpCodeLimitReached)
{
count++;
}
else //some other error: try only one other time!
{
count=retryTimes;
}
//Sleep for 5 minutes and try to get the next 15 min request window
try
{
TimeUnit.MINUTES.sleep(5);
}
catch (InterruptedException e1)
{
}
}
}while(json==null);
try
{
Object obj = parser.parse(json);
@ -475,42 +567,4 @@ public class Main
writeError("Files coulnd't be zipped");
}
}
}
class Tuple
{
private String a;
private String b;
public Tuple(String a, String b)
{
this.a = a;
this.b = b;
}
public String getA()
{
return a;
}
public String getB()
{
return b;
}
}
class Triplet extends Tuple
{
private String c;
public Triplet(String a, String b, String c)
{
super(a, b);
this.c = c;
}
public String getC()
{
return c;
}
}

24
export/src/export/ResponseCodeWrongException.java

@ -0,0 +1,24 @@
package export;
public class ResponseCodeWrongException extends Exception
{
private static final long serialVersionUID = -1593242809354614946L;
private int responseCode;
public ResponseCodeWrongException(int responseCode)
{
this.responseCode=responseCode;
}
public int getResponseCode()
{
return responseCode;
}
public void setResponseCode(int responseCode)
{
this.responseCode = responseCode;
}
}

17
export/src/export/Triplet.java

@ -0,0 +1,17 @@
package export;
public class Triplet extends Tuple
{
private String c;
public Triplet(String a, String b, String c)
{
super(a, b);
this.c = c;
}
public String getC()
{
return c;
}
}

23
export/src/export/Tuple.java

@ -0,0 +1,23 @@
package export;
public class Tuple
{
private String a;
private String b;
public Tuple(String a, String b)
{
this.a = a;
this.b = b;
}
public String getA()
{
return a;
}
public String getB()
{
return b;
}
}
Loading…
Cancel
Save