bs/Strava
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Code now retries to send a request when there was a wrong response code (mostly 429, which is request limit reached). If there was another error, there will only be one retry

Browse Source
This commit is contained in:
bs
2021-10-13 16:04:28 +02:00
parent ae6e61dcd6
commit e582892b9f
4 changed files with 171 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
export/src/export/Main.java
+107 -53
View File
@@ -34,6 +34,8 @@ public class Main
{
public static final String baseUrl = "https://www.strava.com/api/v3/";
public static final String tokenUrl = "https://www.strava.com/api/v3/oauth/token";
private static final int retryTimes = 20;
private static final int httpCodeLimitReached = 429;
private static int athleteId = 0;
private static JSONParser parser = new JSONParser();
private static String testRequest;
@@ -95,6 +97,10 @@ public class Main
// get general information
JSONObject athleteInfo = saveGeneralInformation(token);
if(athleteInfo==null) //error getting General Information
{
athleteInfo = new JSONObject();
}
athleteInfo.put("activities", allActivities);
try
@@ -127,9 +133,10 @@ public class Main
String requestUrlExtension = "activities/" + id + "/streams?"
+ "keys=[time,distance,latlng,altitude,velocity_smooth,heartrate,"
+ "cadence,watts,temp,moving,grade_smooth]&key_by_type=true";
String json = makeOneGetRequest(requestUrlExtension, token);
String json = makeGetRequestWithRetry(requestUrlExtension, token);;
if (json.isEmpty()||json.isBlank()||json.equals(""))
if (json==null ||json.isEmpty()||json.isBlank()||json.equals(""))
{
return data;
}
@@ -179,9 +186,9 @@ public class Main
int pageIndex = 1;
while (true)
{
String requestExtension = "athlete/activities?page=" + pageIndex;
String json = makeOneGetRequest(requestExtension, token);
if (json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know where the last page is...
String requestExtension = "athlete/activities?per_page=500&page=" + pageIndex;
String json = makeGetRequestWithRetry(requestExtension, token);
if (json==null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know where the last page is...
{
break;
}
@@ -217,6 +224,53 @@ public class Main
return result;
}
/**
* Method is used to find the next request window: It tries the same request again after 5 minutes.
* After a set number of times (retryTimes) it stops if it still wasn't successful.
* @param urlExtension UrlExtension for the request
* @param token Token for the request
* @return Data as a String or {@code null} if there is no data
*/
static String makeGetRequestWithRetry(String urlExtension, String token)
{
String json=null;
int count=0;
do
{
try
{
json = makeOneGetRequest(urlExtension, token);
}
catch (ResponseCodeWrongException e)
{
//tried enough times, so stop now
if(count >= retryTimes)
{
return null;
}
//request limit is reached, try again later
if (e.getResponseCode()==httpCodeLimitReached)
{
count++;
}
else //some other error: try only one other time!
{
count=retryTimes;
}
//Sleep for 5 minutes and try to get the next 15 min request window
try
{
TimeUnit.MINUTES.sleep(5);
}
catch (InterruptedException e1)
{
}
}
}while(json==null);
return json;
}
/**
* Extracts an athletes general information.
*
@@ -232,7 +286,8 @@ public class Main
String meas_pref = "measurement_preference"; // Possible values = feet, meters
String weight = "weight";
String json = makeOneGetRequest("athlete", token);
String json = makeGetRequestWithRetry("athlete", token);
JSONObject toSave = new JSONObject();
try
@@ -291,8 +346,9 @@ public class Main
* @param token Identification / authorization token of the
* athlete
* @return The response as a String, an empty String in case of error.
* @throws ResponseCodeWrongException If there was an http error
*/
static String makeOneGetRequest(String requestUrlExtension, String token)
static String makeOneGetRequest(String requestUrlExtension, String token) throws ResponseCodeWrongException
{
if (testRequest != null)
{
@@ -335,20 +391,21 @@ public class Main
*
* @param connection Connection to the site
* @return Response as a String
* @throws IOException in case of error
* @throws IOException in case of error with the stream
* @throws ResponseCodeWrongException if no data was read because of http problems
*/
static String getResponse(HttpsURLConnection connection) throws IOException
static String getResponse(HttpsURLConnection connection) throws IOException, ResponseCodeWrongException
{
StringBuilder result = new StringBuilder();
int responseCode = connection.getResponseCode();
if (responseCode != HttpURLConnection.HTTP_OK )
if (responseCode != HttpURLConnection.HTTP_OK)
{
//excluded error messages appearing on missing streams
if(responseCode != HttpURLConnection.HTTP_NOT_FOUND)
//excluded error messages appearing on missing streams and reached rate limit
if(responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode!= httpCodeLimitReached)
{
writeError("Wrong response code: " + responseCode);
}
return "";
throw new ResponseCodeWrongException(responseCode);
}
try (Reader reader = new BufferedReader(
@@ -372,8 +429,9 @@ public class Main
* @param data Triplet containing: a the client_id, b the client_secret, c the
* refresh_token
* @return The response as a String, an empty String in case of error.
* @throws ResponseCodeWrongException If there was an http error
*/
static String makeOnePostRequest(Triplet data)
static String makeOnePostRequest(Triplet data) throws ResponseCodeWrongException
{
HttpsURLConnection connection = null;
try
@@ -423,7 +481,41 @@ public class Main
*/
static Tuple getAccessToken(Triplet refreshInfo)
{
String json = makeOnePostRequest(refreshInfo);
String json=null;
int count =0;
do
{
try
{
json = makeOnePostRequest(refreshInfo);
}
catch (ResponseCodeWrongException e)
{
//tried enough times, so stop now
if(count >= retryTimes)
{
return null;
}
//request limit is reached, try again later
if (e.getResponseCode()==httpCodeLimitReached)
{
count++;
}
else //some other error: try only one other time!
{
count=retryTimes;
}
//Sleep for 5 minutes and try to get the next 15 min request window
try
{
TimeUnit.MINUTES.sleep(5);
}
catch (InterruptedException e1)
{
}
}
}while(json==null);
try
{
Object obj = parser.parse(json);
@@ -475,42 +567,4 @@ public class Main
writeError("Files coulnd't be zipped");
}
}
}
class Tuple
{
private String a;
private String b;
public Tuple(String a, String b)
{
this.a = a;
this.b = b;
}
public String getA()
{
return a;
}
public String getB()
{
return b;
}
}
class Triplet extends Tuple
{
private String c;
public Triplet(String a, String b, String c)
{
super(a, b);
this.c = c;
}
public String getC()
{
return c;
}
}
export/src/export/ResponseCodeWrongException.java
+24
View File
@@ -0,0 +1,24 @@
package export;
public class ResponseCodeWrongException extends Exception
{
private static final long serialVersionUID = -1593242809354614946L;
private int responseCode;
public ResponseCodeWrongException(int responseCode)
{
this.responseCode=responseCode;
}
public int getResponseCode()
{
return responseCode;
}
public void setResponseCode(int responseCode)
{
this.responseCode = responseCode;
}
}
export/src/export/Triplet.java
+17
View File
@@ -0,0 +1,17 @@
package export;
public class Triplet extends Tuple
{
private String c;
public Triplet(String a, String b, String c)
{
super(a, b);
this.c = c;
}
public String getC()
{
return c;
}
}
export/src/export/Tuple.java
+23
View File
@@ -0,0 +1,23 @@
package export;
public class Tuple
{
private String a;
private String b;
public Tuple(String a, String b)
{
this.a = a;
this.b = b;
}
public String getA()
{
return a;
}
public String getB()
{
return b;
}
}