bs/Strava
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Reconstructed retry System an 15 Minute limitation

Browse Source 
Added more logs
This commit is contained in:
bs
2021-12-10 11:09:52 +01:00
parent 0859d47df0
commit 06b60a0533
1 changed files with 118 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
export/src/export/Main.java
+118 -94
View File
@@ -15,8 +15,10 @@ import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -44,28 +46,31 @@ public class Main
private static String testRequest;
private static File errorFile;
// TODO: enduco needs to insert the correct request limits here
private static final int requestLimit15Minutes = 100;
private static final int requestLimit15Minutes = 100 / 3;
private static final int requestLimitDay = 1000 / 3;
private static int simpleActivityId=0;
private static int dailyRequestCount = 0;
private static int waitTimeMil = 60000 * 3 * 15 / requestLimit15Minutes;
private static long lastRequestTimeInMillis = 0;
private static int fifteenMinuteRequestCount = 0;
private static long firstRequestTimeInCurrent15MinInterval = 0;
private static String accessToken = "";
/**
* a is client_id, b is client_secret, c is refresh_token
*/
private static Triplet refreshInfo;
private static void writeError(String text, InputStream errorStream)
private static void writeLog(String text, InputStream errorStream)
{
SimpleDateFormat format = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
if (errorFile == null)
{
errorFile = new File("error_" + System.currentTimeMillis() + ".txt");
errorFile = new File("log_" + System.currentTimeMillis() + ".txt");
}
try (BufferedWriter bout = new BufferedWriter(new FileWriter(errorFile, true)))
{
bout.write(text);
bout.write(format.format(new Date(System.currentTimeMillis()))+": "+text);
bout.newLine();
if (errorStream!= null)
{
@@ -87,9 +92,9 @@ public class Main
}
}
private static void writeError(String text)
private static void writeLog(String text)
{
writeError(text, null);
writeLog(text, null);
}
@@ -122,7 +127,7 @@ public class Main
}
catch (NoAccessException e1)
{
writeError("Athlete " + athleteId + ": Access expired and no new token possible");
writeLog("Athlete " + athleteId + ": Access expired and no new token possible");
return null; // no data at all. Stop right away
}
if (athleteInfo == null) // error getting General Information
@@ -133,7 +138,7 @@ public class Main
// for each activity: save streams
JSONArray allActivities = new JSONArray();
int simpleActivityId = 0;
simpleActivityId = 0;
for (String id : activities.keySet())
{
JSONObject data;
@@ -143,7 +148,7 @@ public class Main
}
catch (NoAccessException e)
{
writeError("Athlete " + athleteId + ": Access expired and no new token possible");
writeLog("Athlete " + athleteId + ": Access expired and no new token possible");
break; //stop the loop and save what you got up to there
}
data.put("activity_id", simpleActivityId);
@@ -164,7 +169,7 @@ public class Main
}
catch (IOException e)
{
writeError("Athlete " + athleteId + ": Error writing temp file: " + e.toString());
writeLog("Athlete " + athleteId + ": Error writing temp file: " + e.toString());
}
return null;
}
@@ -203,7 +208,7 @@ public class Main
}
catch (ParseException | NumberFormatException e)
{
writeError("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString());
writeLog("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString());
}
return data;
}
@@ -238,10 +243,8 @@ public class Main
{
String requestExtension = "athlete/activities?per_page=100&page=" + pageIndex;
String json = makeGetRequestWithRetry(requestExtension);
if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know
// where the
// last page
// is...
if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]"))
// don't know where the last page is...
{
break;
}
@@ -270,10 +273,11 @@ public class Main
}
catch (ParseException | NumberFormatException e)
{
writeError("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString());
writeLog("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString());
}
pageIndex++;
}
writeLog("Athlete " + athleteId + ": Found "+result.size()+"Activities");
return result;
}
@@ -301,7 +305,31 @@ public class Main
catch (InterruptedException e1)
{
}
//Write info what you did today
writeLog("Daily requests done. Current status: Athlete "+athleteId+", Activity "+simpleActivityId);
dailyRequestCount=0;
}
//check fifteen Minute limit
if (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval > 15*60*1000 )
{
//more than 15 Minutes have passed, so we have a new 15 minute limit
fifteenMinuteRequestCount =0;
}
else if(fifteenMinuteRequestCount >= requestLimit15Minutes)
{
//we are still in the old interval and reached the limit
long sleeptime = 15*60*1000 - (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval);
//sleep for the rest of the interval and start in a new interval
try
{
TimeUnit.MILLISECONDS.sleep(sleeptime);
fifteenMinuteRequestCount=0;
}
catch (InterruptedException e)
{
}
}
}
/**
@@ -323,45 +351,65 @@ public class Main
try
{
json = makeOneGetRequest(urlExtension);
dailyRequestCount++;
}
catch (ResponseCodeWrongException e)
{
// tried enough times, so stop now
if (count >= retryTimes)
count = handleWrongResponseCode(count, e);
if (count == -1)
{
writeError(
"Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode());
return null;
}
if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
{ //token might have expired
if(!getAccessToken()) //token doesn't work anymore and we can't get a new one
{
throw new NoAccessException();
}
}
else if (e.getResponseCode() == httpCodeLimitReached)
{// request limit is reached, try again later
count++;
}
else // some other error: try only one other time!
{
count = retryTimes;
}
// Sleep for 5 minutes and try to get the next 15 min request window
try
{
TimeUnit.MINUTES.sleep(5);
}
catch (InterruptedException e1)
{
}
}
} while (json == null);
return json;
}
/**
* Method processed the wrong response code and adjusts the retry times as needed:
* When the retry limit is reached, -1 is returned.
* When the access limit is reached, the retry count is increased.
* When an unknown error appears, the retry count is set to max to allow only one more retry.
* After that, the method sleeps for 5 Minutes.
* @param count Retry count
* @param e Exception containing the information
* @return the new retry count or -1 if there are no more tries left
* @throws NoAccessException if no AccessToken could be requested
*/
static int handleWrongResponseCode(int count, ResponseCodeWrongException e) throws NoAccessException
{
// tried enough times, so stop now
if (count >= retryTimes)
{
writeLog(
"Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode());
return -1;
}
if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
{ //token might have expired
if(!getAccessToken()) //token doesn't work anymore and we can't get a new one
{
throw new NoAccessException();
}
}
else if (e.getResponseCode() == httpCodeLimitReached)
{// request limit is reached, try again later
count++;
}
else // some other error: try only one other time!
{
count = retryTimes;
}
// Sleep for 5 minutes and try to get the next 15 min request window
try
{
TimeUnit.MINUTES.sleep(5);
}
catch (InterruptedException e1)
{
}
return count;
}
/**
* Extracts an athletes general information.
@@ -396,12 +444,12 @@ public class Main
}
catch (ParseException e)
{
writeError("Athlete " + athleteId + ": Error parsing general information.");
writeLog("Athlete " + athleteId + ": Error parsing general information.");
return null;
}
catch (NullPointerException e)
{
writeError("Athlete " + athleteId + ": No general information found.");
writeLog("Athlete " + athleteId + ": No general information found.");
return null;
}
}
@@ -454,19 +502,6 @@ public class Main
HttpsURLConnection connection = null;
try
{
long timeSinceLastRequest = System.currentTimeMillis() - lastRequestTimeInMillis;
if (timeSinceLastRequest < waitTimeMil)
{
try
{
TimeUnit.MILLISECONDS.sleep(waitTimeMil - timeSinceLastRequest);
}
catch (InterruptedException e)
{
}
}
;
lastRequestTimeInMillis = System.currentTimeMillis();
// Create connection
URL url = new URL(baseUrl + requestUrlExtension);
connection = (HttpsURLConnection) url.openConnection();
@@ -477,7 +512,7 @@ public class Main
}
catch (IOException e)
{
writeError("Athlete: " + athleteId + " Error while handling GET request: " + e.toString());
writeLog("Athlete: " + athleteId + " Error while handling GET request: " + e.toString());
}
return "";
}
@@ -495,12 +530,20 @@ public class Main
{
StringBuilder result = new StringBuilder();
int responseCode = connection.getResponseCode();
if( fifteenMinuteRequestCount == 0) //first request in a 15 Min Interval
{
firstRequestTimeInCurrent15MinInterval = System.currentTimeMillis();
}
dailyRequestCount++;
fifteenMinuteRequestCount++;
if (responseCode != HttpURLConnection.HTTP_OK)
{
// excluded error messages appearing on missing streams and reached rate limit
if (responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode != httpCodeLimitReached)
{
writeError("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream());
writeLog("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream());
}
throw new ResponseCodeWrongException(responseCode);
}
@@ -564,7 +607,7 @@ public class Main
}
catch (IOException e)
{
writeError("Athlete: " + athleteId + "Error while handling POST request: " + e.toString());
writeLog("Athlete: " + athleteId + "Error while handling POST request: " + e.toString());
}
return "";
}
@@ -588,39 +631,20 @@ public class Main
try
{
json = makeOnePostRequest();
dailyRequestCount++;
}
catch (ResponseCodeWrongException e)
{
// tried enough times, so stop now
if (count >= retryTimes)
try
{
count = handleWrongResponseCode(count, e);
}
catch (NoAccessException e2)
{
return false;
}
if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
{ //token might have expired
if(!getAccessToken()) //token doesn't work anymore and we can't get a new one
{
return false;
}
}
else if (e.getResponseCode() == httpCodeLimitReached)
{
// request limit is reached, try again later
count++;
}
else // some other error: try only one other time!
{
count = retryTimes;
}
// Sleep for 5 minutes and try to get the next 15 min request window
try
{
TimeUnit.MINUTES.sleep(5);
}
catch (InterruptedException e1)
if (count == -1)
{
return false;
}
}
@@ -635,7 +659,7 @@ public class Main
}
catch (ParseException e)
{
writeError("Athlete " + athleteId + ": Error parsing refresh info.");
writeLog("Athlete " + athleteId + ": Error parsing refresh info.");
}
return false;
}
@@ -658,7 +682,7 @@ public class Main
athleteId++;
if (!getAccessToken())
{
writeError("Couldn't get new access token for client " + athleteId);
writeLog("Couldn't get new access token for client " + athleteId);
continue;
}
@@ -680,7 +704,7 @@ public class Main
}
catch (IOException e)
{
writeError("Files coulnd't be zipped");
writeLog("Files coulnd't be zipped");
}
}
}
@@ -691,7 +715,7 @@ public class Main
}
catch (IOException e)
{
writeError("Files coulnd't be zipped");
writeLog("Files coulnd't be zipped");
}
}
}