bs
/
Strava
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Reconstructed retry System an 15 Minute limitation 

Added more logs
master
Bianca Steffes 3 years ago
parent
0859d47df0
commit
06b60a0533
1 changed files with 117 additions and 93 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 210
      export/src/export/Main.java

210
export/src/export/Main.java
View File

@ -15,8 +15,10 @@ import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.LinkedHashMap;
@ -44,28 +46,31 @@ public class Main
private static String testRequest;
private static File errorFile;
// TODO: enduco needs to insert the correct request limits here
private static final int requestLimit15Minutes = 100;
private static final int requestLimit15Minutes = 100 / 3;
private static final int requestLimitDay = 1000 / 3;
private static int simpleActivityId=0;
private static int dailyRequestCount = 0;
private static int waitTimeMil = 60000 * 3 * 15 / requestLimit15Minutes;
private static long lastRequestTimeInMillis = 0;
private static int fifteenMinuteRequestCount = 0;
private static long firstRequestTimeInCurrent15MinInterval = 0;
private static String accessToken = "";
/**
* a is client_id, b is client_secret, c is refresh_token
*/
private static Triplet refreshInfo;
private static void writeError(String text, InputStream errorStream)
private static void writeLog(String text, InputStream errorStream)
{
SimpleDateFormat format = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
if (errorFile == null)
{
errorFile = new File("error_" + System.currentTimeMillis() + ".txt");
errorFile = new File("log_" + System.currentTimeMillis() + ".txt");
}
try (BufferedWriter bout = new BufferedWriter(new FileWriter(errorFile, true)))
{
bout.write(text);
bout.write(format.format(new Date(System.currentTimeMillis()))+": "+text);
bout.newLine();
if (errorStream!= null)
{
@ -87,9 +92,9 @@ public class Main
}
}
private static void writeError(String text)
private static void writeLog(String text)
{
writeError(text, null);
writeLog(text, null);
}
@ -122,7 +127,7 @@ public class Main
}
catch (NoAccessException e1)
{
writeError("Athlete " + athleteId + ": Access expired and no new token possible");
writeLog("Athlete " + athleteId + ": Access expired and no new token possible");
return null; // no data at all. Stop right away
}
if (athleteInfo == null) // error getting General Information
@ -133,7 +138,7 @@ public class Main
// for each activity: save streams
JSONArray allActivities = new JSONArray();
int simpleActivityId = 0;
simpleActivityId = 0;
for (String id : activities.keySet())
{
JSONObject data;
@ -143,7 +148,7 @@ public class Main
}
catch (NoAccessException e)
{
writeError("Athlete " + athleteId + ": Access expired and no new token possible");
writeLog("Athlete " + athleteId + ": Access expired and no new token possible");
break; //stop the loop and save what you got up to there
}
data.put("activity_id", simpleActivityId);
@ -164,7 +169,7 @@ public class Main
}
catch (IOException e)
{
writeError("Athlete " + athleteId + ": Error writing temp file: " + e.toString());
writeLog("Athlete " + athleteId + ": Error writing temp file: " + e.toString());
}
return null;
}
@ -203,7 +208,7 @@ public class Main
}
catch (ParseException | NumberFormatException e)
{
writeError("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString());
writeLog("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString());
}
return data;
}
@ -238,10 +243,8 @@ public class Main
{
String requestExtension = "athlete/activities?per_page=100&page=" + pageIndex;
String json = makeGetRequestWithRetry(requestExtension);
if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know
// where the
// last page
// is...
if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]"))
// don't know where the last page is...
{
break;
}
@ -270,10 +273,11 @@ public class Main
}
catch (ParseException | NumberFormatException e)
{
writeError("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString());
writeLog("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString());
}
pageIndex++;
}
writeLog("Athlete " + athleteId + ": Found "+result.size()+"Activities");
return result;
}
@ -301,6 +305,30 @@ public class Main
catch (InterruptedException e1)
{
}
//Write info what you did today
writeLog("Daily requests done. Current status: Athlete "+athleteId+", Activity "+simpleActivityId);
dailyRequestCount=0;
}
//check fifteen Minute limit
if (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval > 15*60*1000 )
{
//more than 15 Minutes have passed, so we have a new 15 minute limit
fifteenMinuteRequestCount =0;
}
else if(fifteenMinuteRequestCount >= requestLimit15Minutes)
{
//we are still in the old interval and reached the limit
long sleeptime = 15*60*1000 - (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval);
//sleep for the rest of the interval and start in a new interval
try
{
TimeUnit.MILLISECONDS.sleep(sleeptime);
fifteenMinuteRequestCount=0;
}
catch (InterruptedException e)
{
}
}
}
@ -323,46 +351,66 @@ public class Main
try
{
json = makeOneGetRequest(urlExtension);
dailyRequestCount++;
}
catch (ResponseCodeWrongException e)
{
// tried enough times, so stop now
if (count >= retryTimes)
count = handleWrongResponseCode(count, e);
if (count == -1)
{
writeError(
"Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode());
return null;
}
if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
{ //token might have expired
if(!getAccessToken()) //token doesn't work anymore and we can't get a new one
{
throw new NoAccessException();
}
}
else if (e.getResponseCode() == httpCodeLimitReached)
{// request limit is reached, try again later
count++;
}
else // some other error: try only one other time!
{
count = retryTimes;
}
// Sleep for 5 minutes and try to get the next 15 min request window
try
{
TimeUnit.MINUTES.sleep(5);
}
catch (InterruptedException e1)
{
}
}
} while (json == null);
return json;
}
/**
* Method processed the wrong response code and adjusts the retry times as needed:
* When the retry limit is reached, -1 is returned.
* When the access limit is reached, the retry count is increased.
* When an unknown error appears, the retry count is set to max to allow only one more retry.
* After that, the method sleeps for 5 Minutes.
* @param count Retry count
* @param e Exception containing the information
* @return the new retry count or -1 if there are no more tries left
* @throws NoAccessException if no AccessToken could be requested
*/
static int handleWrongResponseCode(int count, ResponseCodeWrongException e) throws NoAccessException
{
// tried enough times, so stop now
if (count >= retryTimes)
{
writeLog(
"Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode());
return -1;
}
if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
{ //token might have expired
if(!getAccessToken()) //token doesn't work anymore and we can't get a new one
{
throw new NoAccessException();
}
}
else if (e.getResponseCode() == httpCodeLimitReached)
{// request limit is reached, try again later
count++;
}
else // some other error: try only one other time!
{
count = retryTimes;
}
// Sleep for 5 minutes and try to get the next 15 min request window
try
{
TimeUnit.MINUTES.sleep(5);
}
catch (InterruptedException e1)
{
}
return count;
}
/**
* Extracts an athletes general information.
*
@ -396,12 +444,12 @@ public class Main
}
catch (ParseException e)
{
writeError("Athlete " + athleteId + ": Error parsing general information.");
writeLog("Athlete " + athleteId + ": Error parsing general information.");
return null;
}
catch (NullPointerException e)
{
writeError("Athlete " + athleteId + ": No general information found.");
writeLog("Athlete " + athleteId + ": No general information found.");
return null;
}
}
@ -454,19 +502,6 @@ public class Main
HttpsURLConnection connection = null;
try
{
long timeSinceLastRequest = System.currentTimeMillis() - lastRequestTimeInMillis;
if (timeSinceLastRequest < waitTimeMil)
{
try
{
TimeUnit.MILLISECONDS.sleep(waitTimeMil - timeSinceLastRequest);
}
catch (InterruptedException e)
{
}
}
;
lastRequestTimeInMillis = System.currentTimeMillis();
// Create connection
URL url = new URL(baseUrl + requestUrlExtension);
connection = (HttpsURLConnection) url.openConnection();
@ -477,7 +512,7 @@ public class Main
}
catch (IOException e)
{
writeError("Athlete: " + athleteId + " Error while handling GET request: " + e.toString());
writeLog("Athlete: " + athleteId + " Error while handling GET request: " + e.toString());
}
return "";
}
@ -495,12 +530,20 @@ public class Main
{
StringBuilder result = new StringBuilder();
int responseCode = connection.getResponseCode();
if( fifteenMinuteRequestCount == 0) //first request in a 15 Min Interval
{
firstRequestTimeInCurrent15MinInterval = System.currentTimeMillis();
}
dailyRequestCount++;
fifteenMinuteRequestCount++;
if (responseCode != HttpURLConnection.HTTP_OK)
{
// excluded error messages appearing on missing streams and reached rate limit
if (responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode != httpCodeLimitReached)
{
writeError("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream());
writeLog("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream());
}
throw new ResponseCodeWrongException(responseCode);
}
@ -564,7 +607,7 @@ public class Main
}
catch (IOException e)
{
writeError("Athlete: " + athleteId + "Error while handling POST request: " + e.toString());
writeLog("Athlete: " + athleteId + "Error while handling POST request: " + e.toString());
}
return "";
}
@ -588,39 +631,20 @@ public class Main
try
{
json = makeOnePostRequest();
dailyRequestCount++;
}
catch (ResponseCodeWrongException e)
{
// tried enough times, so stop now
if (count >= retryTimes)
{
return false;
}
if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
{ //token might have expired
if(!getAccessToken()) //token doesn't work anymore and we can't get a new one
{
return false;
}
}
else if (e.getResponseCode() == httpCodeLimitReached)
{
// request limit is reached, try again later
count++;
}
else // some other error: try only one other time!
try
{
count = retryTimes;
count = handleWrongResponseCode(count, e);
}
// Sleep for 5 minutes and try to get the next 15 min request window
try
catch (NoAccessException e2)
{
TimeUnit.MINUTES.sleep(5);
return false;
}
catch (InterruptedException e1)
if (count == -1)
{
return false;
}
}
@ -635,7 +659,7 @@ public class Main
}
catch (ParseException e)
{
writeError("Athlete " + athleteId + ": Error parsing refresh info.");
writeLog("Athlete " + athleteId + ": Error parsing refresh info.");
}
return false;
}
@ -658,7 +682,7 @@ public class Main
athleteId++;
if (!getAccessToken())
{
writeError("Couldn't get new access token for client " + athleteId);
writeLog("Couldn't get new access token for client " + athleteId);
continue;
}
@ -680,7 +704,7 @@ public class Main
}
catch (IOException e)
{
writeError("Files coulnd't be zipped");
writeLog("Files coulnd't be zipped");
}
}
}
@ -691,7 +715,7 @@ public class Main
}
catch (IOException e)
{
writeError("Files coulnd't be zipped");
writeLog("Files coulnd't be zipped");
}
}
}
Write Preview
Loading…
Cancel
Save