From 06b60a05334b401487ad5b3ce11e4fbe0d46f94e Mon Sep 17 00:00:00 2001 From: Bianca Steffes Date: Fri, 10 Dec 2021 11:09:52 +0100 Subject: [PATCH] Reconstructed retry System an 15 Minute limitation Added more logs --- export/src/export/Main.java | 210 ++++++++++++++++++++---------------- 1 file changed, 117 insertions(+), 93 deletions(-) diff --git a/export/src/export/Main.java b/export/src/export/Main.java index 928ac76..eb7bfd0 100644 --- a/export/src/export/Main.java +++ b/export/src/export/Main.java @@ -15,8 +15,10 @@ import java.net.URL; import java.net.URLEncoder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; +import java.util.Date; import java.util.GregorianCalendar; import java.util.HashMap; import java.util.LinkedHashMap; @@ -44,28 +46,31 @@ public class Main private static String testRequest; private static File errorFile; // TODO: enduco needs to insert the correct request limits here - private static final int requestLimit15Minutes = 100; + private static final int requestLimit15Minutes = 100 / 3; private static final int requestLimitDay = 1000 / 3; + private static int simpleActivityId=0; private static int dailyRequestCount = 0; - private static int waitTimeMil = 60000 * 3 * 15 / requestLimit15Minutes; - private static long lastRequestTimeInMillis = 0; + private static int fifteenMinuteRequestCount = 0; + private static long firstRequestTimeInCurrent15MinInterval = 0; private static String accessToken = ""; /** * a is client_id, b is client_secret, c is refresh_token */ private static Triplet refreshInfo; - private static void writeError(String text, InputStream errorStream) + private static void writeLog(String text, InputStream errorStream) { + SimpleDateFormat format = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss"); + if (errorFile == null) { - errorFile = new File("error_" + System.currentTimeMillis() + ".txt"); + errorFile = new File("log_" + System.currentTimeMillis() + ".txt"); } try (BufferedWriter bout = new BufferedWriter(new FileWriter(errorFile, true))) { - bout.write(text); + bout.write(format.format(new Date(System.currentTimeMillis()))+": "+text); bout.newLine(); if (errorStream!= null) { @@ -87,9 +92,9 @@ public class Main } } - private static void writeError(String text) + private static void writeLog(String text) { - writeError(text, null); + writeLog(text, null); } @@ -122,7 +127,7 @@ public class Main } catch (NoAccessException e1) { - writeError("Athlete " + athleteId + ": Access expired and no new token possible"); + writeLog("Athlete " + athleteId + ": Access expired and no new token possible"); return null; // no data at all. Stop right away } if (athleteInfo == null) // error getting General Information @@ -133,7 +138,7 @@ public class Main // for each activity: save streams JSONArray allActivities = new JSONArray(); - int simpleActivityId = 0; + simpleActivityId = 0; for (String id : activities.keySet()) { JSONObject data; @@ -143,7 +148,7 @@ public class Main } catch (NoAccessException e) { - writeError("Athlete " + athleteId + ": Access expired and no new token possible"); + writeLog("Athlete " + athleteId + ": Access expired and no new token possible"); break; //stop the loop and save what you got up to there } data.put("activity_id", simpleActivityId); @@ -164,7 +169,7 @@ public class Main } catch (IOException e) { - writeError("Athlete " + athleteId + ": Error writing temp file: " + e.toString()); + writeLog("Athlete " + athleteId + ": Error writing temp file: " + e.toString()); } return null; } @@ -203,7 +208,7 @@ public class Main } catch (ParseException | NumberFormatException e) { - writeError("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString()); + writeLog("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString()); } return data; } @@ -238,10 +243,8 @@ public class Main { String requestExtension = "athlete/activities?per_page=100&page=" + pageIndex; String json = makeGetRequestWithRetry(requestExtension); - if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know - // where the - // last page - // is... + if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) + // don't know where the last page is... { break; } @@ -270,10 +273,11 @@ public class Main } catch (ParseException | NumberFormatException e) { - writeError("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString()); + writeLog("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString()); } pageIndex++; } + writeLog("Athlete " + athleteId + ": Found "+result.size()+"Activities"); return result; } @@ -301,7 +305,31 @@ public class Main catch (InterruptedException e1) { } + //Write info what you did today + writeLog("Daily requests done. Current status: Athlete "+athleteId+", Activity "+simpleActivityId); + dailyRequestCount=0; + } + + //check fifteen Minute limit + if (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval > 15*60*1000 ) + { + //more than 15 Minutes have passed, so we have a new 15 minute limit + fifteenMinuteRequestCount =0; } + else if(fifteenMinuteRequestCount >= requestLimit15Minutes) + { + //we are still in the old interval and reached the limit + long sleeptime = 15*60*1000 - (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval); + //sleep for the rest of the interval and start in a new interval + try + { + TimeUnit.MILLISECONDS.sleep(sleeptime); + fifteenMinuteRequestCount=0; + } + catch (InterruptedException e) + { + } + } } /** @@ -323,45 +351,65 @@ public class Main try { json = makeOneGetRequest(urlExtension); - dailyRequestCount++; } catch (ResponseCodeWrongException e) { - // tried enough times, so stop now - if (count >= retryTimes) + count = handleWrongResponseCode(count, e); + if (count == -1) { - writeError( - "Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode()); return null; } - if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED) - { //token might have expired - if(!getAccessToken()) //token doesn't work anymore and we can't get a new one - { - throw new NoAccessException(); - } - } - else if (e.getResponseCode() == httpCodeLimitReached) - {// request limit is reached, try again later - count++; - } - else // some other error: try only one other time! - { - count = retryTimes; - } - // Sleep for 5 minutes and try to get the next 15 min request window - try - { - TimeUnit.MINUTES.sleep(5); - } - catch (InterruptedException e1) - { - } } } while (json == null); return json; } + + /** + * Method processed the wrong response code and adjusts the retry times as needed: + * When the retry limit is reached, -1 is returned. + * When the access limit is reached, the retry count is increased. + * When an unknown error appears, the retry count is set to max to allow only one more retry. + * After that, the method sleeps for 5 Minutes. + * @param count Retry count + * @param e Exception containing the information + * @return the new retry count or -1 if there are no more tries left + * @throws NoAccessException if no AccessToken could be requested + */ + static int handleWrongResponseCode(int count, ResponseCodeWrongException e) throws NoAccessException + { + // tried enough times, so stop now + if (count >= retryTimes) + { + writeLog( + "Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode()); + return -1; + } + if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED) + { //token might have expired + if(!getAccessToken()) //token doesn't work anymore and we can't get a new one + { + throw new NoAccessException(); + } + } + else if (e.getResponseCode() == httpCodeLimitReached) + {// request limit is reached, try again later + count++; + } + else // some other error: try only one other time! + { + count = retryTimes; + } + // Sleep for 5 minutes and try to get the next 15 min request window + try + { + TimeUnit.MINUTES.sleep(5); + } + catch (InterruptedException e1) + { + } + return count; + } /** * Extracts an athletes general information. @@ -396,12 +444,12 @@ public class Main } catch (ParseException e) { - writeError("Athlete " + athleteId + ": Error parsing general information."); + writeLog("Athlete " + athleteId + ": Error parsing general information."); return null; } catch (NullPointerException e) { - writeError("Athlete " + athleteId + ": No general information found."); + writeLog("Athlete " + athleteId + ": No general information found."); return null; } } @@ -454,19 +502,6 @@ public class Main HttpsURLConnection connection = null; try { - long timeSinceLastRequest = System.currentTimeMillis() - lastRequestTimeInMillis; - if (timeSinceLastRequest < waitTimeMil) - { - try - { - TimeUnit.MILLISECONDS.sleep(waitTimeMil - timeSinceLastRequest); - } - catch (InterruptedException e) - { - } - } - ; - lastRequestTimeInMillis = System.currentTimeMillis(); // Create connection URL url = new URL(baseUrl + requestUrlExtension); connection = (HttpsURLConnection) url.openConnection(); @@ -477,7 +512,7 @@ public class Main } catch (IOException e) { - writeError("Athlete: " + athleteId + " Error while handling GET request: " + e.toString()); + writeLog("Athlete: " + athleteId + " Error while handling GET request: " + e.toString()); } return ""; } @@ -495,12 +530,20 @@ public class Main { StringBuilder result = new StringBuilder(); int responseCode = connection.getResponseCode(); + + if( fifteenMinuteRequestCount == 0) //first request in a 15 Min Interval + { + firstRequestTimeInCurrent15MinInterval = System.currentTimeMillis(); + } + dailyRequestCount++; + fifteenMinuteRequestCount++; + if (responseCode != HttpURLConnection.HTTP_OK) { // excluded error messages appearing on missing streams and reached rate limit if (responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode != httpCodeLimitReached) { - writeError("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream()); + writeLog("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream()); } throw new ResponseCodeWrongException(responseCode); } @@ -564,7 +607,7 @@ public class Main } catch (IOException e) { - writeError("Athlete: " + athleteId + "Error while handling POST request: " + e.toString()); + writeLog("Athlete: " + athleteId + "Error while handling POST request: " + e.toString()); } return ""; } @@ -588,39 +631,20 @@ public class Main try { json = makeOnePostRequest(); - dailyRequestCount++; } catch (ResponseCodeWrongException e) { - // tried enough times, so stop now - if (count >= retryTimes) - { - return false; - } - - if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED) - { //token might have expired - if(!getAccessToken()) //token doesn't work anymore and we can't get a new one - { - return false; - } - } - else if (e.getResponseCode() == httpCodeLimitReached) + try { - // request limit is reached, try again later - count++; + count = handleWrongResponseCode(count, e); } - else // some other error: try only one other time! + catch (NoAccessException e2) { - count = retryTimes; - } - // Sleep for 5 minutes and try to get the next 15 min request window - try - { - TimeUnit.MINUTES.sleep(5); + return false; } - catch (InterruptedException e1) + if (count == -1) { + return false; } } @@ -635,7 +659,7 @@ public class Main } catch (ParseException e) { - writeError("Athlete " + athleteId + ": Error parsing refresh info."); + writeLog("Athlete " + athleteId + ": Error parsing refresh info."); } return false; } @@ -658,7 +682,7 @@ public class Main athleteId++; if (!getAccessToken()) { - writeError("Couldn't get new access token for client " + athleteId); + writeLog("Couldn't get new access token for client " + athleteId); continue; } @@ -680,7 +704,7 @@ public class Main } catch (IOException e) { - writeError("Files coulnd't be zipped"); + writeLog("Files coulnd't be zipped"); } } } @@ -691,7 +715,7 @@ public class Main } catch (IOException e) { - writeError("Files coulnd't be zipped"); + writeLog("Files coulnd't be zipped"); } } } \ No newline at end of file