Reconstructed retry System an 15 Minute limitation

Added more logs

export/src/export/Main.java

@@ -15,8 +15,10 @@ import java.net.URL;
 import java.net.URLEncoder;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -44,28 +46,31 @@ public class Main
 	private static String testRequest;
 	private static File errorFile;
 	// TODO: enduco needs to insert the correct request limits here
-	private static final int requestLimit15Minutes = 100;
+	private static final int requestLimit15Minutes = 100 / 3;
 	private static final int requestLimitDay = 1000 / 3;
 
+	private static int simpleActivityId=0;
 	private static int dailyRequestCount = 0;
-	private static int waitTimeMil = 60000 * 3 * 15 / requestLimit15Minutes;
-	private static long lastRequestTimeInMillis = 0;
+	private static int fifteenMinuteRequestCount = 0;
+	private static long firstRequestTimeInCurrent15MinInterval = 0;
 	private static String accessToken = "";
 	/**
 	 * a is client_id, b is client_secret, c is refresh_token
 	 */
 	private static Triplet refreshInfo;
 
-	private static void writeError(String text, InputStream errorStream)
+	private static void writeLog(String text, InputStream errorStream)
 	{
+		SimpleDateFormat format = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
+		
 		if (errorFile == null)
 		{
-			errorFile = new File("error_" + System.currentTimeMillis() + ".txt");
+			errorFile = new File("log_" + System.currentTimeMillis() + ".txt");
 		}
 
 		try (BufferedWriter bout = new BufferedWriter(new FileWriter(errorFile, true)))
 		{
-			bout.write(text);
+			bout.write(format.format(new Date(System.currentTimeMillis()))+": "+text);
 			bout.newLine();
 			if (errorStream!= null)
 			{
@@ -87,9 +92,9 @@ public class Main
 		}
 	}
 	
-	private static void writeError(String text)
+	private static void writeLog(String text)
 	{
-		writeError(text, null);
+		writeLog(text, null);
 	}
 	
 	
@@ -122,7 +127,7 @@ public class Main
 		}
 		catch (NoAccessException e1)
 		{
-			writeError("Athlete " + athleteId + ": Access expired and no new token possible");
+			writeLog("Athlete " + athleteId + ": Access expired and no new token possible");
 			return null; // no data at all. Stop right away
 		}
 		if (athleteInfo == null) // error getting General Information
@@ -133,7 +138,7 @@ public class Main
 
 		// for each activity: save streams
 		JSONArray allActivities = new JSONArray();
-		int simpleActivityId = 0;
+		simpleActivityId = 0;
 		for (String id : activities.keySet())
 		{
 			JSONObject data;
@@ -143,7 +148,7 @@ public class Main
 			}
 			catch (NoAccessException e)
 			{
-				writeError("Athlete " + athleteId + ": Access expired and no new token possible");
+				writeLog("Athlete " + athleteId + ": Access expired and no new token possible");
 				break; //stop the loop and save what you got up to there
 			}
 			data.put("activity_id", simpleActivityId);
@@ -164,7 +169,7 @@ public class Main
 		}
 		catch (IOException e)
 		{
-			writeError("Athlete " + athleteId + ": Error writing temp file: " + e.toString());
+			writeLog("Athlete " + athleteId + ": Error writing temp file: " + e.toString());
 		}
 		return null;
 	}
@@ -203,7 +208,7 @@ public class Main
 		}
 		catch (ParseException | NumberFormatException e)
 		{
-			writeError("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString());
+			writeLog("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString());
 		}
 		return data;
 	}
@@ -238,10 +243,8 @@ public class Main
 		{
 			String requestExtension = "athlete/activities?per_page=100&page=" + pageIndex;
 			String json = makeGetRequestWithRetry(requestExtension);
-			if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know
-																											// where the
-																											// last page
-																											// is...
+			if (json == null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) 
+			// don't know where the last page is...
 			{
 				break;
 			}
@@ -270,10 +273,11 @@ public class Main
 			}
 			catch (ParseException | NumberFormatException e)
 			{
-				writeError("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString());
+				writeLog("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString());
 			}
 			pageIndex++;
 		}
+		writeLog("Athlete " + athleteId + ": Found "+result.size()+"Activities");
 		return result;
 	}
 
@@ -301,7 +305,31 @@ public class Main
 			catch (InterruptedException e1)
 			{
 			}
+			//Write info what you did today
+			writeLog("Daily requests done. Current status: Athlete "+athleteId+", Activity "+simpleActivityId);
+			dailyRequestCount=0;
+		}
+		
+		//check fifteen Minute limit
+		if (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval > 15*60*1000 )
+		{
+			//more than 15 Minutes have passed, so we have a new 15 minute limit
+			fifteenMinuteRequestCount =0;
 		}
+		else if(fifteenMinuteRequestCount >= requestLimit15Minutes)	
+		{
+			//we are still in the old interval and reached the limit
+			long sleeptime = 15*60*1000 - (System.currentTimeMillis() - firstRequestTimeInCurrent15MinInterval);
+			//sleep for the rest of the interval and start in a new interval
+			try
+			{
+				TimeUnit.MILLISECONDS.sleep(sleeptime);
+				fifteenMinuteRequestCount=0;
+			}
+			catch (InterruptedException e)
+			{
+			}
+		}	
 	}
 
 	/**
@@ -323,45 +351,65 @@ public class Main
 			try
 			{
 				json = makeOneGetRequest(urlExtension);
-				dailyRequestCount++;
 			}
 			catch (ResponseCodeWrongException e)
 			{
-				// tried enough times, so stop now
-				if (count >= retryTimes)
+				count = handleWrongResponseCode(count, e);
+				if (count == -1)
 				{
-					writeError(
-							"Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode());
 					return null;
 				}
-				if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
-				{	//token might have expired
-					if(!getAccessToken())	//token doesn't work anymore and we can't get a new one
-					{
-						throw new NoAccessException();
-					}
-				}
-				else if (e.getResponseCode() == httpCodeLimitReached)
-				{// request limit is reached, try again later
-					count++;
-				}
-				else // some other error: try only one other time!
-				{
-					count = retryTimes;
-				}
-				// Sleep for 5 minutes and try to get the next 15 min request window
-				try
-				{
-					TimeUnit.MINUTES.sleep(5);
-				}
-				catch (InterruptedException e1)
-				{
-				}
 			}
 
 		} while (json == null);
 		return json;
 	}
+	
+	/**
+	 * Method processed the wrong response code and adjusts the retry times as needed: 
+	 * When the retry limit is reached, -1 is returned.
+	 * When the access limit is reached, the retry count is increased.
+	 * When an unknown error appears, the retry count is set to max to allow only one more retry.
+	 * After that, the method sleeps for 5 Minutes.
+	 * @param count Retry count
+	 * @param e Exception containing the information
+	 * @return the new retry count or -1 if there are no more tries left
+	 * @throws NoAccessException if no AccessToken could be requested
+	 */
+	static int handleWrongResponseCode(int count, ResponseCodeWrongException e) throws NoAccessException
+	{
+		// tried enough times, so stop now
+		if (count >= retryTimes)
+		{
+			writeLog(
+					"Athlete: " + athleteId + " Retry limit reached. Last error code: " + e.getResponseCode());
+			return -1;
+		}
+		if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
+		{	//token might have expired
+			if(!getAccessToken())	//token doesn't work anymore and we can't get a new one
+			{
+				throw new NoAccessException();
+			}
+		}
+		else if (e.getResponseCode() == httpCodeLimitReached)
+		{// request limit is reached, try again later
+			count++;
+		}
+		else // some other error: try only one other time!
+		{
+			count = retryTimes;
+		}
+		// Sleep for 5 minutes and try to get the next 15 min request window
+		try
+		{
+			TimeUnit.MINUTES.sleep(5);
+		}
+		catch (InterruptedException e1)
+		{
+		}
+		return count;
+	}
 
 	/**
 	 * Extracts an athletes general information.
@@ -396,12 +444,12 @@ public class Main
 		}
 		catch (ParseException e)
 		{
-			writeError("Athlete " + athleteId + ": Error parsing general information.");
+			writeLog("Athlete " + athleteId + ": Error parsing general information.");
 			return null;
 		}
 		catch (NullPointerException e)
 		{
-			writeError("Athlete " + athleteId + ": No general information found.");
+			writeLog("Athlete " + athleteId + ": No general information found.");
 			return null;
 		}
 	}
@@ -454,19 +502,6 @@ public class Main
 		HttpsURLConnection connection = null;
 		try
 		{
-			long timeSinceLastRequest = System.currentTimeMillis() - lastRequestTimeInMillis;
-			if (timeSinceLastRequest < waitTimeMil)
-			{
-				try
-				{
-					TimeUnit.MILLISECONDS.sleep(waitTimeMil - timeSinceLastRequest);
-				}
-				catch (InterruptedException e)
-				{
-				}
-			}
-			;
-			lastRequestTimeInMillis = System.currentTimeMillis();
 			// Create connection
 			URL url = new URL(baseUrl + requestUrlExtension);
 			connection = (HttpsURLConnection) url.openConnection();
@@ -477,7 +512,7 @@ public class Main
 		}
 		catch (IOException e)
 		{
-			writeError("Athlete: " + athleteId + " Error while handling GET request: " + e.toString());
+			writeLog("Athlete: " + athleteId + " Error while handling GET request: " + e.toString());
 		}
 		return "";
 	}
@@ -495,12 +530,20 @@ public class Main
 	{
 		StringBuilder result = new StringBuilder();
 		int responseCode = connection.getResponseCode();
+		
+		if( fifteenMinuteRequestCount == 0) //first request in a 15 Min Interval
+		{
+			firstRequestTimeInCurrent15MinInterval = System.currentTimeMillis();			
+		}
+		dailyRequestCount++;
+		fifteenMinuteRequestCount++;
+		
 		if (responseCode != HttpURLConnection.HTTP_OK)
 		{
 			// excluded error messages appearing on missing streams and reached rate limit
 			if (responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode != httpCodeLimitReached)
 			{
-				writeError("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream());
+				writeLog("Athlete: " + athleteId + " Wrong response code: " + responseCode, connection.getErrorStream());
 			}
 			throw new ResponseCodeWrongException(responseCode);
 		}
@@ -564,7 +607,7 @@ public class Main
 		}
 		catch (IOException e)
 		{
-			writeError("Athlete: " + athleteId + "Error while handling POST request: " + e.toString());
+			writeLog("Athlete: " + athleteId + "Error while handling POST request: " + e.toString());
 		}
 		return "";
 	}
@@ -588,39 +631,20 @@ public class Main
 			try
 			{
 				json = makeOnePostRequest();
-				dailyRequestCount++;
 			}
 			catch (ResponseCodeWrongException e)
 			{
-				// tried enough times, so stop now
-				if (count >= retryTimes)
-				{
-					return false;
-				}
-
-				if (e.getResponseCode()==HttpURLConnection.HTTP_UNAUTHORIZED)
-				{	//token might have expired
-					if(!getAccessToken())	//token doesn't work anymore and we can't get a new one
-					{
-						return false;
-					}
-				}
-				else if (e.getResponseCode() == httpCodeLimitReached)
+				try
 				{
-					// request limit is reached, try again later
-					count++;
+					count = handleWrongResponseCode(count, e);
 				}
-				else // some other error: try only one other time!
+				catch (NoAccessException e2)
 				{
-					count = retryTimes;
-				}
-				// Sleep for 5 minutes and try to get the next 15 min request window
-				try
-				{
-					TimeUnit.MINUTES.sleep(5);
+					return false;
 				}
-				catch (InterruptedException e1)
+				if (count == -1)
 				{
+					return false;
 				}
 			}
 
@@ -635,7 +659,7 @@ public class Main
 		}
 		catch (ParseException e)
 		{
-			writeError("Athlete " + athleteId + ": Error parsing refresh info.");
+			writeLog("Athlete " + athleteId + ": Error parsing refresh info.");
 		}
 		return false;
 	}
@@ -658,7 +682,7 @@ public class Main
 			athleteId++;
 			if (!getAccessToken())
 			{
-				writeError("Couldn't get new access token for client " + athleteId);
+				writeLog("Couldn't get new access token for client " + athleteId);
 				continue;
 			}
 
@@ -680,7 +704,7 @@ public class Main
 				}
 				catch (IOException e)
 				{
-					writeError("Files coulnd't be zipped");
+					writeLog("Files coulnd't be zipped");
 				}
 			}
 		}
@@ -691,7 +715,7 @@ public class Main
 		}
 		catch (IOException e)
 		{
-			writeError("Files coulnd't be zipped");
+			writeLog("Files coulnd't be zipped");
 		}
 	}
 }