You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
570 lines
16 KiB
570 lines
16 KiB
package export;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.BufferedWriter;
|
|
import java.io.File;
|
|
import java.io.FileInputStream;
|
|
import java.io.FileOutputStream;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.io.InputStreamReader;
|
|
import java.io.Reader;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.URL;
|
|
import java.net.URLEncoder;
|
|
import java.nio.charset.Charset;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.concurrent.TimeUnit;
|
|
import java.util.zip.ZipEntry;
|
|
import java.util.zip.ZipOutputStream;
|
|
|
|
import javax.net.ssl.HttpsURLConnection;
|
|
|
|
import org.json.simple.JSONArray;
|
|
import org.json.simple.JSONObject;
|
|
import org.json.simple.parser.JSONParser;
|
|
import org.json.simple.parser.ParseException;
|
|
|
|
public class Main
|
|
{
|
|
public static final String baseUrl = "https://www.strava.com/api/v3/";
|
|
public static final String tokenUrl = "https://www.strava.com/api/v3/oauth/token";
|
|
private static final int retryTimes = 20;
|
|
private static final int httpCodeLimitReached = 429;
|
|
private static int athleteId = 0;
|
|
private static JSONParser parser = new JSONParser();
|
|
private static String testRequest;
|
|
private static File errorFile;
|
|
//TODO: enduco needs to insert the correct request limit for 15 Minutes here
|
|
private static final int requestLimit15Minutes = 100;
|
|
private static int waitTimeMil = 60000*3*15/requestLimit15Minutes;
|
|
private static long lastRequest=0;
|
|
|
|
private static void writeError(String text)
|
|
{
|
|
if (errorFile == null)
|
|
{
|
|
errorFile = new File("error_" + System.currentTimeMillis() + ".txt");
|
|
}
|
|
|
|
try (BufferedWriter bout = new BufferedWriter(new FileWriter(errorFile, true)))
|
|
{
|
|
bout.write(text);
|
|
bout.newLine();
|
|
}
|
|
catch (IOException e)
|
|
{
|
|
}
|
|
}
|
|
|
|
/**
|
|
* For testing: ensures the test requests are used
|
|
*
|
|
* @param testrequest Request to be used
|
|
*/
|
|
static void setTest(String testrequest)
|
|
{
|
|
testRequest = testrequest;
|
|
}
|
|
|
|
/**
|
|
* Saves the data of one athlete into a temp file and returns it.
|
|
*
|
|
* @param token Identifier / authorization token of the athlete
|
|
* @return created temp file
|
|
*/
|
|
@SuppressWarnings("unchecked")
|
|
static File oneAthlete(String token)
|
|
{
|
|
// get Activities
|
|
Map<String, JSONObject> activities = getActivities(token);
|
|
|
|
// for each activity: save streams
|
|
JSONArray allActivities = new JSONArray();
|
|
int simpleActivityId = 0;
|
|
for (String id : activities.keySet())
|
|
{
|
|
JSONObject data = addStreams(id, activities.get(id), token);
|
|
data.put("activity_id", simpleActivityId);
|
|
allActivities.add(data);
|
|
simpleActivityId++;
|
|
}
|
|
|
|
// get general information
|
|
JSONObject athleteInfo = saveGeneralInformation(token);
|
|
if(athleteInfo==null) //error getting General Information
|
|
{
|
|
athleteInfo = new JSONObject();
|
|
}
|
|
athleteInfo.put("activities", allActivities);
|
|
|
|
try
|
|
{
|
|
File temp = File.createTempFile("Athlete_" + athleteId, ".json");
|
|
temp.deleteOnExit();
|
|
BufferedWriter bw = new BufferedWriter(new FileWriter(temp));
|
|
bw.write(athleteInfo.toString());
|
|
bw.close();
|
|
return temp;
|
|
}
|
|
catch (IOException e)
|
|
{
|
|
writeError("Athlete " + athleteId + ": Error writing temp file: " + e.toString());
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Adds the streams to the given activity
|
|
*
|
|
* @param id Strava id of the activity
|
|
* @param data general information of the activity
|
|
* @param token Identifier / authorization token of the athlete
|
|
* @return The data with the added streams
|
|
*/
|
|
@SuppressWarnings("unchecked")
|
|
static JSONObject addStreams(String id, JSONObject data, String token)
|
|
{
|
|
String requestUrlExtension = "activities/" + id + "/streams?"
|
|
+ "keys=[time,distance,latlng,altitude,velocity_smooth,heartrate,"
|
|
+ "cadence,watts,temp,moving,grade_smooth]&key_by_type=true";
|
|
|
|
String json = makeGetRequestWithRetry(requestUrlExtension, token);;
|
|
|
|
if (json==null ||json.isEmpty()||json.isBlank()||json.equals(""))
|
|
{
|
|
return data;
|
|
}
|
|
Object obj;
|
|
try
|
|
{
|
|
obj = parser.parse(json);
|
|
JSONObject listOfStreams = (JSONObject) obj;
|
|
for (Object key : listOfStreams.keySet())
|
|
{
|
|
JSONObject oneStream = (JSONObject) listOfStreams.get(key);
|
|
data.put("stream_" + key.toString(), oneStream);
|
|
}
|
|
}
|
|
catch (ParseException | NumberFormatException e)
|
|
{
|
|
writeError("Athlete " + athleteId + ": Error parsing json (Streams): " + e.toString());
|
|
}
|
|
return data;
|
|
}
|
|
|
|
/**
|
|
* Gathers all activities of a user, extracts the general information and the
|
|
* ids.
|
|
*
|
|
* @param token Identifier / authorization token of the athlete
|
|
* @return A Map with key = Strava Id of an activity and value = JSONObject with
|
|
* the general information of the activity
|
|
*/
|
|
@SuppressWarnings("unchecked")
|
|
static Map<String, JSONObject> getActivities(String token)
|
|
{
|
|
Map<String, JSONObject> result = new HashMap<>();
|
|
/*
|
|
* Possible values = AlpineSki, BackcountrySki, Canoeing, Crossfit, EBikeRide,
|
|
* Elliptical, Golf, Handcycle, Hike, IceSkate, InlineSkate, Kayaking, Kitesurf,
|
|
* NordicSki, Ride, RockClimbing, RollerSki, Rowing, Run, Sail, Skateboard,
|
|
* Snowboard, Snowshoe, Soccer, StairStepper, StandUpPaddling, Surfing, Swim,
|
|
* Velomobile, VirtualRide, VirtualRun, Walk, WeightTraining, Wheelchair,
|
|
* Windsurf, Workout, Yoga
|
|
*/
|
|
String type = "type";
|
|
String timezone = "timezone";
|
|
String start = "start_date_local"; // Start date measured in users time zone
|
|
String id = "id";
|
|
|
|
int pageIndex = 1;
|
|
while (true)
|
|
{
|
|
String requestExtension = "athlete/activities?per_page=500&page=" + pageIndex;
|
|
String json = makeGetRequestWithRetry(requestExtension, token);
|
|
if (json==null || json.isEmpty() || json.isBlank() || json.equals("") || json.equals("[]")) // don't know where the last page is...
|
|
{
|
|
break;
|
|
}
|
|
|
|
Object obj;
|
|
try
|
|
{
|
|
obj = parser.parse(json);
|
|
JSONArray listOfActivites = (JSONArray) obj;
|
|
for (int i = 0; i < listOfActivites.size(); i++)
|
|
{
|
|
JSONObject oneActivity = (JSONObject) listOfActivites.get(i);
|
|
|
|
JSONObject toSave = new JSONObject();
|
|
toSave.put(type, oneActivity.get(type));
|
|
toSave.put(timezone, oneActivity.get(timezone));
|
|
toSave.put(start, oneActivity.get(start));
|
|
toSave.put("athlete_id", athleteId);
|
|
|
|
Object idObj = oneActivity.get(id);
|
|
if (idObj != null)
|
|
{
|
|
result.put(oneActivity.get(id).toString(), toSave);
|
|
}
|
|
}
|
|
}
|
|
catch (ParseException | NumberFormatException e)
|
|
{
|
|
writeError("Athlete " + athleteId + ": Error parsing json (Activities): " + e.toString());
|
|
}
|
|
pageIndex++;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Method is used to find the next request window: It tries the same request again after 5 minutes.
|
|
* After a set number of times (retryTimes) it stops if it still wasn't successful.
|
|
* @param urlExtension UrlExtension for the request
|
|
* @param token Token for the request
|
|
* @return Data as a String or {@code null} if there is no data
|
|
*/
|
|
static String makeGetRequestWithRetry(String urlExtension, String token)
|
|
{
|
|
String json=null;
|
|
int count=0;
|
|
do
|
|
{
|
|
try
|
|
{
|
|
json = makeOneGetRequest(urlExtension, token);
|
|
}
|
|
catch (ResponseCodeWrongException e)
|
|
{
|
|
//tried enough times, so stop now
|
|
if(count >= retryTimes)
|
|
{
|
|
return null;
|
|
}
|
|
//request limit is reached, try again later
|
|
if (e.getResponseCode()==httpCodeLimitReached)
|
|
{
|
|
count++;
|
|
}
|
|
else //some other error: try only one other time!
|
|
{
|
|
count=retryTimes;
|
|
}
|
|
//Sleep for 5 minutes and try to get the next 15 min request window
|
|
try
|
|
{
|
|
TimeUnit.MINUTES.sleep(5);
|
|
}
|
|
catch (InterruptedException e1)
|
|
{
|
|
}
|
|
}
|
|
|
|
}while(json==null);
|
|
return json;
|
|
}
|
|
|
|
/**
|
|
* Extracts an athletes general information.
|
|
*
|
|
* @param token Identifier / authorization token of the athlete
|
|
* @return extracted data or null if there was an error
|
|
*/
|
|
@SuppressWarnings("unchecked")
|
|
static JSONObject saveGeneralInformation(String token)
|
|
{
|
|
String sex = "sex"; // Possible values = M, F
|
|
String country = "country";
|
|
String date_pref = "date_preference";
|
|
String meas_pref = "measurement_preference"; // Possible values = feet, meters
|
|
String weight = "weight";
|
|
|
|
|
|
String json = makeGetRequestWithRetry("athlete", token);
|
|
|
|
JSONObject toSave = new JSONObject();
|
|
try
|
|
{
|
|
Object obj = parser.parse(json);
|
|
JSONObject data = (JSONObject) obj;
|
|
toSave.put(sex, data.get(sex));
|
|
toSave.put(country, data.get(country));
|
|
toSave.put(date_pref, data.get(date_pref));
|
|
toSave.put(meas_pref, data.get(meas_pref));
|
|
toSave.put(weight, data.get(weight));
|
|
toSave.put("id", athleteId);
|
|
|
|
return toSave;
|
|
}
|
|
catch (ParseException e)
|
|
{
|
|
writeError("Athlete " + athleteId + ": Error parsing general information.");
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Zip a list of files in one .zip file.
|
|
*
|
|
* @param files HasMap of <intended Filename, File> which should be zipped
|
|
* @throws IOException If there was an error zipping
|
|
*/
|
|
static void zipAllFiles(Map<String, File> files) throws IOException
|
|
{
|
|
FileOutputStream fos = new FileOutputStream("data.zip");
|
|
ZipOutputStream zipOut = new ZipOutputStream(fos);
|
|
for (String key : files.keySet())
|
|
{
|
|
File fileToZip = files.get(key);
|
|
FileInputStream fis = new FileInputStream(fileToZip);
|
|
ZipEntry zipEntry = new ZipEntry(key);
|
|
zipOut.putNextEntry(zipEntry);
|
|
|
|
byte[] bytes = new byte[1024];
|
|
int length;
|
|
while ((length = fis.read(bytes)) >= 0)
|
|
{
|
|
zipOut.write(bytes, 0, length);
|
|
}
|
|
fis.close();
|
|
}
|
|
zipOut.close();
|
|
fos.close();
|
|
}
|
|
|
|
/**
|
|
* Handles one GET request to the API
|
|
*
|
|
* @param requestUrlExtension Extension for the baseUrl (without '/')
|
|
* @param token Identification / authorization token of the
|
|
* athlete
|
|
* @return The response as a String, an empty String in case of error.
|
|
* @throws ResponseCodeWrongException If there was an http error
|
|
*/
|
|
static String makeOneGetRequest(String requestUrlExtension, String token) throws ResponseCodeWrongException
|
|
{
|
|
if (testRequest != null)
|
|
{
|
|
String varTestRequest = testRequest.replaceAll("\\:\\s*([0-9]{15,})\\,", ":\"$1\",");
|
|
testRequest = null;
|
|
return varTestRequest;
|
|
}
|
|
HttpsURLConnection connection = null;
|
|
try
|
|
{
|
|
long timeSinceLastRequest =System.currentTimeMillis()-lastRequest;
|
|
if( timeSinceLastRequest<waitTimeMil)
|
|
{
|
|
try
|
|
{
|
|
TimeUnit.MILLISECONDS.sleep(waitTimeMil-timeSinceLastRequest);
|
|
}
|
|
catch (InterruptedException e)
|
|
{
|
|
}
|
|
};
|
|
lastRequest=System.currentTimeMillis();
|
|
// Create connection
|
|
URL url = new URL(baseUrl + requestUrlExtension);
|
|
connection = (HttpsURLConnection) url.openConnection();
|
|
connection.setRequestMethod("GET");
|
|
connection.setRequestProperty("Authorization", "Bearer " + token);
|
|
|
|
return getResponse(connection);
|
|
}
|
|
catch (IOException e)
|
|
{
|
|
writeError("Error while handling GET request: " + e.toString());
|
|
}
|
|
return "";
|
|
}
|
|
|
|
/**
|
|
* Reads the response from the connection and returns it as a String
|
|
*
|
|
* @param connection Connection to the site
|
|
* @return Response as a String
|
|
* @throws IOException in case of error with the stream
|
|
* @throws ResponseCodeWrongException if no data was read because of http problems
|
|
*/
|
|
static String getResponse(HttpsURLConnection connection) throws IOException, ResponseCodeWrongException
|
|
{
|
|
StringBuilder result = new StringBuilder();
|
|
int responseCode = connection.getResponseCode();
|
|
if (responseCode != HttpURLConnection.HTTP_OK)
|
|
{
|
|
//excluded error messages appearing on missing streams and reached rate limit
|
|
if(responseCode != HttpURLConnection.HTTP_NOT_FOUND && responseCode!= httpCodeLimitReached)
|
|
{
|
|
writeError("Wrong response code: " + responseCode);
|
|
}
|
|
throw new ResponseCodeWrongException(responseCode);
|
|
}
|
|
|
|
try (Reader reader = new BufferedReader(
|
|
new InputStreamReader(connection.getInputStream(), Charset.forName(StandardCharsets.UTF_8.name()))))
|
|
{
|
|
int c = reader.read();
|
|
while (c != -1)
|
|
{
|
|
result.append((char) c);
|
|
c = reader.read();
|
|
}
|
|
}
|
|
|
|
// Numbers too long for int or long are turned into Strings
|
|
return result.toString().replaceAll("\\:\\s*([0-9]{15,})\\,", ":\"$1\",");
|
|
}
|
|
|
|
/**
|
|
* Used for generating the accessToken
|
|
*
|
|
* @param data Triplet containing: a the client_id, b the client_secret, c the
|
|
* refresh_token
|
|
* @return The response as a String, an empty String in case of error.
|
|
* @throws ResponseCodeWrongException If there was an http error
|
|
*/
|
|
static String makeOnePostRequest(Triplet data) throws ResponseCodeWrongException
|
|
{
|
|
HttpsURLConnection connection = null;
|
|
try
|
|
{
|
|
// Create connection
|
|
URL url = new URL(tokenUrl);
|
|
connection = (HttpsURLConnection) url.openConnection();
|
|
connection.setRequestMethod("POST");
|
|
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
|
|
|
|
Map<String, String> params = new LinkedHashMap<>();
|
|
params.put("client_id", data.getA());
|
|
params.put("client_secret", data.getB());
|
|
params.put("grant_type", "refresh_token");
|
|
params.put("refresh_token", data.getC());
|
|
|
|
StringBuilder postData = new StringBuilder();
|
|
for (String key : params.keySet())
|
|
{
|
|
if (postData.length() != 0)
|
|
{
|
|
postData.append('&');
|
|
}
|
|
postData.append(URLEncoder.encode(key, "UTF-8"));
|
|
postData.append('=');
|
|
postData.append(URLEncoder.encode(params.get(key), "UTF-8"));
|
|
}
|
|
byte[] postDataBytes = postData.toString().getBytes("UTF-8");
|
|
|
|
connection.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
|
|
connection.setDoOutput(true);
|
|
connection.getOutputStream().write(postDataBytes);
|
|
|
|
return getResponse(connection);
|
|
}
|
|
catch (IOException e)
|
|
{
|
|
writeError("Error while handling POST request: " + e.toString());
|
|
}
|
|
return "";
|
|
}
|
|
|
|
/**
|
|
* Sends the old refresh token to strava and retrieves a new one and an access token
|
|
* @param refreshInfo Refresh data with a the cliend_id, b the client_secret and c the refresh_token
|
|
* @return A tuple with a the access_token and b the new refresh_token or {@code null} if there was an error
|
|
*/
|
|
static Tuple getAccessToken(Triplet refreshInfo)
|
|
{
|
|
String json=null;
|
|
int count =0;
|
|
do
|
|
{
|
|
try
|
|
{
|
|
json = makeOnePostRequest(refreshInfo);
|
|
}
|
|
catch (ResponseCodeWrongException e)
|
|
{
|
|
//tried enough times, so stop now
|
|
if(count >= retryTimes)
|
|
{
|
|
return null;
|
|
}
|
|
//request limit is reached, try again later
|
|
if (e.getResponseCode()==httpCodeLimitReached)
|
|
{
|
|
count++;
|
|
}
|
|
else //some other error: try only one other time!
|
|
{
|
|
count=retryTimes;
|
|
}
|
|
//Sleep for 5 minutes and try to get the next 15 min request window
|
|
try
|
|
{
|
|
TimeUnit.MINUTES.sleep(5);
|
|
}
|
|
catch (InterruptedException e1)
|
|
{
|
|
}
|
|
}
|
|
|
|
}while(json==null);
|
|
try
|
|
{
|
|
Object obj = parser.parse(json);
|
|
JSONObject data = (JSONObject) obj;
|
|
return new Tuple(data.get("access_token").toString(), data.get("refresh_token").toString());
|
|
}
|
|
catch (ParseException e)
|
|
{
|
|
writeError("Athlete " + athleteId + ": Error parsing general information.");
|
|
}
|
|
return null;
|
|
}
|
|
|
|
public static void main(String[] args)
|
|
{
|
|
// TODO: tokens need to be added by Enduco
|
|
|
|
// Triplet a is client_id, b is client_secret, c is refresh_token
|
|
List<Triplet> refreshTokens = new ArrayList<>();
|
|
// don't know if you need it but here you will find the new refresh tokens after
|
|
// the requests
|
|
List<Triplet> newRefreshTokens = new ArrayList<>();
|
|
|
|
Map<String, File> allFiles = new HashMap<>();
|
|
for (Triplet oneUser : refreshTokens)
|
|
{
|
|
// a is access_token and b is new refresh_token
|
|
Tuple withAccessToken = getAccessToken(oneUser);
|
|
if (withAccessToken == null)
|
|
{
|
|
writeError("Coulnd't get new access token for client "+athleteId);
|
|
continue;
|
|
}
|
|
newRefreshTokens.add(new Triplet(oneUser.getA(), oneUser.getB(), withAccessToken.getB()));
|
|
|
|
File athlete = oneAthlete(withAccessToken.getA());
|
|
if (athlete != null)
|
|
{
|
|
allFiles.put("Athlete_" + athleteId + ".json", athlete);
|
|
}
|
|
athleteId++;
|
|
}
|
|
try
|
|
{
|
|
zipAllFiles(allFiles);
|
|
}
|
|
catch (IOException e)
|
|
{
|
|
writeError("Files coulnd't be zipped");
|
|
}
|
|
}
|
|
}
|