Major optimizations & consolidation of code

fix_dates.py

@@ -1,6 +1,8 @@
 import argparse
 import datetime
 import time
+import urllib.parse
+import sys
 import xml.etree.ElementTree as ET
 
 import requests
@@ -9,8 +11,10 @@ import requests
 # some constants
 FILES_PATH_PREFIX = "/remote.php/dav/files/"
 VERSIONS_PATH_PREFIX = "/remote.php/dav/versions/"
+# the threshold for file timestamps (dates older than this are considered invalid)
+DATE_THRESHOLD = datetime.datetime(1990, 1, 1)
 # we only need one session for the whole script
-session = requests.Session()
+SESSION = requests.Session()
 
 
 def propfind(path, auth):
@@ -18,8 +22,11 @@ def propfind(path, auth):
     Get a file's Last Modified timestamp and FileID via a PROPFIND request
     :param path: The path of the file in question
     :param auth: Auth data for the HTTP request (e.g. a requests.auth.HTTPBasicAuth object)
-    :return: The properties in XML format
+    :return: An iterator of dictionaries, one for every directory entry. Entry properties are taken from the PROPFIND
+             response
     """
+    # do not descend further into subdirectories\
+    # TODO: we could probably be faster if we did
     headers = {"Depth": "1"}
     # This body returns only the timelastmodified and the fileid variable
     requested_data = \
@@ -27,96 +34,56 @@ def propfind(path, auth):
         <d:propfind  xmlns:d=\"DAV:\" xmlns:oc=\"http://owncloud.org/ns\" xmlns:nc=\"http://nextcloud.org/ns\">
            <d:prop>
                <d:getlastmodified />
+			   <d:resourcetype />
                <oc:fileid />
            </d:prop>
        </d:propfind>
        """
     req = requests.Request("PROPFIND", path, headers=headers, auth=auth, data=requested_data)
-    resp = session.send(req.prepare())
+    resp = SESSION.send(req.prepare())
-    print(resp.text)
+    et = ET.fromstring(resp.text)
-    return resp.text
+    for dav_response in et.findall('{DAV:}response'):
+        entry = {}
+        entry["path"] = dav_response.find("{DAV:}href").text
+        # skip this entry itself
+        if path.endswith(entry["path"]):
+            continue
+        props = dav_response.find("{DAV:}propstat").find("{DAV:}prop")
+        try:
+            entry["last_modified"] = datetime.datetime.strptime(
+                props.find("{DAV:}getlastmodified").text,
+                "%a, %d %b %Y %H:%M:%S GMT"
+            )
+        except (AttributeError, TypeError):
+            pass
+        entry["resource_type"] = []
+        try:
+            for resourcetype in props.find("{DAV:}resourcetype"):
+                entry["resource_type"].append(resourcetype.tag)
+        except (AttributeError, TypeError):
+            pass
+        try:
+            entry["file_id"] = int(props.find("{http://owncloud.org/ns}fileid").text)
+        except (AttributeError, TypeError):
+            pass
+        yield entry
 
 
-def search_folder(requestreturn):
+def find_valid_version(versions):
-    """
-    Iterates through a folder's properties XML and find entries with invalid timestamps
-    :param requestreturn: The XML returned by propfind()
-    :return: A tuple of two lists. The first list contains all subfolders, the second contains all FileIDs of entries
-             with an invalid timestamp.
-    """
-    # List to collect path of folders stored in path
-    innerfolders = []
-    # List to collect path of files with wrong timestamp
-    linkswrongtime = []
-    # First folder provided in the xml file is always the folder we are currently in.
-    # To prevent searching this folder twice, or ending in a loop, we can not store this folder in the folderlist again.
-    firstfolder = True
-    # Get a xml tree
-    tree = ET.ElementTree(ET.fromstring(requestreturn))
-    # Find all responses in the tree, those contain the fielpath, lasttimemodified, typeoffile etc.
-    for resp in tree.findall('.//{DAV:}response'):
-        # Here we can get the filpath out of href and get further information in prop (lastimemodified, typeoffile, etc.)
-        for p in resp:
-            # In case p.text is not none, it contains the filepath
-            if not (p.text is None):
-                if (p.text[-1] == '/'):
-                    # If the current object is a folder, check it its not first folder
-                    if not (firstfolder):
-                        innerfolders.append(p.text)
-                    else:
-                        firstfolder = False
-                    break
-            # In case p.text is none, it contains the further information
-            else:
-                for t in p.findall('.//{DAV:}getlastmodified'):
-                    # this function converts the given date to unix timestamp
-                    lastmodified = time.mktime(
-                        datetime.datetime.strptime(t.text, "%a, %d %b %Y %H:%M:%S GMT").timetuple())
-                    # 631148400 is the unix timestamp of 01.01.1990 00:00:00, because we know there is no file older
-                    # than this in our nextcloud
-                    if lastmodified < 631148400:
-                        for fileid in p.findall('.//{http://owncloud.org/ns}fileid'):
-                            linkswrongtime.append(fileid.text)
-    return (innerfolders, linkswrongtime)
-
-
-def version_check(xmlfile):
     """
     This function returns the fileid of the version of a given fileid with the most current timestamp or None if
-    there are no versions with a timestamp younger than 01.01.1990
+    there are no versions with a timestamp younger than the threshold
-    :param xmlfile: An XML file with Last Modified timestamps and FileIDs as returned by propfind()
+    :param versions: An iterator as returned by propfind()
-    :return: The FileID of the most recent version or None if no valid version exists
+    :return: The entry of the iterator which has the most recent date or None if none exists
     """
-    tree = ET.ElementTree(ET.fromstring(xmlfile))
+    # mock entry for comparison
-    # Name of files are stored as a string. But we can check the timestamp of the file only after we can check the name,
+    most_recent = {"last_modified": DATE_THRESHOLD}
-    # so it needs to be stored temporary in case the timestamp is the most current
+    for version in versions:
-    temp = "this is a temporary string"
+        if "last_modified" in version and version["last_modified"] > most_recent["last_modified"]:
-    # These two variables are used to store the highest/most current timestamp and the associated fileid
+            most_recent = version
-    most_current_timestamp = 631148400
+    if most_recent["last_modified"] == DATE_THRESHOLD:
-    most_current_timestamp_fileid = 0
-    for resp in tree.findall('.//{DAV:}response'):
-        for p in resp:
-            # In case p.text is not none, it contains the filepath
-            if not (p.text is None):
-                temp = p.text
-            else:
-                for t in p.findall('.//{DAV:}getlastmodified'):
-                    if not (t.text is None):
-                        # this function converts the given date to unix timestamp
-                        lastmodified = time.mktime(
-                            datetime.datetime.strptime(t.text, "%a, %d %b %Y %H:%M:%S GMT").timetuple())
-                        #
-                        if lastmodified > most_current_timestamp:
-                            most_current_timestamp = lastmodified
-                            fileid_old_version = temp.split('/')
-                            most_current_timestamp_fileid = fileid_old_version[-1]
-                    else:
-                        break
-    # Check if there is another version and a file with a current timestamp
-    if most_current_timestamp_fileid != 0:
-        return most_current_timestamp_fileid
-    else:
         return None
+    return most_recent
 
 
 if __name__ == "__main__":
@@ -138,24 +105,28 @@ if __name__ == "__main__":
     mainpath = FILES_PATH_PREFIX + arguments.username + arguments.search_path
     # List of all folders we need to enter
     folders = [mainpath]
-    # List of all fileids with wrong time
+    # List of all entries with wrong time
     wrongtime = []
 
     # Iterate through all folders and check for wrong timestamps
     while folders:
-        path_suffix = folders.pop(0)
+        url = arguments.server + folders.pop(0)
-        path = arguments.server + str(path_suffix)
+        print("+", end="", flush=True)
-        r = propfind(path, auth)
+        for entry in propfind(url, auth):
-        new_folders, new_wrongtime = search_folder(r)
+            print(".", end="", flush=True)
-        # Append all found folders and files with wrong timestamps to global list
+            # put directories in search list
-        folders += new_folders
+            if "resource_type" in entry and "{DAV:}collection" in entry["resource_type"]:
-        wrongtime += new_wrongtime
+                folders.append(entry["path"])
-        # Iterate through all fileids with wrong timestamps and check for versions with intact timestamp
+            # put files with wrong date in wrong date list (we don't know what to do if a directory has an invalid date)
-        while wrongtime:
+            elif "last_modified" in entry and entry["last_modified"] < DATE_THRESHOLD:
-            fileid = wrongtime.pop(0)
+                wrongtime.append(entry)
-            version_suffix = VERSIONS_PATH_PREFIX + arguments.username + "/versions/" + fileid
+    # Iterate through all fileids with wrong timestamps and check for versions with intact timestamp
-            version_path = arguments.server + version_suffix
+    print()
-            versions = propfind(version_path, auth)
+    # NOTE: you can indent this into the loop above to fix things on-the-fly instead of all at once
-            mrv = version_check(versions)
+    for entry in wrongtime:
-            print(fileid, end=": ")
+        print(urllib.parse.unquote(entry["path"][len(FILES_PATH_PREFIX):]))
-            print(mrv)
+        fixed_version = find_valid_version(propfind(arguments.server + VERSIONS_PATH_PREFIX + arguments.username + "/versions/" + str(entry["file_id"]), auth))
+        if fixed_version:
+            print("Restore from {}".format(fixed_version))
+        else:
+            print("Touch file.")