feat: retry on http failure (#35)

canvassyncer/__main__.py

@@ -57,14 +57,23 @@ async def downloadMany(self, infos, totalSize=0):
                 print(text)
 
     async def json(self, *args, **kwargs):
+        retryTimes = 0
         checkError = bool(kwargs.pop("checkError", False))
-        async with self.sem:
-            resp = await self.client.get(*args, **kwargs)
-        res = resp.json()
-        if checkError and isinstance(res, dict) and res.get("errors"):
-            errMsg = res["errors"][0].get("message", "unknown error.")
-            print(f"\nError: {errMsg}")
-            exit(1)
+        debugMode = bool(kwargs.pop("debug", False))
+        while retryTimes <= 5:
+            try:
+                async with self.sem:
+                    resp = await self.client.get(*args, **kwargs)
+                res = resp.json()
+                if checkError and isinstance(res, dict) and res.get("errors"):
+                    errMsg = res["errors"][0].get("message", "unknown error.")
+                    print(f"\nError: {errMsg}")
+                    exit(1)
+                return res
+            except Exception as e:
+                retryTimes += 1
+                if debugMode:
+                    print(f"{e.__class__.__name__}. Retry. {retryTimes} times.")
         return res
 
     async def head(self, *args, **kwargs):
@@ -135,7 +144,7 @@ def prepareLocalFiles(self, courseID, folders):
     async def getCourseFoldersWithIDHelper(self, page, courseID):
         res = {}
         url = f"{self.baseUrl}/courses/{courseID}/folders?page={page}"
-        folders = await self.client.json(url)
+        folders = await self.client.json(url, debug=self.config["debug"])
         for folder in folders:
             if folder["full_name"].startswith("course files"):
                 folder["full_name"] = folder["full_name"][len("course files") :]
@@ -148,7 +157,7 @@ async def getCourseFoldersWithIDHelper(self, page, courseID):
     async def getCourseFilesHelper(self, page, courseID, folders):
         files = {}
         url = f"{self.baseUrl}/courses/{courseID}/files?page={page}"
-        canvasFiles = await self.client.json(url)
+        canvasFiles = await self.client.json(url, debug=self.config["debug"])
         if not canvasFiles or isinstance(canvasFiles, dict):
             return files
         for f in canvasFiles:
@@ -170,7 +179,7 @@ async def getCourseFiles(self, courseID):
     async def getCourseIdByCourseCodeHelper(self, page, lowerCourseCodes):
         res = {}
         url = f"{self.baseUrl}/courses?page={page}"
-        courses = await self.client.json(url, checkError=True)
+        courses = await self.client.json(url, checkError=True, debug=self.config["debug"])
         if not courses:
             return res
         for course in courses:
@@ -187,7 +196,7 @@ async def getCourseIdByCourseCode(self):
 
     async def getCourseCodeByCourseIDHelper(self, courseID):
         url = f"{self.baseUrl}/courses/{courseID}"
-        clientRes = await self.client.json(url)
+        clientRes = await self.client.json(url, debug=self.config["debug"])
         if clientRes.get("course_code") is None:
             return
         self.courseCode[courseID] = clientRes["course_code"]