Implement request throttling

1 files changed, 24 insertions, 27 deletions

diff --git a/steamrelationships/sr.py b/steamrelationships/sr.py
index 4bad716..1cf9c2e 100644
--- a/steamrelationships/sr.py
+++ b/steamrelationships/sr.py
@@ -1,6 +1,7 @@
 import requests
 import json
 import time
+import random
 
 from steamrelationships.constants import _Const
 CONST = _Const()
@@ -8,33 +9,19 @@ CONST = _Const()
 class SteamRelationships:
     session: object = requests.Session()    # Session object so repeated querries to steam's api use the same TCP connection
     scanlist: dict = {}                     # To be populated by recursivescan()
-    reqjson: str = "requests.json"          # Path to the JSON file that handles requests
 
-    def __init__(self, webapikey: str, timeout: int = 30, timeout_retries: int = 5, reqdelay: float = 0.5, delayrand: float = 1.25, reqsafetybuffer: float = 0.9, reqjson: str = "requests.json") -> None:
+    def __init__(self, webapikey: str, timeout: int = 30, timeout_retries: int = 5, reqdelay: float = 0.5, delayrand: int = 25, reqsafetybuffer: float = 0.9, reqjson: str = "requests.json") -> None:
         """
-        (str/int) webapikey - Steam dev api key required to use Steam's ISteamUser/GetFriendList interface
-
-        Request Options:
-        (int) timeout (Default: 30) - Seconds to wait before timing out a request.
-        (int) timeout_retries (Default: 5) Number of times to retry a request after timing out
-        (float) reqdelay (Default: 0.5) - Default amount of seconds to wait between sending each request to Steam
-        (float) delayrand (Default: 1.25) - The max percentage of extra delay to randomly add to each request (1 = no randomness, <1 = randomly shorter, >1 = randomly longer)
-        (float) reqsafetybuffer (Default: 0.9) - Highest percent of Steam's API request limit you are willing to run
-                                                    > Steam has an API request limit of 100,000 requests per day.
-                                                    reqsafetybuffer = 0.9 means SteamRequests will send a max of 90,000 requests per day.
-                                                    Entering a number higher than 1 will not result in sending more than 100,000 requests per day
-
-        (str) reqjson (Default: "requests.json") - The name/filepath of the file to store request times in
         """ # I can't make this a formatted string :sob:
 
-        self.webapikey = webapikey
-        self.timeout = timeout
-        self.timeout_retries = timeout_retries
-        self.reqdelay = reqdelay
-        self.delayrand = delayrand
-        self.reqsafetybuffer = reqsafetybuffer
-        self.reqjson = reqjson
-
+        # This used to be in a try-except block, but I decided that it should error out on a proper error
+        self.webapikey: str = str(webapikey)
+        self.timeout: int = int(abs(timeout))
+        self.timeout_retries: int = int(abs(timeout_retries))
+        self.reqdelay: float = float(abs(reqdelay))
+        self.delayrand: int = int(abs(delayrand) % 100)
+        self.reqsafetybuffer: float = float(abs(reqsafetybuffer))
+        self.reqjson: str = str(reqjson)
 
 
     def _readjsonfile(self, filepath: str = "") -> dict:
@@ -80,7 +67,7 @@ class SteamRelationships:
             jsoncontents = self._readjsonfile(filename)
 
         except Exception as e:
-            print(f"[_checkrequests] Could not get the contents of file {filename} ({e})")
+            print(f"[_checkrequests] Could not get the contents of file \"{filename}\" ({e})")
             return -1
 
         # Check the current date. If over 1 day since last entry, add a new entry. Otherwise, edit the current day's entry [note, 1 day in nanoseconds = (8.64 * (10 ** 13)) ]
@@ -96,7 +83,7 @@ class SteamRelationships:
                 jsoncontents[list(jsoncontents.keys())[-1]][1].append(checktime)
 
             else:
-                print(f"[_checkrequests] Daily request limit reached ({jsoncontents[list(jsoncontents.keys())[-1]][0]}/{CONST.STEAMAPI_MAXREQ * self.reqsafetybuffer}). Please try again tomorrow, or increase \"reqsafetybuffer\" (currently: {self.reqsafetybuffer})")
+                print(f"[_checkrequests] Daily request limit reached ({currentreqs}/{CONST.STEAMAPI_MAXREQ * self.reqsafetybuffer}). Please try again tomorrow, or increase \"reqsafetybuffer\" (currently: {self.reqsafetybuffer})")
                 return 0
 
         # Update the json file
@@ -106,7 +93,7 @@ class SteamRelationships:
                 jsonfile.close()
 
         except Exception as e:
-            print("[_checkrequests] Could not update json file ({e})")
+            print(f"[_checkrequests] Could not update json file ({e})")
             return -1
 
         return jsoncontents[list(jsoncontents.keys())[-1]][0]
@@ -146,7 +133,11 @@ class SteamRelationships:
 
         # Error out on request error
         if result.status_code != requests.codes.ok:
-            print(f"[_getFriendsList] Got bad status code (Requested id: {steamid64}, status: {result.status_code})")
+            print(f"[_getFriendsList] Got bad status code (Requested id: {steamid64}, status: {result.status_code})", end="")
+            if result.status_code == 401: # Steam returns a 401 on profiles with private friends lists
+                print(f". It is likely that the requested id has a private profile / private friends list", end="")
+            
+            print("\n", end="")
             return {}
 
         # Get the json contents from the response
@@ -222,6 +213,9 @@ class SteamRelationships:
             nature of friendship relations, you will very quickly spam Steam with tens of thousands of requests.
             If this concept is unfamiliar to you, please take a quick glance at the Wikipedia page for 
             "six degress of separation": https://en.wikipedia.org/wiki/Six_degrees_of_separation
+
+
+            TLDR: recursivescan is exponential and you will reach 100,000 requests very quickly if you recurse greater than 3
         '''
 
         testlist: dict = self.basicscan(steamid64)
@@ -234,6 +228,9 @@ class SteamRelationships:
                     if friend not in alreadyscanned:
                         tempdict.update(self.basicscan(friend))
                         alreadyscanned.append(friend)
+
+                        sleepytime: float = abs(random.randrange(100 - self.delayrand, 100 + self.delayrand) / 100 * self.reqdelay)
+                        time.sleep(sleepytime)
             
             testlist.update(tempdict)
             tempdict.clear()