From d6832a62fe0272ba725b3d5ceab64b5fb3a07276 Mon Sep 17 00:00:00 2001 From: NW/RL Date: Tue, 2 Apr 2024 14:31:33 -0500 Subject: Implement request throttling --- steamrelationships/sr.py | 51 +++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/steamrelationships/sr.py b/steamrelationships/sr.py index 4bad716..1cf9c2e 100644 --- a/steamrelationships/sr.py +++ b/steamrelationships/sr.py @@ -1,6 +1,7 @@ import requests import json import time +import random from steamrelationships.constants import _Const CONST = _Const() @@ -8,33 +9,19 @@ CONST = _Const() class SteamRelationships: session: object = requests.Session() # Session object so repeated querries to steam's api use the same TCP connection scanlist: dict = {} # To be populated by recursivescan() - reqjson: str = "requests.json" # Path to the JSON file that handles requests - def __init__(self, webapikey: str, timeout: int = 30, timeout_retries: int = 5, reqdelay: float = 0.5, delayrand: float = 1.25, reqsafetybuffer: float = 0.9, reqjson: str = "requests.json") -> None: + def __init__(self, webapikey: str, timeout: int = 30, timeout_retries: int = 5, reqdelay: float = 0.5, delayrand: int = 25, reqsafetybuffer: float = 0.9, reqjson: str = "requests.json") -> None: """ - (str/int) webapikey - Steam dev api key required to use Steam's ISteamUser/GetFriendList interface - - Request Options: - (int) timeout (Default: 30) - Seconds to wait before timing out a request. - (int) timeout_retries (Default: 5) Number of times to retry a request after timing out - (float) reqdelay (Default: 0.5) - Default amount of seconds to wait between sending each request to Steam - (float) delayrand (Default: 1.25) - The max percentage of extra delay to randomly add to each request (1 = no randomness, <1 = randomly shorter, >1 = randomly longer) - (float) reqsafetybuffer (Default: 0.9) - Highest percent of Steam's API request limit you are willing to run - > Steam has an API request limit of 100,000 requests per day. - reqsafetybuffer = 0.9 means SteamRequests will send a max of 90,000 requests per day. - Entering a number higher than 1 will not result in sending more than 100,000 requests per day - - (str) reqjson (Default: "requests.json") - The name/filepath of the file to store request times in """ # I can't make this a formatted string :sob: - self.webapikey = webapikey - self.timeout = timeout - self.timeout_retries = timeout_retries - self.reqdelay = reqdelay - self.delayrand = delayrand - self.reqsafetybuffer = reqsafetybuffer - self.reqjson = reqjson - + # This used to be in a try-except block, but I decided that it should error out on a proper error + self.webapikey: str = str(webapikey) + self.timeout: int = int(abs(timeout)) + self.timeout_retries: int = int(abs(timeout_retries)) + self.reqdelay: float = float(abs(reqdelay)) + self.delayrand: int = int(abs(delayrand) % 100) + self.reqsafetybuffer: float = float(abs(reqsafetybuffer)) + self.reqjson: str = str(reqjson) def _readjsonfile(self, filepath: str = "") -> dict: @@ -80,7 +67,7 @@ class SteamRelationships: jsoncontents = self._readjsonfile(filename) except Exception as e: - print(f"[_checkrequests] Could not get the contents of file {filename} ({e})") + print(f"[_checkrequests] Could not get the contents of file \"{filename}\" ({e})") return -1 # Check the current date. If over 1 day since last entry, add a new entry. Otherwise, edit the current day's entry [note, 1 day in nanoseconds = (8.64 * (10 ** 13)) ] @@ -96,7 +83,7 @@ class SteamRelationships: jsoncontents[list(jsoncontents.keys())[-1]][1].append(checktime) else: - print(f"[_checkrequests] Daily request limit reached ({jsoncontents[list(jsoncontents.keys())[-1]][0]}/{CONST.STEAMAPI_MAXREQ * self.reqsafetybuffer}). Please try again tomorrow, or increase \"reqsafetybuffer\" (currently: {self.reqsafetybuffer})") + print(f"[_checkrequests] Daily request limit reached ({currentreqs}/{CONST.STEAMAPI_MAXREQ * self.reqsafetybuffer}). Please try again tomorrow, or increase \"reqsafetybuffer\" (currently: {self.reqsafetybuffer})") return 0 # Update the json file @@ -106,7 +93,7 @@ class SteamRelationships: jsonfile.close() except Exception as e: - print("[_checkrequests] Could not update json file ({e})") + print(f"[_checkrequests] Could not update json file ({e})") return -1 return jsoncontents[list(jsoncontents.keys())[-1]][0] @@ -146,7 +133,11 @@ class SteamRelationships: # Error out on request error if result.status_code != requests.codes.ok: - print(f"[_getFriendsList] Got bad status code (Requested id: {steamid64}, status: {result.status_code})") + print(f"[_getFriendsList] Got bad status code (Requested id: {steamid64}, status: {result.status_code})", end="") + if result.status_code == 401: # Steam returns a 401 on profiles with private friends lists + print(f". It is likely that the requested id has a private profile / private friends list", end="") + + print("\n", end="") return {} # Get the json contents from the response @@ -222,6 +213,9 @@ class SteamRelationships: nature of friendship relations, you will very quickly spam Steam with tens of thousands of requests. If this concept is unfamiliar to you, please take a quick glance at the Wikipedia page for "six degress of separation": https://en.wikipedia.org/wiki/Six_degrees_of_separation + + + TLDR: recursivescan is exponential and you will reach 100,000 requests very quickly if you recurse greater than 3 ''' testlist: dict = self.basicscan(steamid64) @@ -234,6 +228,9 @@ class SteamRelationships: if friend not in alreadyscanned: tempdict.update(self.basicscan(friend)) alreadyscanned.append(friend) + + sleepytime: float = abs(random.randrange(100 - self.delayrand, 100 + self.delayrand) / 100 * self.reqdelay) + time.sleep(sleepytime) testlist.update(tempdict) tempdict.clear() -- cgit v1.2.3