From 3521106098606b99c01064f6a3db980178b38f60 Mon Sep 17 00:00:00 2001 From: Breezebuilder Date: Tue, 11 Mar 2025 23:51:46 +1100 Subject: [PATCH] Improvements to mod version checking and GitHub API rate limit checking - Check if download url links to latest head, and if so, use version of latest commit hash instead of release version - Merge `get_latest_release` and `get_latest_commit` into `get_version_string` for less duplicate code - Check and print GitHub API rate limit details including api resource, remaining calls, and reset time - On exceeding rate limit or 403 error, check if primary or secondary rate limit has been reached - On primary rate limit breach, wait until hourly rate reset time - On secondary rate limit breach, wait for `retry-after` response time or an exponential time, starting at 60 seconds and doubling for each attempt, following GitHub API docs - Prevent program waiting for more than 30 minutes for API rate reset time --- .github/scripts/update_mod_versions.py | 123 ++++++++++++++++--------- 1 file changed, 77 insertions(+), 46 deletions(-) diff --git a/.github/scripts/update_mod_versions.py b/.github/scripts/update_mod_versions.py index a838ca22..b1426fdd 100755 --- a/.github/scripts/update_mod_versions.py +++ b/.github/scripts/update_mod_versions.py @@ -24,53 +24,74 @@ def extract_repo_info(repo_url): return owner, repo return None, None -def get_latest_release(owner, repo): - """Get the latest release version from GitHub.""" - url = f'https://api.github.com/repos/{owner}/{repo}/releases/latest' +def get_version_string(source, owner, repo, start_timestamp, n = 1): + """Get the version string from a given GitHub repo.""" + + if source == 'release': + url = f'https://api.github.com/repos/{owner}/{repo}/releases/latest' + else: + url = f'https://api.github.com/repos/{owner}/{repo}/commits' + try: response = requests.get(url, headers=HEADERS) + api_rate_limit = int(response.headers.get('x-ratelimit-limit')) + api_rate_usage = int(response.headers.get('x-ratelimit-used')) + api_rate_remaining = int(response.headers.get('x-ratelimit-remaining')) + api_reset_timestamp = int(response.headers.get('x-ratelimit-reset')) + api_resource = response.headers.get('x-ratelimit-resource') + print(f"GitHub API ({api_resource}) calls: {api_rate_usage}/{api_rate_limit}") + if response.status_code == 200: - data = response.json() - return data.get('tag_name') + if source == 'release': + data = response.json() + # Return name of latest tag + return data.get('tag_name') + else: + commits = response.json() + if commits and len(commits) > 0: + # Return shortened commit hash (first 7 characters) + return commits[0]['sha'][:7] elif response.status_code == 404: - # No releases found + # Not found return None - elif response.status_code == 403 and 'rate limit exceeded' in response.text.lower(): - print("GitHub API rate limit exceeded. Waiting for 5 minutes...") - time.sleep(300) # Wait for 5 minutes - return get_latest_release(owner, repo) # Retry + elif api_rate_remaining == 0 or (response.status_code == 403 and 'rate limit exceeded' in response.text.lower()): + print(f"GitHub API access is being rate limited!") + current_timestamp = int(time.time()) + + # Check if primary rate limit is okay + if api_rate_remaining > 0: + # Secondary rate limit hit, follow GitHub instructions + if 'retry-after' in response.headers: + wait_time = int(response.headers.get('retry-after')) + 5 + print(f"Response retry-after {wait_time}s") + else: + # Start at 60 seconds and double wait time with each new attempt + print(f"Attempt {n}") + wait_time = 60 * pow(2, n - 1) + else: + api_reset_time = datetime.fromtimestamp(api_reset_timestamp).strftime('%H:%M:%S') + print(f"GitHub API rate limit resets at {api_reset_time}") + wait_time = api_reset_timestamp - current_timestamp + 5 + + # Wait only if the wait time would finish less than 1800 seconds (30 min) after program start time + if current_timestamp + wait_time - start_timestamp < 1800: + print(f"Waiting {wait_time} seconds until next attempt...") + time.sleep(wait_time) + n += 1 + return get_version_string(source, owner, repo, start_timestamp, n) # Retry + else: + print(f"Next attempt in {wait_time} seconds, but Action run time would exceed 1800 seconds - Stopping...") + sys.exit(1) + else: - print(f"Error fetching releases: HTTP {response.status_code} - {response.text}") + print(f"Error fetching {source}s: HTTP {response.status_code} - {response.text}") return None except Exception as e: - print(f"Exception while fetching releases: {str(e)}") + print(f"Exception while fetching {source}s: {str(e)}") return None -def get_latest_commit(owner, repo): - """Get the latest commit hash from GitHub.""" - url = f'https://api.github.com/repos/{owner}/{repo}/commits' - try: - response = requests.get(url, headers=HEADERS) - - if response.status_code == 200: - commits = response.json() - if commits and len(commits) > 0: - # Return shortened commit hash (first 7 characters) - return commits[0]['sha'][:7] - elif response.status_code == 403 and 'rate limit exceeded' in response.text.lower(): - print("GitHub API rate limit exceeded. Waiting for 5 minutes...") - time.sleep(300) # Wait for 5 minutes - return get_latest_commit(owner, repo) # Retry - else: - print(f"Error fetching commits: HTTP {response.status_code} - {response.text}") - - return None - except Exception as e: - print(f"Exception while fetching commits: {str(e)}") - return None - -def process_mods(): +def process_mods(start_timestamp): """Process all mods and update versions where needed.""" mods_dir = Path('mods') updated_mods = [] @@ -106,16 +127,24 @@ def process_mods(): print(f"Checking GitHub repo: {owner}/{repo}") - # Try to get latest release version first - new_version = get_latest_release(owner, repo) - version_source = "release" - - # If no releases, fall back to latest commit - if not new_version: - print("No releases found, checking latest commit...") - new_version = get_latest_commit(owner, repo) + # If download url links to latest head, use version of latest commit hash + download_url = meta.get('downloadURL') + if "/archive/refs/heads/" in download_url: + print("Download URL links to HEAD, checking latest commit...") version_source = "commit" + new_version = get_version_string(version_source, owner, repo, start_timestamp) + else: + # Try to get latest release version + print("Checking releases for latest version tag...") + version_source = "release" + new_version = get_version_string(version_source, owner, repo, start_timestamp) + # If no releases, fall back to latest commit + if not new_version: + print("No releases found, checking latest commit...") + version_source = "commit" + new_version = get_version_string(version_source, owner, repo, start_timestamp) + if not new_version: print(f"⚠️ Warning: Could not determine version for {mod_dir.name}") continue @@ -163,8 +192,10 @@ def generate_commit_message(updated_mods): return message if __name__ == "__main__": - print(f"🔄 Starting automatic mod version update at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}...") - updated_mods = process_mods() + start_timestamp = int(time.time()) + start_datetime = datetime.fromtimestamp(start_timestamp).strftime('%H:%M:%S') + print(f"🔄 Starting automatic mod version update at {start_datetime}...") + updated_mods = process_mods(start_timestamp) if updated_mods: # Write commit message to a file that the GitHub Action can use