Improvements to mod version checking and GitHub API rate limit checking
- Check if download url links to latest head, and if so, use version of latest commit hash instead of release version
- Merge `get_latest_release` and `get_latest_commit` into `get_version_string` for less duplicate code
- Check and print GitHub API rate limit details including api resource, remaining calls, and reset time
- On exceeding rate limit or 403 error, check if primary or secondary rate limit has been reached
- On primary rate limit breach, wait until hourly rate reset time
- On secondary rate limit breach, wait for `retry-after` response time or an exponential time, starting at 60 seconds and doubling for each attempt, following GitHub API docs
- Prevent program waiting for more than 30 minutes for API rate reset time
This commit is contained in:
Breezebuilder
123
.github/scripts/update_mod_versions.py
vendored
123
.github/scripts/update_mod_versions.py
vendored
@@ -24,53 +24,74 @@ def extract_repo_info(repo_url):
|
||||
return owner, repo
|
||||
return None, None
|
||||
|
||||
def get_latest_release(owner, repo):
|
||||
"""Get the latest release version from GitHub."""
|
||||
url = f'https://api.github.com/repos/{owner}/{repo}/releases/latest'
|
||||
def get_version_string(source, owner, repo, start_timestamp, n = 1):
|
||||
"""Get the version string from a given GitHub repo."""
|
||||
|
||||
if source == 'release':
|
||||
url = f'https://api.github.com/repos/{owner}/{repo}/releases/latest'
|
||||
else:
|
||||
url = f'https://api.github.com/repos/{owner}/{repo}/commits'
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=HEADERS)
|
||||
|
||||
api_rate_limit = int(response.headers.get('x-ratelimit-limit'))
|
||||
api_rate_usage = int(response.headers.get('x-ratelimit-used'))
|
||||
api_rate_remaining = int(response.headers.get('x-ratelimit-remaining'))
|
||||
api_reset_timestamp = int(response.headers.get('x-ratelimit-reset'))
|
||||
api_resource = response.headers.get('x-ratelimit-resource')
|
||||
print(f"GitHub API ({api_resource}) calls: {api_rate_usage}/{api_rate_limit}")
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return data.get('tag_name')
|
||||
if source == 'release':
|
||||
data = response.json()
|
||||
# Return name of latest tag
|
||||
return data.get('tag_name')
|
||||
else:
|
||||
commits = response.json()
|
||||
if commits and len(commits) > 0:
|
||||
# Return shortened commit hash (first 7 characters)
|
||||
return commits[0]['sha'][:7]
|
||||
elif response.status_code == 404:
|
||||
# No releases found
|
||||
# Not found
|
||||
return None
|
||||
elif response.status_code == 403 and 'rate limit exceeded' in response.text.lower():
|
||||
print("GitHub API rate limit exceeded. Waiting for 5 minutes...")
|
||||
time.sleep(300) # Wait for 5 minutes
|
||||
return get_latest_release(owner, repo) # Retry
|
||||
elif api_rate_remaining == 0 or (response.status_code == 403 and 'rate limit exceeded' in response.text.lower()):
|
||||
print(f"GitHub API access is being rate limited!")
|
||||
current_timestamp = int(time.time())
|
||||
|
||||
# Check if primary rate limit is okay
|
||||
if api_rate_remaining > 0:
|
||||
# Secondary rate limit hit, follow GitHub instructions
|
||||
if 'retry-after' in response.headers:
|
||||
wait_time = int(response.headers.get('retry-after')) + 5
|
||||
print(f"Response retry-after {wait_time}s")
|
||||
else:
|
||||
# Start at 60 seconds and double wait time with each new attempt
|
||||
print(f"Attempt {n}")
|
||||
wait_time = 60 * pow(2, n - 1)
|
||||
else:
|
||||
api_reset_time = datetime.fromtimestamp(api_reset_timestamp).strftime('%H:%M:%S')
|
||||
print(f"GitHub API rate limit resets at {api_reset_time}")
|
||||
wait_time = api_reset_timestamp - current_timestamp + 5
|
||||
|
||||
# Wait only if the wait time would finish less than 1800 seconds (30 min) after program start time
|
||||
if current_timestamp + wait_time - start_timestamp < 1800:
|
||||
print(f"Waiting {wait_time} seconds until next attempt...")
|
||||
time.sleep(wait_time)
|
||||
n += 1
|
||||
return get_version_string(source, owner, repo, start_timestamp, n) # Retry
|
||||
else:
|
||||
print(f"Next attempt in {wait_time} seconds, but Action run time would exceed 1800 seconds - Stopping...")
|
||||
sys.exit(1)
|
||||
|
||||
else:
|
||||
print(f"Error fetching releases: HTTP {response.status_code} - {response.text}")
|
||||
print(f"Error fetching {source}s: HTTP {response.status_code} - {response.text}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"Exception while fetching releases: {str(e)}")
|
||||
print(f"Exception while fetching {source}s: {str(e)}")
|
||||
return None
|
||||
|
||||
def get_latest_commit(owner, repo):
|
||||
"""Get the latest commit hash from GitHub."""
|
||||
url = f'https://api.github.com/repos/{owner}/{repo}/commits'
|
||||
try:
|
||||
response = requests.get(url, headers=HEADERS)
|
||||
|
||||
if response.status_code == 200:
|
||||
commits = response.json()
|
||||
if commits and len(commits) > 0:
|
||||
# Return shortened commit hash (first 7 characters)
|
||||
return commits[0]['sha'][:7]
|
||||
elif response.status_code == 403 and 'rate limit exceeded' in response.text.lower():
|
||||
print("GitHub API rate limit exceeded. Waiting for 5 minutes...")
|
||||
time.sleep(300) # Wait for 5 minutes
|
||||
return get_latest_commit(owner, repo) # Retry
|
||||
else:
|
||||
print(f"Error fetching commits: HTTP {response.status_code} - {response.text}")
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"Exception while fetching commits: {str(e)}")
|
||||
return None
|
||||
|
||||
def process_mods():
|
||||
def process_mods(start_timestamp):
|
||||
"""Process all mods and update versions where needed."""
|
||||
mods_dir = Path('mods')
|
||||
updated_mods = []
|
||||
@@ -106,16 +127,24 @@ def process_mods():
|
||||
|
||||
print(f"Checking GitHub repo: {owner}/{repo}")
|
||||
|
||||
# Try to get latest release version first
|
||||
new_version = get_latest_release(owner, repo)
|
||||
version_source = "release"
|
||||
|
||||
# If no releases, fall back to latest commit
|
||||
if not new_version:
|
||||
print("No releases found, checking latest commit...")
|
||||
new_version = get_latest_commit(owner, repo)
|
||||
# If download url links to latest head, use version of latest commit hash
|
||||
download_url = meta.get('downloadURL')
|
||||
if "/archive/refs/heads/" in download_url:
|
||||
print("Download URL links to HEAD, checking latest commit...")
|
||||
version_source = "commit"
|
||||
new_version = get_version_string(version_source, owner, repo, start_timestamp)
|
||||
else:
|
||||
# Try to get latest release version
|
||||
print("Checking releases for latest version tag...")
|
||||
version_source = "release"
|
||||
new_version = get_version_string(version_source, owner, repo, start_timestamp)
|
||||
|
||||
# If no releases, fall back to latest commit
|
||||
if not new_version:
|
||||
print("No releases found, checking latest commit...")
|
||||
version_source = "commit"
|
||||
new_version = get_version_string(version_source, owner, repo, start_timestamp)
|
||||
|
||||
if not new_version:
|
||||
print(f"⚠️ Warning: Could not determine version for {mod_dir.name}")
|
||||
continue
|
||||
@@ -163,8 +192,10 @@ def generate_commit_message(updated_mods):
|
||||
return message
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"🔄 Starting automatic mod version update at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}...")
|
||||
updated_mods = process_mods()
|
||||
start_timestamp = int(time.time())
|
||||
start_datetime = datetime.fromtimestamp(start_timestamp).strftime('%H:%M:%S')
|
||||
print(f"🔄 Starting automatic mod version update at {start_datetime}...")
|
||||
updated_mods = process_mods(start_timestamp)
|
||||
|
||||
if updated_mods:
|
||||
# Write commit message to a file that the GitHub Action can use
|
||||
|
||||
Reference in New Issue
Block a user