Fix rate limiting by not fetching all posts
All checks were successful
Build and Push Docker Image / build (push) Successful in 2m5s

- Break early when finding target post instead of list()
- Limit poll_loop and initial_seed to max 40 posts per fetch
- Changed replies=True to replies=False for fewer API calls

The list() call was consuming the entire generator, triggering
Cloudflare's ~40 post rate limit.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2026-01-23 03:20:54 -05:00
parent 984233d538
commit 7e138ae65e

View File

@@ -97,8 +97,9 @@ def fetch_and_relay_post(post_url: str) -> bool:
try: try:
api = Api() api = Api()
# Fetch user's statuses and find the matching post # Fetch user's statuses and find the matching post
# Iterate without converting to list to avoid fetching all pages
print(f"Fetching statuses for @{username} to find post {post_id}...") print(f"Fetching statuses for @{username} to find post {post_id}...")
statuses_gen = api.pull_statuses(username, replies=True) statuses_gen = api.pull_statuses(username, replies=False)
if statuses_gen is None: if statuses_gen is None:
print( print(
@@ -108,19 +109,17 @@ def fetch_and_relay_post(post_url: str) -> bool:
) )
return False return False
statuses = list(statuses_gen) # Iterate through the generator, breaking early when found
found = False
if not statuses: for post in statuses_gen:
print(
"No statuses returned. The API may be rate-limited or blocked.",
file=sys.stderr,
)
return False
for post in statuses:
if post and str(post.get("id", "")) == post_id: if post and str(post.get("id", "")) == post_id:
print(f"Found post {post_id}") print(f"Found post {post_id}")
return process_single_post(post, username) found = True
result = process_single_post(post, username)
break
if found:
return result
print(f"Post not found in recent statuses: {post_id}", file=sys.stderr) print(f"Post not found in recent statuses: {post_id}", file=sys.stderr)
return False return False
@@ -151,7 +150,7 @@ def poll_loop() -> None:
try: try:
print(f"Checking for new posts from @{TRUTH_USER}...") print(f"Checking for new posts from @{TRUTH_USER}...")
# Fetch recent statuses # Fetch recent statuses (limit to avoid rate limiting)
statuses_gen = api.pull_statuses(TRUTH_USER, replies=False) statuses_gen = api.pull_statuses(TRUTH_USER, replies=False)
if statuses_gen is None: if statuses_gen is None:
print( print(
@@ -161,12 +160,18 @@ def poll_loop() -> None:
time.sleep(POLL_INTERVAL) time.sleep(POLL_INTERVAL)
continue continue
statuses = list(statuses_gen) # Only fetch first batch of posts to avoid rate limiting
# The API returns newest first, so we collect and reverse
statuses: list[dict] = []
max_fetch = 40 # Stay under Cloudflare's ~40 post limit
for i, post in enumerate(statuses_gen):
if i >= max_fetch:
break
if post is not None:
statuses.append(post)
# Process in chronological order (oldest first) # Process in chronological order (oldest first)
for post in reversed(statuses): for post in reversed(statuses):
if post is None:
continue
post_id = str(post.get("id", "")) post_id = str(post.get("id", ""))
if not post_id: if not post_id:
continue continue
@@ -216,9 +221,12 @@ def initial_seed() -> None:
) )
return return
statuses = list(statuses_gen) # Limit fetch to avoid rate limiting
count = 0 count = 0
for post in statuses: max_fetch = 40
for i, post in enumerate(statuses_gen):
if i >= max_fetch:
break
if post is None: if post is None:
continue continue
post_id = str(post.get("id", "")) post_id = str(post.get("id", ""))