Fix rate limiting by not fetching all posts
All checks were successful
Build and Push Docker Image / build (push) Successful in 2m5s
All checks were successful
Build and Push Docker Image / build (push) Successful in 2m5s
- Break early when finding target post instead of list() - Limit poll_loop and initial_seed to max 40 posts per fetch - Changed replies=True to replies=False for fewer API calls The list() call was consuming the entire generator, triggering Cloudflare's ~40 post rate limit. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -97,8 +97,9 @@ def fetch_and_relay_post(post_url: str) -> bool:
|
||||
try:
|
||||
api = Api()
|
||||
# Fetch user's statuses and find the matching post
|
||||
# Iterate without converting to list to avoid fetching all pages
|
||||
print(f"Fetching statuses for @{username} to find post {post_id}...")
|
||||
statuses_gen = api.pull_statuses(username, replies=True)
|
||||
statuses_gen = api.pull_statuses(username, replies=False)
|
||||
|
||||
if statuses_gen is None:
|
||||
print(
|
||||
@@ -108,19 +109,17 @@ def fetch_and_relay_post(post_url: str) -> bool:
|
||||
)
|
||||
return False
|
||||
|
||||
statuses = list(statuses_gen)
|
||||
|
||||
if not statuses:
|
||||
print(
|
||||
"No statuses returned. The API may be rate-limited or blocked.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False
|
||||
|
||||
for post in statuses:
|
||||
# Iterate through the generator, breaking early when found
|
||||
found = False
|
||||
for post in statuses_gen:
|
||||
if post and str(post.get("id", "")) == post_id:
|
||||
print(f"Found post {post_id}")
|
||||
return process_single_post(post, username)
|
||||
found = True
|
||||
result = process_single_post(post, username)
|
||||
break
|
||||
|
||||
if found:
|
||||
return result
|
||||
|
||||
print(f"Post not found in recent statuses: {post_id}", file=sys.stderr)
|
||||
return False
|
||||
@@ -151,7 +150,7 @@ def poll_loop() -> None:
|
||||
try:
|
||||
print(f"Checking for new posts from @{TRUTH_USER}...")
|
||||
|
||||
# Fetch recent statuses
|
||||
# Fetch recent statuses (limit to avoid rate limiting)
|
||||
statuses_gen = api.pull_statuses(TRUTH_USER, replies=False)
|
||||
if statuses_gen is None:
|
||||
print(
|
||||
@@ -161,12 +160,18 @@ def poll_loop() -> None:
|
||||
time.sleep(POLL_INTERVAL)
|
||||
continue
|
||||
|
||||
statuses = list(statuses_gen)
|
||||
# Only fetch first batch of posts to avoid rate limiting
|
||||
# The API returns newest first, so we collect and reverse
|
||||
statuses: list[dict] = []
|
||||
max_fetch = 40 # Stay under Cloudflare's ~40 post limit
|
||||
for i, post in enumerate(statuses_gen):
|
||||
if i >= max_fetch:
|
||||
break
|
||||
if post is not None:
|
||||
statuses.append(post)
|
||||
|
||||
# Process in chronological order (oldest first)
|
||||
for post in reversed(statuses):
|
||||
if post is None:
|
||||
continue
|
||||
post_id = str(post.get("id", ""))
|
||||
if not post_id:
|
||||
continue
|
||||
@@ -216,9 +221,12 @@ def initial_seed() -> None:
|
||||
)
|
||||
return
|
||||
|
||||
statuses = list(statuses_gen)
|
||||
# Limit fetch to avoid rate limiting
|
||||
count = 0
|
||||
for post in statuses:
|
||||
max_fetch = 40
|
||||
for i, post in enumerate(statuses_gen):
|
||||
if i >= max_fetch:
|
||||
break
|
||||
if post is None:
|
||||
continue
|
||||
post_id = str(post.get("id", ""))
|
||||
|
||||
Reference in New Issue
Block a user