Fix rate limiting by not fetching all posts
All checks were successful
Build and Push Docker Image / build (push) Successful in 2m5s
All checks were successful
Build and Push Docker Image / build (push) Successful in 2m5s
- Break early when finding target post instead of list() - Limit poll_loop and initial_seed to max 40 posts per fetch - Changed replies=True to replies=False for fewer API calls The list() call was consuming the entire generator, triggering Cloudflare's ~40 post rate limit. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -97,8 +97,9 @@ def fetch_and_relay_post(post_url: str) -> bool:
|
|||||||
try:
|
try:
|
||||||
api = Api()
|
api = Api()
|
||||||
# Fetch user's statuses and find the matching post
|
# Fetch user's statuses and find the matching post
|
||||||
|
# Iterate without converting to list to avoid fetching all pages
|
||||||
print(f"Fetching statuses for @{username} to find post {post_id}...")
|
print(f"Fetching statuses for @{username} to find post {post_id}...")
|
||||||
statuses_gen = api.pull_statuses(username, replies=True)
|
statuses_gen = api.pull_statuses(username, replies=False)
|
||||||
|
|
||||||
if statuses_gen is None:
|
if statuses_gen is None:
|
||||||
print(
|
print(
|
||||||
@@ -108,19 +109,17 @@ def fetch_and_relay_post(post_url: str) -> bool:
|
|||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
statuses = list(statuses_gen)
|
# Iterate through the generator, breaking early when found
|
||||||
|
found = False
|
||||||
if not statuses:
|
for post in statuses_gen:
|
||||||
print(
|
|
||||||
"No statuses returned. The API may be rate-limited or blocked.",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
return False
|
|
||||||
|
|
||||||
for post in statuses:
|
|
||||||
if post and str(post.get("id", "")) == post_id:
|
if post and str(post.get("id", "")) == post_id:
|
||||||
print(f"Found post {post_id}")
|
print(f"Found post {post_id}")
|
||||||
return process_single_post(post, username)
|
found = True
|
||||||
|
result = process_single_post(post, username)
|
||||||
|
break
|
||||||
|
|
||||||
|
if found:
|
||||||
|
return result
|
||||||
|
|
||||||
print(f"Post not found in recent statuses: {post_id}", file=sys.stderr)
|
print(f"Post not found in recent statuses: {post_id}", file=sys.stderr)
|
||||||
return False
|
return False
|
||||||
@@ -151,7 +150,7 @@ def poll_loop() -> None:
|
|||||||
try:
|
try:
|
||||||
print(f"Checking for new posts from @{TRUTH_USER}...")
|
print(f"Checking for new posts from @{TRUTH_USER}...")
|
||||||
|
|
||||||
# Fetch recent statuses
|
# Fetch recent statuses (limit to avoid rate limiting)
|
||||||
statuses_gen = api.pull_statuses(TRUTH_USER, replies=False)
|
statuses_gen = api.pull_statuses(TRUTH_USER, replies=False)
|
||||||
if statuses_gen is None:
|
if statuses_gen is None:
|
||||||
print(
|
print(
|
||||||
@@ -161,12 +160,18 @@ def poll_loop() -> None:
|
|||||||
time.sleep(POLL_INTERVAL)
|
time.sleep(POLL_INTERVAL)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
statuses = list(statuses_gen)
|
# Only fetch first batch of posts to avoid rate limiting
|
||||||
|
# The API returns newest first, so we collect and reverse
|
||||||
|
statuses: list[dict] = []
|
||||||
|
max_fetch = 40 # Stay under Cloudflare's ~40 post limit
|
||||||
|
for i, post in enumerate(statuses_gen):
|
||||||
|
if i >= max_fetch:
|
||||||
|
break
|
||||||
|
if post is not None:
|
||||||
|
statuses.append(post)
|
||||||
|
|
||||||
# Process in chronological order (oldest first)
|
# Process in chronological order (oldest first)
|
||||||
for post in reversed(statuses):
|
for post in reversed(statuses):
|
||||||
if post is None:
|
|
||||||
continue
|
|
||||||
post_id = str(post.get("id", ""))
|
post_id = str(post.get("id", ""))
|
||||||
if not post_id:
|
if not post_id:
|
||||||
continue
|
continue
|
||||||
@@ -216,9 +221,12 @@ def initial_seed() -> None:
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
statuses = list(statuses_gen)
|
# Limit fetch to avoid rate limiting
|
||||||
count = 0
|
count = 0
|
||||||
for post in statuses:
|
max_fetch = 40
|
||||||
|
for i, post in enumerate(statuses_gen):
|
||||||
|
if i >= max_fetch:
|
||||||
|
break
|
||||||
if post is None:
|
if post is None:
|
||||||
continue
|
continue
|
||||||
post_id = str(post.get("id", ""))
|
post_id = str(post.get("id", ""))
|
||||||
|
|||||||
Reference in New Issue
Block a user