Skip to content

Commit

Permalink
Set exit code if a scrape fails
Browse files Browse the repository at this point in the history
  • Loading branch information
RealOrangeOne committed Jan 19, 2025
1 parent d2b3479 commit 7a5fc73
Showing 1 changed file with 12 additions and 1 deletion.
13 changes: 12 additions & 1 deletion scrape.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,11 @@ def create_episode(api_episode, show_config, output_dir):
with open(output_file, "w") as f:
print("Saving", api_episode["url"])
f.write(output)
return True
except Exception as e:
print(f"Skipping {api_episode['url']} because of error")
traceback.print_exception(e)
return False


def api_data_from_rss_item(item):
Expand Down Expand Up @@ -202,6 +204,8 @@ def main():
with open("mkdocs.yml") as f:
shows = yaml.load(f, Loader=yaml.SafeLoader)['extra']['shows']

exit_code = 0

with concurrent.futures.ThreadPoolExecutor() as executor:
futures = []
for show_slug, show_config in shows.items():
Expand All @@ -223,7 +227,14 @@ def main():

# Drain to get exceptions. Still have to mash CTRL-C, though.
for future in concurrent.futures.as_completed(futures):
future.result()
success = future.result()

if not success:
exit_code = 1

exit(exit_code)




if __name__ == "__main__":
Expand Down

0 comments on commit 7a5fc73

Please sign in to comment.