Wait for data mounting loop in supervisor

Update changelog

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## December 6, 2023
+
+- Fixed an issue during code execution where the mounted data drive appeared empty when scanning for files with `glob`, `listdir`, or `iterdir` due to a transient effect from mounting. The supervisor code now waits up to 30 seconds for scanning to return results before running your code.
+
 ## December 3, 2023
 
 - Fixed bug in parsing CPC Precipitation Outlooks in `wsfr_read.climate.cpc_outlooks` to additionally handle case where 2006 also has a slightly different format.

runtime/entrypoint.sh

@@ -22,9 +22,6 @@ main () {
     LOGURU_LEVEL=INFO python supervisor.py
 }
 
-# Let transient filesystem stuff settle
-sleep 5
-
 main |& tee "/code_execution/submission/log.txt"
 exit_code=${PIPESTATUS[0]}
 

runtime/supervisor.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+from time import sleep
 from typing import Any
 
 from loguru import logger
@@ -35,9 +36,28 @@ def main():
     logger.info("IS_SMOKE: {}", IS_SMOKE)
     logger.info("src_directory: {}", src_directory)
     logger.info("data_directory: {}", data_directory)
-    assert wsfr_read.config.DATA_ROOT == data_directory
     logger.info("preprocessed_directory: {}", preprocessed_directory)
 
+    # Check that DATA_ROOT is consistent
+    assert wsfr_read.config.DATA_ROOT == data_directory
+
+    # Check that data drive is fully mounted so that scanning returns data
+    for i in range(30):
+        try:
+            next(data_directory.iterdir())
+        except StopIteration:
+            sleep(1)
+        else:
+            break
+    try:
+        next(data_directory.iterdir())
+    except StopIteration:
+        logger.error("Data directory not properly mounted after waiting 30 seconds.")
+        raise
+    else:
+        logger.info("data_directory.iterdir returned results after waiting {} seconds.", i)
+
+    # Create preprocessed directory
     try:
         preprocessed_directory.mkdir(parents=True)
     except FileExistsError: