Suprsync update tcdirs (#564)

* Make suprsync regularly add timecode dirs for recently added files

* add log statement

* bugfix

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add docstring for min_ctime

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

socs/agents/suprsync/agent.py

@@ -140,6 +140,10 @@ def run(self, session, params=None):
 
         next_feed_update = 0
 
+        # update tcdirs every six-hours
+        last_tcdir_update = 0
+        tcdir_update_interval = 6 * 3600
+
         while self.running:
             counters['iterations'] += 1
 
@@ -161,6 +165,15 @@ def run(self, session, params=None):
 
             now = time.time()
 
+            if now - last_tcdir_update > tcdir_update_interval:
+                # add timecode-dirs for all files from the last week
+                self.log.info("Creating timecode dirs for recent files.....")
+                srfm.create_all_timecode_dirs(
+                    self.archive_name, min_ctime=now - (7 * 24 * 3600)
+                )
+                self.log.info("Finished creating tcdirs")
+                last_tcdir_update = now
+
             archive_stats = srfm.get_archive_stats(self.archive_name)
             if archive_stats is not None:
                 self.agent.publish_to_feed('archive_stats', {

socs/db/suprsync.py

@@ -394,7 +394,7 @@ def get_deletable_files(self, archive_name, delete_after, session=None):
 
         return files
 
-    def get_known_files(self, archive_name, session=None):
+    def get_known_files(self, archive_name, session=None, min_ctime=None):
         """Gets all files.  This can be used to help avoid
         double-registering files.
 
@@ -404,12 +404,19 @@ def get_known_files(self, archive_name, session=None):
                 Name of archive to pull files from
             session : sqlalchemy session
                 Session to use to query files.
+            min_ctime : float, optional
+                minimum ctime to use when querying files.
+
         """
         if session is None:
             session = self.Session()
 
+        if min_ctime is None:
+            min_ctime = 0
+
         query = session.query(SupRsyncFile).filter(
             SupRsyncFile.archive_name == archive_name,
+            SupRsyncFile.timestamp > min_ctime,
         ).order_by(asc(SupRsyncFile.timestamp))
 
         return list(query.all())
@@ -437,9 +444,10 @@ def _add_file_tcdir(self, file: SupRsyncFile, session):
         session.add(tcdir)
         return tcdir
 
-    def create_all_timecode_dirs(self, archive_name):
+    def create_all_timecode_dirs(self, archive_name, min_ctime=None):
         with self.Session.begin() as session:
-            files = self.get_known_files(archive_name, session=session)
+            files = self.get_known_files(
+                archive_name, session=session, min_ctime=min_ctime)
             for file in files:
                 self._add_file_tcdir(file, session)