update_manager: delete all entries of unused cached TI files if they'…

…re not in the given config

modules/update_manager/update_manager.py

@@ -11,6 +11,8 @@
     IO,
     Optional,
     Tuple,
+    Dict,
+    List,
 )
 
 import requests
@@ -130,12 +132,13 @@ def read_riskiq_creds(risk_iq_credentials_path):
 
     def get_feed_details(self, feeds_path):
         """
-        Parse links, threat level and tags from the feeds_path file and return
+        Parse links, threat level and tags from the given feeds_path file and
+        return
          a dict with feed info
         """
         try:
             with open(feeds_path, "r") as feeds_file:
-                feeds = feeds_file.read()
+                feeds: str = feeds_file.read()
         except FileNotFoundError:
             self.print(
                 f"Error finding {feeds_path}. Feed won't be added to slips."
@@ -201,8 +204,10 @@ def read_ports_info(self, ports_info_filepath) -> int:
         """
 
         # there are ports that are by default considered unknown to slips,
-        # but if it's known to be used by a specific organization, slips won't consider it 'unknown'.
-        # in ports_info_filepath  we have a list of organizations range/ip and the port it's known to use
+        # but if it's known to be used by a specific organization, slips won't
+        # consider it 'unknown'.
+        # in ports_info_filepath  we have a list of organizations range/ip and
+        # the port it's known to use
         with open(ports_info_filepath, "r") as f:
             line_number = 0
             while True:
@@ -1604,6 +1609,41 @@ def should_update_mac_db(self) -> bool:
 
         return self.download_mac_db()
 
+    def delete_unused_cached_remote_feeds(self):
+        """
+        Slips caches all the feeds it downloads. If the user deleted any of
+        the feeds used, like literally deleted it (not using ;) the feeds
+        will still be there in the cache. the purpose of this function is
+        to delete these unused feeds from the cache
+        """
+        # get the cached feeds
+        loaded_feeds: Dict[str, Dict[str, str]] = self.db.get_loaded_ti_feeds()
+        # filter remote ones only, bc the loaded feeds have local ones too
+        cached_remote_feeds: List[str] = [
+            feed for feed in loaded_feeds if feed.startswith("http")
+        ]
+
+        # get the remote feeds that should be used from the config file
+        remote_feeds_from_config: List[str] = (
+            list(self.url_feeds.keys())
+            + list(self.ja3_feeds)
+            + list(self.ssl_feeds)
+            + [self.mac_db_link]
+        )
+        for feed in cached_remote_feeds:
+            # check is the feed should be used. is it in the given config
+            # of this run?
+            if feed not in remote_feeds_from_config:
+                # delete the feed from the cache
+                self.db.delete_ti_feed(feed)
+                self.db.delete_feed_entries(feed)
+                self.print(
+                    f"Deleted feed {feed} from cache",
+                    2,
+                    0,
+                    log_to_logfiles_only=True,
+                )
+
     async def update(self) -> bool:
         """
         Main function. It tries to update the TI files from a remote server
@@ -1637,6 +1677,11 @@ async def update(self) -> bool:
             files_to_download.update(self.ja3_feeds)
             files_to_download.update(self.ssl_feeds)
 
+            # before updating any feeds, make sure that the cached feeds
+            # are not using any feed that is not given in the config of
+            # this run (self.url_feeds, self.ja3_feeds, self.ssl_feeds)
+            self.delete_unused_cached_remote_feeds()
+
             for file_to_download in files_to_download:
                 if self.should_update(file_to_download, self.update_period):
                     # failed to get the response, either a server problem
@@ -1683,7 +1728,7 @@ async def update_ti_files(self):
         self.update_finished = asyncio.create_task(self.update())
         await self.update_finished
         self.print(
-            f"{self.db.get_loaded_ti_feeds()} "
+            f"{self.db.get_loaded_ti_feeds_number()} "
             f"TI files successfully loaded."
         )