Merge pull request #1176 from stratosphereips/alya/remove_feed_from_c…

…ache_if_not_in_given_config

Remove feed from cache if not in given config

modules/update_manager/update_manager.py

@@ -11,6 +11,8 @@
     IO,
     Optional,
     Tuple,
+    Dict,
+    List,
 )
 
 import requests
@@ -130,12 +132,13 @@ def read_riskiq_creds(risk_iq_credentials_path):
 
     def get_feed_details(self, feeds_path):
         """
-        Parse links, threat level and tags from the feeds_path file and return
+        Parse links, threat level and tags from the given feeds_path file and
+        return
          a dict with feed info
         """
         try:
             with open(feeds_path, "r") as feeds_file:
-                feeds = feeds_file.read()
+                feeds: str = feeds_file.read()
         except FileNotFoundError:
             self.print(
                 f"Error finding {feeds_path}. Feed won't be added to slips."
@@ -201,8 +204,10 @@ def read_ports_info(self, ports_info_filepath) -> int:
         """
 
         # there are ports that are by default considered unknown to slips,
-        # but if it's known to be used by a specific organization, slips won't consider it 'unknown'.
-        # in ports_info_filepath  we have a list of organizations range/ip and the port it's known to use
+        # but if it's known to be used by a specific organization, slips won't
+        # consider it 'unknown'.
+        # in ports_info_filepath  we have a list of organizations range/ip and
+        # the port it's known to use
         with open(ports_info_filepath, "r") as f:
             line_number = 0
             while True:
@@ -1604,6 +1609,41 @@ def should_update_mac_db(self) -> bool:
 
         return self.download_mac_db()
 
+    def delete_unused_cached_remote_feeds(self):
+        """
+        Slips caches all the feeds it downloads. If the user deleted any of
+        the feeds used, like literally deleted it (not using ;) the feeds
+        will still be there in the cache. the purpose of this function is
+        to delete these unused feeds from the cache
+        """
+        # get the cached feeds
+        loaded_feeds: Dict[str, Dict[str, str]] = self.db.get_loaded_ti_feeds()
+        # filter remote ones only, bc the loaded feeds have local ones too
+        cached_remote_feeds: List[str] = [
+            feed for feed in loaded_feeds if feed.startswith("http")
+        ]
+
+        # get the remote feeds that should be used from the config file
+        remote_feeds_from_config: List[str] = (
+            list(self.url_feeds.keys())
+            + list(self.ja3_feeds)
+            + list(self.ssl_feeds)
+            + [self.mac_db_link]
+        )
+        for feed in cached_remote_feeds:
+            # check is the feed should be used. is it in the given config
+            # of this run?
+            if feed not in remote_feeds_from_config:
+                # delete the feed from the cache
+                self.db.delete_ti_feed(feed)
+                self.db.delete_feed_entries(feed)
+                self.print(
+                    f"Deleted feed {feed} from cache",
+                    2,
+                    0,
+                    log_to_logfiles_only=True,
+                )
+
     async def update(self) -> bool:
         """
         Main function. It tries to update the TI files from a remote server
@@ -1637,6 +1677,11 @@ async def update(self) -> bool:
             files_to_download.update(self.ja3_feeds)
             files_to_download.update(self.ssl_feeds)
 
+            # before updating any feeds, make sure that the cached feeds
+            # are not using any feed that is not given in the config of
+            # this run (self.url_feeds, self.ja3_feeds, self.ssl_feeds)
+            self.delete_unused_cached_remote_feeds()
+
             for file_to_download in files_to_download:
                 if self.should_update(file_to_download, self.update_period):
                     # failed to get the response, either a server problem
@@ -1683,7 +1728,7 @@ async def update_ti_files(self):
         self.update_finished = asyncio.create_task(self.update())
         await self.update_finished
         self.print(
-            f"{self.db.get_loaded_ti_feeds()} "
+            f"{self.db.get_loaded_ti_feeds_number()} "
             f"TI files successfully loaded."
         )
 

slips_files/core/database/database_manager.py

@@ -469,6 +469,9 @@ def update_threat_level(self, *args, **kwargs):
     def set_loaded_ti_files(self, *args, **kwargs):
         return self.rdb.set_loaded_ti_files(*args, **kwargs)
 
+    def get_loaded_ti_feeds_number(self, *args, **kwargs):
+        return self.rdb.get_loaded_ti_feeds_number(*args, **kwargs)
+
     def get_loaded_ti_feeds(self, *args, **kwargs):
         return self.rdb.get_loaded_ti_feeds(*args, **kwargs)
 

slips_files/core/database/redis_db/constants.py

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2021 Sebastian Garcia <[email protected]>
 # SPDX-License-Identifier: GPL-2.0-only
 class Constants:
-    LOADED_TI_FILES = "loaded TI files"
+    LOADED_TI_FILES = "loaded_TI_files_number"
     TI_FILES_INFO = "TI_files_info"
     GIVE_TI = "give_threat_intelligence"
     # all keys starting with IoC_* are used for storing IoCs read from

slips_files/core/database/redis_db/ioc_handler.py

@@ -63,7 +63,7 @@ def set_loaded_ti_files(self, number_of_loaded_files: int):
         """
         self.r.set(self.constants.LOADED_TI_FILES, number_of_loaded_files)
 
-    def get_loaded_ti_feeds(self):
+    def get_loaded_ti_feeds_number(self):
         """
         returns the number of successfully loaded TI files. or 0 if none is loaded
         """
@@ -96,6 +96,12 @@ def delete_feed_entries(self, url: str):
     def delete_ti_feed(self, file):
         self.rcache.hdel(self.constants.TI_FILES_INFO, file)
 
+    def get_loaded_ti_feeds(self):
+        """
+        returns the successfully loaded/cached TI files.
+        """
+        return self.rcache.hgetall(self.constants.TI_FILES_INFO)
+
     def set_feed_last_update_time(self, file: str, time: float):
         """
         sets the 'time' of last update of the given file