From c73b0139fdbeb888182ac63df217091fa322b636 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Thu, 21 May 2020 17:04:49 +0200 Subject: [PATCH] Changed storage to use session configuration #109 --- plaso/cli/log2timeline_tool.py | 7 +- plaso/cli/pinfo_tool.py | 27 ++---- plaso/cli/psort_tool.py | 9 ++ plaso/cli/psteal_tool.py | 8 +- plaso/engine/knowledge_base.py | 21 ++++- plaso/engine/single_process.py | 9 +- plaso/multi_processing/psort.py | 9 +- plaso/multi_processing/task_engine.py | 12 ++- plaso/storage/fake/writer.py | 64 ++++--------- plaso/storage/file_interface.py | 58 +++++------- plaso/storage/interface.py | 131 +++++++++++++++----------- tests/engine/knowledge_base.py | 26 ++++- tests/engine/single_process.py | 3 +- tests/multi_processing/psort.py | 32 ++++++- tests/multi_processing/task_engine.py | 2 +- tests/output/test_lib.py | 10 +- tests/storage/sqlite/sqlite_file.py | 8 +- 17 files changed, 250 insertions(+), 186 deletions(-) diff --git a/plaso/cli/log2timeline_tool.py b/plaso/cli/log2timeline_tool.py index 1e742ceb72..66e5496788 100644 --- a/plaso/cli/log2timeline_tool.py +++ b/plaso/cli/log2timeline_tool.py @@ -434,14 +434,15 @@ def ExtractEventsFromSources(self): logger.debug('Starting extraction in single process mode.') processing_status = extraction_engine.ProcessSources( - self._source_path_specs, storage_writer, self._resolver_context, - configuration, status_update_callback=status_update_callback) + session, self._source_path_specs, storage_writer, + self._resolver_context, configuration, + status_update_callback=status_update_callback) else: logger.debug('Starting extraction in multi process mode.') processing_status = extraction_engine.ProcessSources( - session.identifier, self._source_path_specs, storage_writer, + session, self._source_path_specs, storage_writer, configuration, enable_sigsegv_handler=self._enable_sigsegv_handler, number_of_worker_processes=self._number_of_extraction_workers, status_update_callback=status_update_callback, diff --git a/plaso/cli/pinfo_tool.py b/plaso/cli/pinfo_tool.py index 68324a9de5..d17a47b739 100644 --- a/plaso/cli/pinfo_tool.py +++ b/plaso/cli/pinfo_tool.py @@ -16,7 +16,6 @@ from plaso.cli import tools from plaso.cli import views from plaso.cli.helpers import manager as helpers_manager -from plaso.engine import knowledge_base from plaso.lib import definitions from plaso.lib import errors from plaso.lib import loggers @@ -436,27 +435,16 @@ def _PrintParsersCounter(self, parsers_counter, session_identifier=None): table_view.Write(self._output_writer) - def _PrintPreprocessingInformation( - self, storage_reader, session_identifier=None): - """Prints the details of the preprocessing information. + def _PrintSourceConfiguration( + self, source_configuration, session_identifier=None): + """Prints the details of a source configuration. Args: - storage_reader (StorageReader): storage reader. + source_configuration (SourceConfiguration): source configuration. session_identifier (Optional[str]): session identifier, formatted as a UUID. """ - knowledge_base_object = knowledge_base.KnowledgeBase() - - storage_reader.ReadSystemConfiguration(knowledge_base_object) - - lookup_identifier = session_identifier - if lookup_identifier: - # The knowledge base requires the session identifier to be formatted in - # hexadecimal representation. - lookup_identifier = lookup_identifier.replace('-', '') - - system_configuration = knowledge_base_object.GetSystemConfigurationArtifact( - session_identifier=lookup_identifier) + system_configuration = source_configuration.system_configuration if not system_configuration: return @@ -574,8 +562,9 @@ def _PrintSessionsDetails(self, storage_reader): table_view.Write(self._output_writer) if self._verbose: - self._PrintPreprocessingInformation( - storage_reader, session_identifier=session_identifier) + for source_configuration in session.source_configurations: + self._PrintSourceConfiguration( + source_configuration, session_identifier=session_identifier) self._PrintParsersCounter( session.parsers_counter, session_identifier=session_identifier) diff --git a/plaso/cli/psort_tool.py b/plaso/cli/psort_tool.py index 4d5191a283..a0d536d25f 100644 --- a/plaso/cli/psort_tool.py +++ b/plaso/cli/psort_tool.py @@ -520,6 +520,15 @@ def ProcessStorage(self): 'Format of storage file: {0:s} not supported'.format( self._storage_file_path)) + for session in storage_reader.GetSessions(): + if not session.source_configurations: + storage_reader.ReadSystemConfiguration(self._knowledge_base) + else: + for source_configuration in session.source_configurations: + self._knowledge_base.ReadSystemConfigurationArtifact( + source_configuration.system_configuration, + session_identifier=session.identifier) + self._number_of_analysis_reports = ( storage_reader.GetNumberOfAnalysisReports()) storage_reader.Close() diff --git a/plaso/cli/psteal_tool.py b/plaso/cli/psteal_tool.py index 088c45a54c..3b8f72d6fe 100644 --- a/plaso/cli/psteal_tool.py +++ b/plaso/cli/psteal_tool.py @@ -327,15 +327,15 @@ def ExtractEventsFromSources(self): logger.debug('Starting extraction in single process mode.') processing_status = extraction_engine.ProcessSources( - self._source_path_specs, storage_writer, self._resolver_context, - configuration, status_update_callback=status_update_callback) + session, self._source_path_specs, storage_writer, + self._resolver_context, configuration, + status_update_callback=status_update_callback) else: logger.debug('Starting extraction in multi process mode.') processing_status = extraction_engine.ProcessSources( - session.identifier, self._source_path_specs, storage_writer, - configuration, + session, self._source_path_specs, storage_writer, configuration, enable_sigsegv_handler=self._enable_sigsegv_handler, number_of_worker_processes=self._number_of_extraction_workers, status_update_callback=status_update_callback) diff --git a/plaso/engine/knowledge_base.py b/plaso/engine/knowledge_base.py index db51711e72..f92604fe3c 100644 --- a/plaso/engine/knowledge_base.py +++ b/plaso/engine/knowledge_base.py @@ -172,7 +172,26 @@ def GetHostname(self, session_identifier=None): return hostname_artifact.name or '' - def GetSystemConfigurationArtifact(self, session_identifier=None): + def GetSourceConfigurationArtifacts(self, session_identifier=None): + """Retrieves the knowledge base as a source configuration artifacts. + + Args: + session_identifier (Optional[str])): session identifier, where + None represents the active session. + + Returns: + list[SourceConfigurationArtifact]: source configuration artifacts. + """ + source_configuration = artifacts.SourceConfigurationArtifact() + + # TODO: set path_spec + source_configuration.system_configuration = ( + self._GetSystemConfigurationArtifact( + session_identifier=session_identifier)) + + return [source_configuration] + + def _GetSystemConfigurationArtifact(self, session_identifier=None): """Retrieves the knowledge base as a system configuration artifact. Args: diff --git a/plaso/engine/single_process.py b/plaso/engine/single_process.py index 0aef424b8d..817deff778 100644 --- a/plaso/engine/single_process.py +++ b/plaso/engine/single_process.py @@ -213,11 +213,12 @@ def _UpdateStatus( self._last_status_update_timestamp = current_timestamp def ProcessSources( - self, source_path_specs, storage_writer, resolver_context, + self, session, source_path_specs, storage_writer, resolver_context, processing_configuration, status_update_callback=None): """Processes the sources. Args: + session (Session): session in which the sources are processed. source_path_specs (list[dfvfs.PathSpec]): path specifications of the sources to process. storage_writer (StorageWriter): storage writer for a session storage. @@ -272,8 +273,12 @@ def ProcessSources( storage_writer.Open() storage_writer.WriteSessionStart() + # TODO: decouple session and storage writer? + session.source_configurations = ( + self.knowledge_base.GetSourceConfigurationArtifacts()) + try: - storage_writer.WritePreprocessingInformation(self.knowledge_base) + storage_writer.WriteSessionConfiguration() self._ProcessSources( source_path_specs, extraction_worker, parser_mediator, storage_writer) diff --git a/plaso/multi_processing/psort.py b/plaso/multi_processing/psort.py index de2e7f8cc8..c123cb8bbd 100644 --- a/plaso/multi_processing/psort.py +++ b/plaso/multi_processing/psort.py @@ -8,10 +8,10 @@ import os import time +from plaso.containers import tasks from plaso.engine import plaso_queue from plaso.engine import processing_status from plaso.engine import zeromq_queue -from plaso.containers import tasks from plaso.lib import bufferlib from plaso.lib import definitions from plaso.multi_processing import analysis_process @@ -901,10 +901,11 @@ def AnalyzeEvents( # the ZIP storage file will remain locked as long as the worker processes # are alive. storage_writer.Open() - storage_writer.ReadSystemConfiguration(knowledge_base_object) storage_writer.WriteSessionStart() try: + storage_writer.WriteSessionConfiguration() + self._AnalyzeEvents( storage_writer, analysis_plugins, event_filter=event_filter) @@ -981,11 +982,10 @@ def ExportEvents( an event of interest. """ self._events_status = processing_status.EventsStatus() + self._knowledge_base = knowledge_base_object self._processing_configuration = processing_configuration self._status_update_callback = status_update_callback - storage_reader.ReadSystemConfiguration(knowledge_base_object) - total_number_of_events = 0 for session in storage_reader.GetSessions(): total_number_of_events += session.parsers_counter['total'] @@ -1023,4 +1023,5 @@ def ExportEvents( # Reset values. self._status_update_callback = None self._processing_configuration = None + self._knowledge_base = None self._events_status = None diff --git a/plaso/multi_processing/task_engine.py b/plaso/multi_processing/task_engine.py index 7c9f8a6922..a99c973de1 100644 --- a/plaso/multi_processing/task_engine.py +++ b/plaso/multi_processing/task_engine.py @@ -714,14 +714,14 @@ def _UpdateProcessingStatus(self, pid, process_status, used_memory): process.name, task_identifier)) def ProcessSources( - self, session_identifier, source_path_specs, storage_writer, + self, session, source_path_specs, storage_writer, processing_configuration, enable_sigsegv_handler=False, number_of_worker_processes=0, status_update_callback=None, worker_memory_limit=None): """Processes the sources and extract events. Args: - session_identifier (str): identifier of the session. + session (Session): session in which the sources are processed. source_path_specs (list[dfvfs.PathSpec]): path specifications of the sources to process. storage_writer (StorageWriter): storage writer for a session storage. @@ -777,7 +777,7 @@ def ProcessSources( self._debug_output = processing_configuration.debug_output self._log_filename = processing_configuration.log_filename - self._session_identifier = session_identifier + self._session_identifier = session.identifier self._status_update_callback = status_update_callback self._storage_writer = storage_writer @@ -816,6 +816,10 @@ def ProcessSources( self._StartStatusUpdateThread() + # TODO: decouple session and storage writer? + session.source_configurations = ( + self.knowledge_base.GetSourceConfigurationArtifacts()) + try: # Open the storage file after creating the worker processes otherwise # the ZIP storage file will remain locked as long as the worker processes @@ -824,7 +828,7 @@ def ProcessSources( storage_writer.WriteSessionStart() try: - storage_writer.WritePreprocessingInformation(self.knowledge_base) + storage_writer.WriteSessionConfiguration() self._ProcessSources(source_path_specs, storage_writer) diff --git a/plaso/storage/fake/writer.py b/plaso/storage/fake/writer.py index 0833ce5c01..84ececb245 100644 --- a/plaso/storage/fake/writer.py +++ b/plaso/storage/fake/writer.py @@ -18,6 +18,8 @@ class FakeStorageWriter(interface.StorageWriter): analysis_reports (list[AnalysisReport]): analysis reports. session_completion (SessionCompletion): session completion attribute container. + session_configuration (SessionConfiguration): session configuration + attribute container. session_start (SessionStart): session start attribute container. task_completion (TaskCompletion): task completion attribute container. task_start (TaskStart): task start attribute container. @@ -43,6 +45,7 @@ def __init__( self._task_storage_writers = {} self.analysis_reports = [] self.session_completion = None + self.session_configuration = None self.session_start = None self.task_completion = None self.task_start = None @@ -426,29 +429,6 @@ def PrepareMergeTaskStorage(self, task): raise IOError('Storage writer for task: {0:s} does not exist.'.format( task.identifier)) - # pylint: disable=unused-argument - def ReadSystemConfiguration(self, knowledge_base): - """Reads system configuration information. - - The system configuration contains information about various system specific - configuration data, for example the user accounts. - - Args: - knowledge_base (KnowledgeBase): is used to store the system configuration. - - Raises: - IOError: if the storage type does not support writing preprocessing - information or when the storage writer is closed. - OSError: if the storage type does not support writing preprocessing - information or when the storage writer is closed. - """ - self._RaiseIfNotWritable() - - if self._storage_type != definitions.STORAGE_TYPE_SESSION: - raise IOError('Preprocessing information not supported by storage type.') - - # TODO: implement. - def RemoveProcessedTaskStorage(self, task): """Removes a processed task storage. @@ -481,27 +461,6 @@ def SetStorageProfiler(self, storage_profiler): """ return - # pylint: disable=unused-argument - def WritePreprocessingInformation(self, knowledge_base): - """Writes preprocessing information. - - Args: - knowledge_base (KnowledgeBase): used to store the preprocessing - information. - - Raises: - IOError: if the storage type does not support writing preprocessing - information or when the storage writer is closed. - OSError: if the storage type does not support writing preprocessing - information or when the storage writer is closed. - """ - self._RaiseIfNotWritable() - - if self._storage_type != definitions.STORAGE_TYPE_SESSION: - raise IOError('Preprocessing information not supported by storage type.') - - # TODO: implement. - def WriteSessionCompletion(self, aborted=False): """Writes session completion information. @@ -522,6 +481,23 @@ def WriteSessionCompletion(self, aborted=False): self._session.aborted = aborted self.session_completion = self._session.CreateSessionCompletion() + def WriteSessionConfiguration(self): + """Writes session configuration information. + + + Raises: + IOError: if the storage type does not support writing session + configuration information or when the storage writer is closed. + OSError: if the storage type does not support writing session + configuration information or when the storage writer is closed. + """ + self._RaiseIfNotWritable() + + if self._storage_type != definitions.STORAGE_TYPE_SESSION: + raise IOError('Session configuration not supported by storage type.') + + self.session_configuration = self._session.CreateSessionConfiguration() + def WriteSessionStart(self): """Writes session start information. diff --git a/plaso/storage/file_interface.py b/plaso/storage/file_interface.py index 5d83a4a512..f8e344a1a0 100644 --- a/plaso/storage/file_interface.py +++ b/plaso/storage/file_interface.py @@ -406,6 +406,7 @@ def HasWarnings(self): """ return self._storage_file.HasWarnings() + # TODO: remove, this method is kept for backwards compatibility reasons. def ReadSystemConfiguration(self, knowledge_base): """Reads system configuration information. @@ -891,24 +892,6 @@ def PrepareMergeTaskStorage(self, task): 'Unable to rename task storage file: {0:s} with error: ' '{1!s}').format(processed_storage_file_path, exception)) - def ReadSystemConfiguration(self, knowledge_base): - """Reads system configuration information. - - The system configuration contains information about various system specific - configuration data, for example the user accounts. - - Args: - knowledge_base (KnowledgeBase): is used to store the system configuration. - - Raises: - IOError: when the storage writer is closed. - OSError: when the storage writer is closed. - """ - if not self._storage_file: - raise IOError('Unable to read from closed storage writer.') - - self._storage_file.ReadSystemConfiguration(knowledge_base) - def RemoveProcessedTaskStorage(self, task): """Removes a processed task storage. @@ -1053,25 +1036,6 @@ def StopTaskStorage(self, abort=False): self._processed_task_storage_path = None self._task_storage_path = None - def WritePreprocessingInformation(self, knowledge_base): - """Writes preprocessing information. - - Args: - knowledge_base (KnowledgeBase): contains the preprocessing information. - - Raises: - IOError: if the storage type does not support writing preprocessing - information or when the storage writer is closed. - OSError: if the storage type does not support writing preprocessing - information or when the storage writer is closed. - """ - self._RaiseIfNotWritable() - - if self._storage_type != definitions.STORAGE_TYPE_SESSION: - raise IOError('Preprocessing information not supported by storage type.') - - self._storage_file.WritePreprocessingInformation(knowledge_base) - def WriteSessionCompletion(self, aborted=False): """Writes session completion information. @@ -1089,10 +1053,29 @@ def WriteSessionCompletion(self, aborted=False): if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') + # TODO: move self._session out of the StorageFileWriter? self._session.aborted = aborted session_completion = self._session.CreateSessionCompletion() self._storage_file.WriteSessionCompletion(session_completion) + def WriteSessionConfiguration(self): + """Writes session configuration information. + + Raises: + IOError: if the storage type does not support writing session + configuration information or when the storage writer is closed. + OSError: if the storage type does not support writing session + configuration information or when the storage writer is closed. + """ + self._RaiseIfNotWritable() + + if self._storage_type != definitions.STORAGE_TYPE_SESSION: + raise IOError('Session configuration not supported by storage type.') + + # TODO: move self._session out of the StorageFileWriter? + session_configuration = self._session.CreateSessionConfiguration() + self._storage_file.WriteSessionConfiguration(session_configuration) + def WriteSessionStart(self): """Writes session start information. @@ -1107,6 +1090,7 @@ def WriteSessionStart(self): if self._storage_type != definitions.STORAGE_TYPE_SESSION: raise IOError('Unsupported storage type.') + # TODO: move self._session out of the StorageFileWriter? session_start = self._session.CreateSessionStart() self._storage_file.WriteSessionStart(session_start) diff --git a/plaso/storage/interface.py b/plaso/storage/interface.py index 5db6107c7c..90873c3874 100644 --- a/plaso/storage/interface.py +++ b/plaso/storage/interface.py @@ -34,6 +34,8 @@ class BaseStore(object): warnings.ExtractionError.CONTAINER_TYPE) _CONTAINER_TYPE_EXTRACTION_WARNING = warnings.ExtractionWarning.CONTAINER_TYPE _CONTAINER_TYPE_SESSION_COMPLETION = sessions.SessionCompletion.CONTAINER_TYPE + _CONTAINER_TYPE_SESSION_CONFIGURATION = ( + sessions.SessionConfiguration.CONTAINER_TYPE) _CONTAINER_TYPE_SESSION_START = sessions.SessionStart.CONTAINER_TYPE _CONTAINER_TYPE_SYSTEM_CONFIGURATION = ( artifacts.SystemConfigurationArtifact.CONTAINER_TYPE) @@ -49,6 +51,7 @@ class BaseStore(object): _CONTAINER_TYPE_EVENT_SOURCE, _CONTAINER_TYPE_EVENT_TAG, _CONTAINER_TYPE_SESSION_COMPLETION, + _CONTAINER_TYPE_SESSION_CONFIGURATION, _CONTAINER_TYPE_SESSION_START, _CONTAINER_TYPE_SYSTEM_CONFIGURATION, _CONTAINER_TYPE_TASK_COMPLETION, @@ -57,13 +60,13 @@ class BaseStore(object): def __init__(self): """Initializes a store.""" super(BaseStore, self).__init__() - self.format_version = None - self.serialization_format = None - self.storage_type = None self._last_session = 0 + self._serializer = json_serializer.JSONAttributeContainerSerializer self._serializers_profiler = None self._storage_profiler = None - self._serializer = json_serializer.JSONAttributeContainerSerializer + self.format_version = None + self.serialization_format = None + self.storage_type = None @abc.abstractmethod def _AddAttributeContainer( @@ -337,19 +340,49 @@ def GetSessions(self): session_completion_generator = self._GetAttributeContainers( self._CONTAINER_TYPE_SESSION_COMPLETION) - for session_index in range(0, self._last_session): - session_start = next(session_start_generator) # pylint: disable=stop-iteration-return - session_completion = next(session_completion_generator) # pylint: disable=stop-iteration-return + if self._HasAttributeContainers(self._CONTAINER_TYPE_SESSION_CONFIGURATION): + session_configuration_generator = self._GetAttributeContainers( + self._CONTAINER_TYPE_SESSION_CONFIGURATION) + else: + session_configuration_generator = None + + for session_index in range(1, self._last_session + 1): + try: + session_start = next(session_start_generator) + except StopIteration: + raise IOError('Missing session start: {0:d}'.format(session_index)) + + try: + session_completion = next(session_completion_generator) + except StopIteration: + pass + + session_configuration = None + if session_configuration_generator: + try: + session_configuration = next(session_configuration_generator) + except StopIteration: + raise IOError('Missing session configuration: {0:d}'.format( + session_index)) session = sessions.Session() session.CopyAttributesFromSessionStart(session_start) + + if session_configuration: + try: + session.CopyAttributesFromSessionConfiguration(session_configuration) + except ValueError: + raise IOError(( + 'Session identifier mismatch for session configuration: ' + '{0:d}').format(session_index)) + if session_completion: try: session.CopyAttributesFromSessionCompletion(session_completion) except ValueError: - raise IOError( - 'Session identifier mismatch for session: {0:d}'.format( - session_index)) + raise IOError(( + 'Session identifier mismatch for session completion: ' + '{0:d}').format(session_index)) yield session @@ -411,6 +444,7 @@ def HasEventTags(self): def Open(self, **kwargs): """Opens the storage.""" + # TODO: remove, this method is kept for backwards compatibility reasons. def ReadSystemConfiguration(self, knowledge_base): """Reads system configuration information. @@ -420,10 +454,13 @@ def ReadSystemConfiguration(self, knowledge_base): Args: knowledge_base (KnowledgeBase): is used to store the system configuration. """ - generator = self._GetAttributeContainers( - self._CONTAINER_TYPE_SYSTEM_CONFIGURATION) - for system_configuration in generator: - knowledge_base.ReadSystemConfigurationArtifact(system_configuration) + # Backwards compatibility for older session storage files that do not + # store system configuration as part of the session configuration. + if self._HasAttributeContainers(self._CONTAINER_TYPE_SYSTEM_CONFIGURATION): + generator = self._GetAttributeContainers( + self._CONTAINER_TYPE_SYSTEM_CONFIGURATION) + for system_configuration in generator: + knowledge_base.ReadSystemConfigurationArtifact(system_configuration) def SetSerializersProfiler(self, serializers_profiler): """Sets the serializers profiler. @@ -441,27 +478,6 @@ def SetStorageProfiler(self, storage_profiler): """ self._storage_profiler = storage_profiler - def WritePreprocessingInformation(self, knowledge_base): - """Writes preprocessing information. - - Args: - knowledge_base (KnowledgeBase): contains the preprocessing information. - - Raises: - IOError: if the storage type does not support writing preprocess - information or the storage file is closed or read-only. - OSError: if the storage type does not support writing preprocess - information or the storage file is closed or read-only. - """ - self._RaiseIfNotWritable() - - if self.storage_type != definitions.STORAGE_TYPE_SESSION: - raise IOError('Preprocess information not supported by storage type.') - - system_configuration = knowledge_base.GetSystemConfigurationArtifact() - - self._WriteAttributeContainer(system_configuration) - def WriteSessionCompletion(self, session_completion): """Writes session completion information. @@ -481,6 +497,23 @@ def WriteSessionCompletion(self, session_completion): self._WriteAttributeContainer(session_completion) + def WriteSessionConfiguration(self, session_configuration): + """Writes session configuration information. + + Args: + session_configuration (SessionConfiguration): session configuration + information. + + Raises: + IOError: when the storage file is closed or read-only. + OSError: when the storage file is closed or read-only. + """ + self._RaiseIfNotWritable() + + if not self._HasAttributeContainers( + self._CONTAINER_TYPE_SYSTEM_CONFIGURATION): + self._WriteAttributeContainer(session_configuration) + def WriteSessionStart(self, session_start): """Writes session start information. @@ -565,7 +598,7 @@ def _DeserializeAttributeContainer(self, container_type, serialized_data): except UnicodeDecodeError as exception: raise IOError('Unable to decode serialized data: {0!s}'.format(exception)) - except (ValueError, TypeError) as exception: + except (TypeError, ValueError) as exception: # TODO: consider re-reading attribute container with error correction. raise IOError('Unable to read serialized data: {0!s}'.format(exception)) @@ -649,7 +682,7 @@ def _DeserializeAttributeContainer(self, container_type, serialized_data): except UnicodeDecodeError as exception: raise IOError('Unable to decode serialized data: {0!s}'.format(exception)) - except (ValueError, TypeError) as exception: + except (TypeError, ValueError) as exception: # TODO: consider re-reading attribute container with error correction. raise IOError('Unable to read serialized data: {0!s}'.format(exception)) @@ -825,6 +858,7 @@ def HasWarnings(self): bool: True if the store contains extraction warnings. """ + # TODO: remove, this method is kept for backwards compatibility reasons. @abc.abstractmethod def ReadSystemConfiguration(self, knowledge_base): """Reads system configuration information. @@ -1054,17 +1088,6 @@ def PrepareMergeTaskStorage(self, task): """ raise NotImplementedError() - @abc.abstractmethod - def ReadSystemConfiguration(self, knowledge_base): - """Reads system configuration information. - - The system configuration contains information about various system specific - configuration data, for example the user accounts. - - Args: - knowledge_base (KnowledgeBase): is used to store the system configuration. - """ - # pylint: disable=unused-argument def RemoveProcessedTaskStorage(self, task): """Removes a processed task storage. @@ -1093,14 +1116,6 @@ def SetStorageProfiler(self, storage_profiler): storage_profiler (StorageProfiler): storage profiler. """ - @abc.abstractmethod - def WritePreprocessingInformation(self, knowledge_base): - """Writes preprocessing information. - - Args: - knowledge_base (KnowledgeBase): contains the preprocessing information. - """ - @abc.abstractmethod def WriteSessionCompletion(self, aborted=False): """Writes session completion information. @@ -1109,6 +1124,10 @@ def WriteSessionCompletion(self, aborted=False): aborted (Optional[bool]): True if the session was aborted. """ + @abc.abstractmethod + def WriteSessionConfiguration(self): + """Writes session configuration information.""" + @abc.abstractmethod def WriteSessionStart(self): """Writes session start information.""" diff --git a/tests/engine/knowledge_base.py b/tests/engine/knowledge_base.py index ded80e74f8..35ed65c13d 100644 --- a/tests/engine/knowledge_base.py +++ b/tests/engine/knowledge_base.py @@ -183,8 +183,30 @@ def testGetHostname(self): hostname = knowledge_base_object.GetHostname() self.assertEqual(hostname, '') + def testGetSourceConfigurationArtifacts(self): + """Tests the GetSourceConfigurationArtifacts function.""" + knowledge_base_object = knowledge_base.KnowledgeBase() + + hostname_artifact = artifacts.HostnameArtifact(name='myhost.mydomain') + knowledge_base_object.SetHostname(hostname_artifact) + + user_account = artifacts.UserAccountArtifact( + identifier='1000', user_directory='/home/testuser', + username='testuser') + knowledge_base_object.AddUserAccount(user_account) + + source_configurations = ( + knowledge_base_object.GetSourceConfigurationArtifacts()) + self.assertEqual(len(source_configurations), 1) + self.assertIsNotNone(source_configurations[0]) + + system_configuration = source_configurations[0].system_configuration + self.assertIsNotNone(system_configuration) + self.assertIsNotNone(system_configuration.hostname) + self.assertEqual(system_configuration.hostname.name, 'myhost.mydomain') + def testGetSystemConfigurationArtifact(self): - """Tests the GetSystemConfigurationArtifact function.""" + """Tests the _GetSystemConfigurationArtifact function.""" knowledge_base_object = knowledge_base.KnowledgeBase() hostname_artifact = artifacts.HostnameArtifact(name='myhost.mydomain') @@ -196,7 +218,7 @@ def testGetSystemConfigurationArtifact(self): knowledge_base_object.AddUserAccount(user_account) system_configuration = ( - knowledge_base_object.GetSystemConfigurationArtifact()) + knowledge_base_object._GetSystemConfigurationArtifact()) self.assertIsNotNone(system_configuration) self.assertIsNotNone(system_configuration.hostname) self.assertEqual(system_configuration.hostname.name, 'myhost.mydomain') diff --git a/tests/engine/single_process.py b/tests/engine/single_process.py index a52b0c1ee8..fe8ed207b2 100644 --- a/tests/engine/single_process.py +++ b/tests/engine/single_process.py @@ -55,7 +55,8 @@ def testProcessSources(self): configuration.parser_filter_expression = 'filestat' test_engine.ProcessSources( - [source_path_spec], storage_writer, resolver_context, configuration) + session, [source_path_spec], storage_writer, resolver_context, + configuration) self.assertEqual(storage_writer.number_of_events, 15) diff --git a/tests/multi_processing/psort.py b/tests/multi_processing/psort.py index 6ebbb26def..3fb02f9686 100644 --- a/tests/multi_processing/psort.py +++ b/tests/multi_processing/psort.py @@ -306,6 +306,30 @@ def _CreateTestStorageFile(self, path): storage_file.Close() + def _ReadSessionConfiguration(self, path, knowledge_base_object): + """Reads session configuration information. + + The session configuration contains the system configuration, which contains + information about various system specific configuration data, for example + the user accounts. + + Args: + path (str): path. + knowledge_base_object (KnowledgeBase): is used to store the system + configuration. + """ + storage_reader = storage_factory.StorageFactory.CreateStorageReaderForFile( + path) + + for session in storage_reader.GetSessions(): + if not session.source_configurations: + storage_reader.ReadSystemConfiguration(knowledge_base_object) + else: + for source_configuration in session.source_configurations: + knowledge_base_object.ReadSystemConfigurationArtifact( + source_configuration.system_configuration, + session_identifier=session.identifier) + def testInternalAnalyzeEvents(self): """Tests the _AnalyzeEvents function.""" session = sessions.Session() @@ -318,6 +342,7 @@ def testInternalAnalyzeEvents(self): with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, 'storage.plaso') self._CreateTestStorageFile(temp_file) + self._ReadSessionConfiguration(temp_file, knowledge_base_object) storage_writer = storage_factory.StorageFactory.CreateStorageWriter( definitions.DEFAULT_STORAGE_FORMAT, session, temp_file) @@ -325,7 +350,6 @@ def testInternalAnalyzeEvents(self): storage_writer.StartTaskStorage() storage_writer.Open() - storage_writer.ReadSystemConfiguration(knowledge_base_object) # TODO: implement, this currently loops infinite. # test_engine._AnalyzeEvents(storage_writer, [test_plugin]) @@ -336,6 +360,7 @@ def testInternalAnalyzeEvents(self): with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, 'storage.plaso') self._CreateTestStorageFile(temp_file) + self._ReadSessionConfiguration(temp_file, knowledge_base_object) storage_writer = storage_factory.StorageFactory.CreateStorageWriter( definitions.DEFAULT_STORAGE_FORMAT, session, temp_file) @@ -343,7 +368,6 @@ def testInternalAnalyzeEvents(self): storage_writer.StartTaskStorage() storage_writer.Open() - storage_writer.ReadSystemConfiguration(knowledge_base_object) # TODO: implement, this currently loops infinite. _ = test_engine @@ -376,10 +400,10 @@ def testInternalExportEvents(self): with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, 'storage.plaso') self._CreateTestStorageFile(temp_file) + self._ReadSessionConfiguration(temp_file, knowledge_base_object) storage_reader = ( storage_factory.StorageFactory.CreateStorageReaderForFile(temp_file)) - storage_reader.ReadSystemConfiguration(knowledge_base_object) test_engine._ExportEvents( storage_reader, output_module, deduplicate_events=False) @@ -409,10 +433,10 @@ def testInternalExportEventsDeduplicate(self): with shared_test_lib.TempDirectory() as temp_directory: temp_file = os.path.join(temp_directory, 'storage.plaso') self._CreateTestStorageFile(temp_file) + self._ReadSessionConfiguration(temp_file, knowledge_base_object) storage_reader = ( storage_factory.StorageFactory.CreateStorageReaderForFile(temp_file)) - storage_reader.ReadSystemConfiguration(knowledge_base_object) test_engine._ExportEvents(storage_reader, output_module) diff --git a/tests/multi_processing/task_engine.py b/tests/multi_processing/task_engine.py index c0fda76fe2..0d9c88e3ce 100644 --- a/tests/multi_processing/task_engine.py +++ b/tests/multi_processing/task_engine.py @@ -58,7 +58,7 @@ def testProcessSources(self): storage_writer = sqlite_writer.SQLiteStorageFileWriter(session, temp_file) test_engine.ProcessSources( - session.identifier, [source_path_spec], storage_writer, configuration) + session, [source_path_spec], storage_writer, configuration) # TODO: implement a way to obtain the results without relying # on multi-process primitives e.g. by writing to a file. diff --git a/tests/output/test_lib.py b/tests/output/test_lib.py index b12a192b48..e48649baa1 100644 --- a/tests/output/test_lib.py +++ b/tests/output/test_lib.py @@ -82,7 +82,15 @@ def _CreateOutputMediator(self, storage_file=None): knowledge_base_object = knowledge_base.KnowledgeBase() if storage_file: - storage_file.ReadSystemConfiguration(knowledge_base_object) + # TODO: clean up + for session in storage_file.GetSessions(): + if not session.source_configurations: + storage_file.ReadSystemConfiguration(knowledge_base_object) + else: + for source_configuration in session.source_configurations: + knowledge_base_object.ReadSystemConfigurationArtifact( + source_configuration.system_configuration, + session_identifier=session.identifier) formatter_mediator = formatters_mediator.FormatterMediator() output_mediator = mediator.OutputMediator( diff --git a/tests/storage/sqlite/sqlite_file.py b/tests/storage/sqlite/sqlite_file.py index 25e170994e..2943757a0b 100644 --- a/tests/storage/sqlite/sqlite_file.py +++ b/tests/storage/sqlite/sqlite_file.py @@ -532,12 +532,13 @@ def testGetSortedEvents(self): # TODO: add tests for Open and Close # TODO: add tests for ReadSystemConfiguration - # TODO: add tests for WritePreprocessingInformation - def testWriteSessionStartAndCompletion(self): - """Tests the WriteSessionStart and WriteSessionCompletion functions.""" + def testWriteSessionStartConfigurationAndCompletion(self): + """Tests the WriteSessionStart, Configuration and Completion functions.""" session = sessions.Session() session_start = sessions.SessionStart(identifier=session.identifier) + session_configuration = sessions.SessionConfiguration( + identifier=session.identifier) session_completion = sessions.SessionCompletion( identifier=session.identifier) @@ -548,6 +549,7 @@ def testWriteSessionStartAndCompletion(self): storage_file.Open(path=temp_file, read_only=False) storage_file.WriteSessionStart(session_start) + storage_file.WriteSessionConfiguration(session_configuration) storage_file.WriteSessionCompletion(session_completion) storage_file.Close()