From f4a9a7b5af98341d10b608709f0abe659066fd65 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Thu, 6 Apr 2017 06:54:33 +0200 Subject: [PATCH] Clean up after adding attribute container identifiers #771 --- plaso/containers/events.py | 2 -- plaso/containers/reports.py | 2 +- plaso/engine/knowledge_base.py | 5 ++++- plaso/lib/definitions.py | 5 +---- plaso/multi_processing/analysis_process.py | 4 ++++ plaso/output/dynamic.py | 26 +++++----------------- plaso/output/event_buffer.py | 7 +++--- plaso/output/interface.py | 26 +++++----------------- plaso/output/mediator.py | 6 ++--- tests/analysis/nsrlsvr.py | 2 -- tests/analysis/tagging.py | 1 + tests/analysis/viper.py | 3 +-- tests/analysis/virustotal.py | 3 +-- tests/output/elastic.py | 9 ++++---- tests/output/l2t_csv.py | 7 +++--- tools/preg.py | 6 ++--- 16 files changed, 39 insertions(+), 75 deletions(-) diff --git a/plaso/containers/events.py b/plaso/containers/events.py index 19d3c29d84..882e2d56a8 100644 --- a/plaso/containers/events.py +++ b/plaso/containers/events.py @@ -63,8 +63,6 @@ class EventObject(interface.AttributeContainer): def __init__(self): """Initializes an event object.""" super(EventObject, self).__init__() - self._store_index = None - self._store_number = None self.data_type = self.DATA_TYPE self.display_name = None self.filename = None diff --git a/plaso/containers/reports.py b/plaso/containers/reports.py index cd036698b2..e7522703b4 100644 --- a/plaso/containers/reports.py +++ b/plaso/containers/reports.py @@ -10,7 +10,7 @@ class AnalysisReport(interface.AttributeContainer): """Class to represent an analysis report attribute container. Attributes: - filter_string (str): ??? + filter_string (str): event filter expression. plugin_name (str): name of the analysis plugin that generated the report. report_array (array[str]): ??? report_dict (dict[str]): ??? diff --git a/plaso/engine/knowledge_base.py b/plaso/engine/knowledge_base.py index b1cdd4b370..ee66b83d8b 100644 --- a/plaso/engine/knowledge_base.py +++ b/plaso/engine/knowledge_base.py @@ -282,6 +282,9 @@ def ReadSystemConfigurationArtifact( user_account.username: user_account for user_account in system_configuration.user_accounts} + if not system_configuration.time_zone: + return + try: self.SetTimeZone(system_configuration.time_zone) except ValueError: @@ -333,7 +336,7 @@ def SetTimeZone(self, time_zone): try: self._time_zone = pytz.timezone(time_zone) except (AttributeError, pytz.UnknownTimeZoneError): - raise ValueError(u'Unsupported timezone: {0:s}'.format(time_zone)) + raise ValueError(u'Unsupported timezone: {0!s}'.format(time_zone)) def SetValue(self, identifier, value): """Sets a value by identifier. diff --git a/plaso/lib/definitions.py b/plaso/lib/definitions.py index 569a21f57a..e38ebcdd37 100644 --- a/plaso/lib/definitions.py +++ b/plaso/lib/definitions.py @@ -36,8 +36,6 @@ PROCESSING_STATUS_KILLED]) RESERVED_VARIABLE_NAMES = frozenset([ - u'_store_index', - u'_store_number', u'body', u'data_type', u'display_name', @@ -59,8 +57,7 @@ u'timestamp', u'timestamp_desc', u'timezone', - u'username', - u'uuid']) + u'username']) SERIALIZER_FORMAT_JSON = u'json' diff --git a/plaso/multi_processing/analysis_process.py b/plaso/multi_processing/analysis_process.py index 96200da3c2..a1062c288f 100644 --- a/plaso/multi_processing/analysis_process.py +++ b/plaso/multi_processing/analysis_process.py @@ -208,6 +208,8 @@ def _ProcessEvent(self, mediator, event): self._analysis_plugin.ExamineEvent(mediator, event) except Exception as exception: # pylint: disable=broad-except + self.SignalAbort() + # TODO: write analysis error. if self._debug_output: @@ -219,3 +221,5 @@ def SignalAbort(self): self._abort = True if self._foreman_status_wait_event: self._foreman_status_wait_event.set() + if self._analysis_mediator: + self._analysis_mediator.SignalAbort() diff --git a/plaso/output/dynamic.py b/plaso/output/dynamic.py index 6148c4525e..3bee1e34e6 100644 --- a/plaso/output/dynamic.py +++ b/plaso/output/dynamic.py @@ -291,23 +291,6 @@ def _FormatZone(self, unused_event): """ return self._output_mediator.timezone - def _GetEventStorageIdentifier(self, event): - """Retrieves the event storage identifier of an event. - - Args: - event (EventObject): event. - - Returns: - str: event storage identifier or "N/A". - """ - store_number = getattr(event, u'_store_number', None) - store_index = getattr(event, u'_store_index', None) - - if store_number is None or store_index is None: - return u'N/A' - - return u'{0:d}:{1:d}'.format(store_number, store_index) - def _ReportEventError(self, event, error_message): """Reports an event related error. @@ -315,12 +298,13 @@ def _ReportEventError(self, event, error_message): event (EventObject): event. error_message: a string containing the error message. """ - event_storage_identifier = self._GetEventStorageIdentifier(event) + event_identifier = event.GetIdentifier() + event_identifier_string = event_identifier.CopyToString() error_message = ( - u'Event: {0:s} data type: {1:s} display name: {2:s} ' + u'Event: {0!s} data type: {1:s} display name: {2:s} ' u'parser chain: {3:s} with error: {4:s}').format( - event_storage_identifier, event.data_type, - event.display_name, event.parser, error_message) + event_identifier_string, event.data_type, event.display_name, + event.parser, error_message) logging.error(error_message) def GetFormattedField(self, event, field_name): diff --git a/plaso/output/event_buffer.py b/plaso/output/event_buffer.py index b6dbf43d2a..cd6be5a0a3 100644 --- a/plaso/output/event_buffer.py +++ b/plaso/output/event_buffer.py @@ -41,9 +41,9 @@ def PushEvent(self, event): event (EventObject): event. """ event_identifier = event.GetIdentifier() - event_identifier = event_identifier.CopyToString() + event_identifier_string = event_identifier.CopyToString() heap_values = ( - event.timestamp, event.timestamp_desc, event_identifier, event) + event.timestamp, event.timestamp_desc, event_identifier_string, event) heapq.heappush(self._heap, heap_values) def PushEvents(self, events): @@ -81,8 +81,7 @@ class EventBuffer(object): u'parser', u'pathspec', u'tag', - u'timestamp', - u'uuid']) + u'timestamp']) def __init__(self, output_module, check_dedups=True): """Initializes an event buffer object. diff --git a/plaso/output/interface.py b/plaso/output/interface.py index 01524101a1..f118f5c0e6 100644 --- a/plaso/output/interface.py +++ b/plaso/output/interface.py @@ -26,23 +26,6 @@ def __init__(self, output_mediator): super(OutputModule, self).__init__() self._output_mediator = output_mediator - def _GetEventStorageIdentifier(self, event): - """Retrieves the event storage identifier of an event object. - - Args: - event (EventObject): event. - - Returns: - str: event storage identifier or "N/A". - """ - store_number = getattr(event, u'_store_number', None) - store_index = getattr(event, u'_store_index', None) - - if store_number is None or store_index is None: - return u'N/A' - - return u'{0:d}:{1:d}'.format(store_number, store_index) - def _ReportEventError(self, event, error_message): """Reports an event related error. @@ -50,12 +33,13 @@ def _ReportEventError(self, event, error_message): event (EventObject): event. error_message (str): error message. """ - event_storage_identifier = self._GetEventStorageIdentifier(event) + event_identifier = event.GetIdentifier() + event_identifier_string = event_identifier.CopyToString() error_message = ( - u'Event: {0:s} data type: {1:s} display name: {2:s} ' + u'Event: {0!s} data type: {1:s} display name: {2:s} ' u'parser chain: {3:s} with error: {4:s}').format( - event_storage_identifier, event.data_type, - event.display_name, event.parser, error_message) + event_identifier_string, event.data_type, event.display_name, + event.parser, error_message) logging.error(error_message) def Close(self): diff --git a/plaso/output/mediator.py b/plaso/output/mediator.py index f9f4306b1a..9e11e91b4c 100644 --- a/plaso/output/mediator.py +++ b/plaso/output/mediator.py @@ -133,8 +133,7 @@ def GetHostname(self, event, default_hostname=u'-'): if hostname: return hostname - # TODO: replace store_number by session_identifier. - session_identifier = getattr(event, u'_store_number', None) + session_identifier = event.GetSessionIdentifier() if session_identifier is None: return default_hostname @@ -234,8 +233,7 @@ def GetUsername(self, event, default_username=u'-'): if username and username != u'-': return username - # TODO: replace store_number by session_identifier. - session_identifier = getattr(event, u'_store_number', None) + session_identifier = event.GetSessionIdentifier() if session_identifier is None: return default_username diff --git a/tests/analysis/nsrlsvr.py b/tests/analysis/nsrlsvr.py index 29bfbf6f4e..8bed2a0234 100644 --- a/tests/analysis/nsrlsvr.py +++ b/tests/analysis/nsrlsvr.py @@ -53,7 +53,6 @@ class NsrlSvrTest(test_lib.AnalysisPluginTestCase): {u'timestamp': timelib.Timestamp.CopyFromString(u'2015-01-01 17:00:00'), u'timestamp_desc': eventdata.EventTimestamp.CREATION_TIME, u'sha256_hash': EVENT_1_HASH, - u'uuid': u'8', u'data_type': u'fs:stat', u'pathspec': fake_path_spec.FakePathSpec( location=u'C:\\WINDOWS\\system32\\good.exe') @@ -61,7 +60,6 @@ class NsrlSvrTest(test_lib.AnalysisPluginTestCase): {u'timestamp': timelib.Timestamp.CopyFromString(u'2016-01-01 17:00:00'), u'timestamp_desc': eventdata.EventTimestamp.CREATION_TIME, u'sha256_hash': _EVENT_2_HASH, - u'uuid': u'9', u'data_type': u'fs:stat:ntfs', u'pathspec': fake_path_spec.FakePathSpec( location=u'C:\\WINDOWS\\system32\\evil.exe')}] diff --git a/tests/analysis/tagging.py b/tests/analysis/tagging.py index 8a3c1d56fa..00091dee43 100644 --- a/tests/analysis/tagging.py +++ b/tests/analysis/tagging.py @@ -113,6 +113,7 @@ def testExamineEventAndCompileReport(self): labels = [] for event_tag in storage_writer.event_tags: labels.extend(event_tag.labels) + self.assertEqual(len(labels), 5) # This is from a tag rule declared in objectfilter syntax. diff --git a/tests/analysis/viper.py b/tests/analysis/viper.py index 4a23ded727..0a8fba3fca 100644 --- a/tests/analysis/viper.py +++ b/tests/analysis/viper.py @@ -37,8 +37,7 @@ class ViperTest(test_lib.AnalysisPluginTestCase): _TEST_EVENTS = [{ u'timestamp': timelib.Timestamp.CopyFromString(u'2015-01-01 17:00:00'), - u'sha256_hash': _EVENT_1_HASH, - u'uuid': u'8'}] + u'sha256_hash': _EVENT_1_HASH}] def _MockPost(self, unused_url, data=None): """Mock funtion to simulate a Viper API request. diff --git a/tests/analysis/virustotal.py b/tests/analysis/virustotal.py index 908d23722c..c40b291aac 100644 --- a/tests/analysis/virustotal.py +++ b/tests/analysis/virustotal.py @@ -38,8 +38,7 @@ class VirusTotalTest(test_lib.AnalysisPluginTestCase): _TEST_EVENTS = [{ u'timestamp': timelib.Timestamp.CopyFromString(u'2015-01-01 17:00:00'), - u'sha256_hash': _EVENT_1_HASH, - u'uuid': u'8'}] + u'sha256_hash': _EVENT_1_HASH}] def _MockGet(self, url, params): """Mock function to simulate a VirusTotal API request. diff --git a/tests/output/elastic.py b/tests/output/elastic.py index 188d29d7f8..fb78422c8c 100644 --- a/tests/output/elastic.py +++ b/tests/output/elastic.py @@ -44,18 +44,19 @@ class ElasticSearchHelperTest(test_lib.OutputModuleTestCase): def testEventToDict(self): """Tests the _EventToDict function.""" - event_timestamp = timelib.Timestamp.CopyFromString( - u'2012-06-27 18:17:01+00:00') label = u'Test' - event = ElasticTestEvent(event_timestamp) event_tag = events.EventTag() event_tag.AddLabel(label) + + event_timestamp = timelib.Timestamp.CopyFromString( + u'2012-06-27 18:17:01+00:00') + event = ElasticTestEvent(event_timestamp) event.tag = event_tag output_mediator = self._CreateOutputMediator() - elasticsearch_helper = elastic.ElasticSearchHelper( output_mediator, u'127.0.0.1', 9200, 1000, u'test', {}, u'test_type') + expected_dict = { u'data_type': u'syslog:line', u'datetime': u'2012-06-27T18:17:01+00:00', diff --git a/tests/output/l2t_csv.py b/tests/output/l2t_csv.py index 9fe212ea91..a43195b276 100644 --- a/tests/output/l2t_csv.py +++ b/tests/output/l2t_csv.py @@ -52,7 +52,6 @@ def setUp(self): self._output_writer = cli_test_lib.TestOutputWriter() self._formatter = l2t_csv.L2TCSVOutputModule(output_mediator) self._formatter.SetOutputWriter(self._output_writer) - self._event_object = L2TTestEvent() def testWriteHeader(self): """Tests the WriteHeader function.""" @@ -73,8 +72,10 @@ def testWriteEventBody(self): event_tag = events.EventTag() event_tag.AddLabels([u'Malware', u'Printed']) - self._event_object.tag = event_tag - self._formatter.WriteEventBody(self._event_object) + event = L2TTestEvent() + event.tag = event_tag + + self._formatter.WriteEventBody(event) expected_event_body = ( b'06/27/2012,18:17:01,UTC,M...,LOG,Syslog,Content Modification Time,-,' diff --git a/tools/preg.py b/tools/preg.py index 33ec10d7a1..1e97ee8c9e 100755 --- a/tools/preg.py +++ b/tools/preg.py @@ -117,16 +117,14 @@ class PregTool(storage_media_tool.StorageMediaTool): RUN_MODE_REG_KEY = 5 _EXCLUDED_ATTRIBUTE_NAMES = frozenset([ - u'_store_index', - u'_store_number', u'data_type', u'display_name', u'filename', u'inode', + u'parser', u'pathspec', u'tag', - u'timestamp', - u'uuid']) + u'timestamp']) def __init__(self, input_reader=None, output_writer=None): """Initializes the CLI tool object.