Skip to content

Commit

Permalink
Worked on attribute container identifiers log2timeline#771
Browse files Browse the repository at this point in the history
  • Loading branch information
@joachimmetz
joachimmetz committed Aug 10, 2016
1 parent 89e4542 commit 507542d
Show file tree
Hide file tree
Showing 14 changed files with 100 additions and 168 deletions.
4 changes: 4 additions & 0 deletions plaso/storage/gzip_file.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,10 @@ def _WriteAttributeContainer(self, attribute_container):
if self._read_only:
raise IOError(u'Unable to write to read-only storage file.')

attribute_container_identifier = identifiers.SerializedStreamIdentifier(
1, len(self._attribute_containers))
attribute_container.SetIdentifier(attribute_container_identifier)

attribute_container_data = self._SerializeAttributeContainer(
attribute_container)
self._gzip_file.write(attribute_container_data)
Expand Down
4 changes: 3 additions & 1 deletion plaso/storage/zip_file.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2050,7 +2050,9 @@ def AddAnalysisReport(self, analysis_report):
if self._read_only:
raise IOError(u'Unable to write to read-only storage file.')

# TODO: add SetIdentifier.
analysis_report_identifier = identifiers.SerializedStreamIdentifier(
self._last_analysis_report, 0)
analysis_report.SetIdentifier(analysis_report_identifier)

stream_name = u'analysis_report_data.{0:06}'.format(
self._last_analysis_report)
Expand Down
12 changes: 6 additions & 6 deletions test_data/end_to_end/dynamic.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ datetime,timestamp_desc,source,source_long,message,parser,display_name,tag
2016-02-29T01:15:43+00:00,Content Modification Time,LOG,Log File,testing leap year in parsing events take place in 2012,syslog,OS:/tmp/test/test_data/syslog,-
2016-03-23T23:01:18+00:00,Content Modification Time,LOG,Log File,[somrandomexe pid: 19] This syslog message has a fractional value for seconds.,syslog,OS:/tmp/test/test_data/syslog,-
2016-03-23T23:01:18+00:00,Content Modification Time,LOG,Log File,[somrandomexe pid: 1915] This syslog message is brought to you by me (and not the other guy),syslog,OS:/tmp/test/test_data/syslog,-
2016-07-18T05:37:35+00:00,atime,FILE,OS atime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-07-18T05:37:35+00:00,ctime,FILE,OS ctime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-07-18T05:37:35+00:00,ctime,FILE,OS ctime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-07-18T05:37:35+00:00,mtime,FILE,OS mtime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-07-18T05:37:35+00:00,mtime,FILE,OS mtime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-07-18T05:37:36+00:00,atime,FILE,OS atime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-08-08T19:24:34+00:00,ctime,FILE,OS ctime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-08-08T19:24:34+00:00,ctime,FILE,OS ctime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-08-08T19:24:34+00:00,mtime,FILE,OS mtime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-08-08T19:24:34+00:00,mtime,FILE,OS mtime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-08-08T19:24:35+00:00,atime,FILE,OS atime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-08-08T19:24:35+00:00,atime,FILE,OS atime,OS:/tmp/test/test_data/syslog Type: file,filestat,OS:/tmp/test/test_data/syslog,-
2016-11-18T01:15:20+00:00,Content Modification Time,LOG,Log File,[aprocess pid: 10100] This is a multi-line message that screws up many syslog parsers.,syslog,OS:/tmp/test/test_data/syslog,-
2016-11-18T01:15:43+00:00,Content Modification Time,LOG,Log File,last message repeated 5 times,syslog,OS:/tmp/test/test_data/syslog,repeated
2016-12-18T17:54:32+00:00,Content Modification Time,LOG,Log File,[anacron pid: 1234] No true exit can exist (124 job run),syslog,OS:/tmp/test/test_data/syslog,-
Expand Down
38 changes: 19 additions & 19 deletions test_data/end_to_end/json.log
View file

Large diffs are not rendered by default.

38 changes: 19 additions & 19 deletions test_data/end_to_end/json_line.log
View file

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions test_data/end_to_end/l2tcsv.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ date,time,timezone,MACB,source,sourcetype,type,user,host,short,desc,version,file
02/29/2016,01:15:43,UTC,M...,LOG,Log File,Content Modification Time,-,-,testing leap year in parsing events take place in 2012,testing leap year in parsing events take place in 2012,2,OS:/tmp/test/test_data/syslog,-,-,syslog,sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
03/23/2016,23:01:18,UTC,M...,LOG,Log File,Content Modification Time,-,myhostname.myhost.com,[somrandomexe pid: 19] This syslog message has a fractional value for seconds.,[somrandomexe pid: 19] This syslog message has a fractional value for seconds.,2,OS:/tmp/test/test_data/syslog,-,-,syslog,sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
03/23/2016,23:01:18,UTC,M...,LOG,Log File,Content Modification Time,-,myhostname.myhost.com,[somrandomexe pid: 1915] This syslog message is brought to you by me (and no...,[somrandomexe pid: 1915] This syslog message is brought to you by me (and not the other guy),2,OS:/tmp/test/test_data/syslog,-,-,syslog,sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
07/18/2016,05:37:35,UTC,.A..,FILE,OS atime,atime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
07/18/2016,05:37:35,UTC,..C.,FILE,OS ctime,ctime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
07/18/2016,05:37:35,UTC,..C.,FILE,OS ctime,ctime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
07/18/2016,05:37:35,UTC,M...,FILE,OS mtime,mtime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
07/18/2016,05:37:35,UTC,M...,FILE,OS mtime,mtime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
07/18/2016,05:37:36,UTC,.A..,FILE,OS atime,atime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
08/08/2016,19:24:34,UTC,..C.,FILE,OS ctime,ctime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
08/08/2016,19:24:34,UTC,..C.,FILE,OS ctime,ctime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
08/08/2016,19:24:34,UTC,M...,FILE,OS mtime,mtime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
08/08/2016,19:24:34,UTC,M...,FILE,OS mtime,mtime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
08/08/2016,19:24:35,UTC,.A..,FILE,OS atime,atime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
08/08/2016,19:24:35,UTC,.A..,FILE,OS atime,atime,-,-,/tmp/test/test_data/syslog,OS:/tmp/test/test_data/syslog Type: file,2,OS:/tmp/test/test_data/syslog,-,-,filestat,file_size: (1270 ) file_system_type: OS is_allocated: True sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
11/18/2016,01:15:20,UTC,M...,LOG,Log File,Content Modification Time,-,myhostname.myhost.com,[aprocess pid: 10100] This is a multi-line message that screws up many syslo...,[aprocess pid: 10100] This is a multi-line message that screws up many syslog parsers.,2,OS:/tmp/test/test_data/syslog,-,-,syslog,sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
11/18/2016,01:15:43,UTC,M...,LOG,Log File,Content Modification Time,-,-,last message repeated 5 times,last message repeated 5 times,2,OS:/tmp/test/test_data/syslog,-,repeated,syslog,sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
12/18/2016,17:54:32,UTC,M...,LOG,Log File,Content Modification Time,-,myhostname.myhost.com,[anacron pid: 1234] No true exit can exist (124 job run),[anacron pid: 1234] No true exit can exist (124 job run),2,OS:/tmp/test/test_data/syslog,-,-,syslog,sha256_hash: fe86fd5cf0680b86a0518c19f01098e5f890c7c9718d1510a755298cfc748ce5
Expand Down
12 changes: 6 additions & 6 deletions test_data/end_to_end/l2ttln.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ Time|Source|Host|User|Description|TZ|Notes
1456708543|LOG|-|-|2016-02-29T01:15:43+00:00; Content Modification Time; testing leap year in parsing, events take place in 2012|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1458774078|LOG|myhostname.myhost.com|-|2016-03-23T23:01:18+00:00; Content Modification Time; [somrandomexe, pid: 19] This syslog message has a fractional value for seconds.|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1458774078|LOG|myhostname.myhost.com|-|2016-03-23T23:01:18+00:00; Content Modification Time; [somrandomexe, pid: 1915] This syslog message is brought to you by me (and not the other guy)|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1468820255|FILE|-|-|2016-07-18T05:37:35+00:00; atime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1468820255|FILE|-|-|2016-07-18T05:37:35+00:00; ctime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1468820255|FILE|-|-|2016-07-18T05:37:35+00:00; ctime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1468820255|FILE|-|-|2016-07-18T05:37:35+00:00; mtime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1468820255|FILE|-|-|2016-07-18T05:37:35+00:00; mtime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1468820256|FILE|-|-|2016-07-18T05:37:36+00:00; atime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1470684274|FILE|-|-|2016-08-08T19:24:34+00:00; ctime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1470684274|FILE|-|-|2016-08-08T19:24:34+00:00; ctime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1470684274|FILE|-|-|2016-08-08T19:24:34+00:00; mtime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1470684274|FILE|-|-|2016-08-08T19:24:34+00:00; mtime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1470684275|FILE|-|-|2016-08-08T19:24:35+00:00; atime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1470684275|FILE|-|-|2016-08-08T19:24:35+00:00; atime; OS:/tmp/test/test_data/syslog Type: file|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1479431720|LOG|myhostname.myhost.com|-|2016-11-18T01:15:20+00:00; Content Modification Time; [aprocess, pid: 10100] This is a multi-line message that screws up many syslog parsers.|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1479431743|LOG|-|-|2016-11-18T01:15:43+00:00; Content Modification Time; last message repeated 5 times|UTC|File: OS:/tmp/test/test_data/syslog inode: -
1482083672|LOG|myhostname.myhost.com|-|2016-12-18T17:54:32+00:00; Content Modification Time; [anacron, pid: 1234] No true exit can exist (124 job run)|UTC|File: OS:/tmp/test/test_data/syslog inode: -
Expand Down
Loading

0 comments on commit 507542d

Please sign in to comment.