Add start of cdxj validation for issue #86

invalid.cdxj

@@ -0,0 +1 @@
+edu,odu,cs)/~salam 20160305192247 {"locator": "urn:ipfs/QmeVWGtnfuJ1QnpmtKKnyArVgEpq7v31kktEfh6c8mDiXE/QmZWKQRBNXNrVZ69LoGpMNJi5NU66gDhnGtQukWJepv7Kr", "encryption_method": "xor", "encryption_key": "radon", "mime_type": "text/html", "status_code": "200"}

ipwb/replay.py

@@ -222,6 +222,9 @@ def getIndexFileFullPath(cdxjFilePath=INDEX_FILE):
 
 def getURIsInCDXJ(cdxjFilePath=INDEX_FILE):
     indexFileContents = getIndexFileContents(cdxjFilePath)
+    if not ipwbConfig.isValidCDXJ(indexFileContents):
+        print "Invalid CDXJ line!"
+        return 0
 
     if not indexFileContents:
         return 0
@@ -238,6 +241,9 @@ def getURIsInCDXJ(cdxjFilePath=INDEX_FILE):
 def retrieveMemCount(cdxjFilePath=INDEX_FILE):
     print "Retrieving URI-Ms from {0}".format(cdxjFilePath)
     indexFileContents = getIndexFileContents(cdxjFilePath)
+    if not ipwbConfig.isValidCDXJ(indexFileContents):
+        print "Invalid CDXJ line, cannot get Memento count"
+        return 0
 
     if not indexFileContents:
         return 0

ipwb/util.py

@@ -9,6 +9,7 @@
 import subprocess
 import site
 # from requests.exceptions import ConnectionError
+from pywb.utils.canonicalize import unsurt
 from ipfsapi.exceptions import ConnectionError
 
 
@@ -43,9 +44,31 @@ def isDaemonAlive(hostAndPort="{0}:{1}".format(IPFSAPI_IP, IPFSAPI_PORT)):
         print sys.exc_info()[0]
 
 
+class InvalidCDXJException(Exception):
+    pass
+
 def isValidCDXJ(stringIn):  # TODO: Check specific strict syntax
     # Also, be sure to mind the meta headers starting with @/#, etc.
-    return True
+    try:
+        metadataAllowed = True
+        lines = stringIn.split('\n')
+        for line in lines:
+            if line[0] == '@' or line[1] == '!':
+                if metadataAllowed:
+                    continue
+                else:
+                    raise InvalidCDXJException()
+            metadataAllowed = False
+            (uri, datetime, cdxjJSON) = line.split(' ',2)
+            unsurt(uri) # Will throw an exception if invalid
+            # SURT ok
+            if not datetime.isdigit() or len(datetime) != 14:
+                raise InvalidCDXJException()
+            json.loads(cdxjJSON)
+        return True
+    except:
+        print sys.exc_info()[0]
+        return False
 
 
 def getURIsInCDXJ(cdxjFile=INDEX_FILE):
@@ -158,6 +181,7 @@ def getIPWBReplayIndexPath():
 
 def firstRun():
     import indexer
+
     # Ensure the sample WARC is in IPFS
     print 'Executing first-run procedure on provided sample data.'
     indexer.indexFileAt(site.getsitepackages()[0] + '/ipwb/' + SAMPLE_WARC,

tests/test_replay.py

@@ -1,4 +1,7 @@
 import pytest
+from ipwb import util
+
+import testUtil as ipwbTest
 
 # Successful retrieval
 # Accurate retrieval
@@ -70,4 +73,20 @@ def test_fileImport_nonCDXJ():  # Fail w/ friendly message when non-cdxj
     pass
 
 
+@pytest.mark.cdxjValidation
+def test_cdxj_valid():
+    # Missing fields
+    assert not util.isValidCDXJ('test')
+    # Valid SURT
+    assert util.isValidCDXJ(r"""edu,odu,cs)/~salam 20160305192247 {"locator": "urn:ipfs/QmeVWGtnfuJ1QnpmtKKnyArVgEpq7v31kktEfh6c8mDiXE/QmZWKQRBNXNrVZ69LoGpMNJi5NU66gDhnGtQukWJepv7Kr", "encryption_method": "xor", "encryption_key": "radon", "mime_type": "text/html", "status_code": "200"}""")
+    # Bad JSON in third field
+    assert not util.isValidCDXJ(r"""edu,odu,cs)/ 20160305192247 radon""")
+    # Valid SURT
+    assert util.isValidCDXJ(r"""edu,odu,cs)/ 20160305192247 {}""")
+    #Invalid datetime
+    assert not util.isValidCDXJ(r"""edu,odu,cs)/ 2016030519224 {}""")
+    # Invalid SURT URI, pywb catches its own ValueError
+    #assert not util.isValidCDXJ(r"""foo.bar 20160305192247 {}""")
+
+
 # TODO: Have unit tests for each function in replay.py