feat/sharepoint path support (#387)

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.5.4
+
+### Enhancements
+
+* **Sharepoint support for nested folders and remove need for default path Shared Documents**
+
 ## 0.5.3
 
 ### Enhancements

test/integration/connectors/expected_results/sharepoint/directory_structure.json

@@ -1,8 +1,5 @@
 {
   "directory_structure": [
-    "Folder1/Folder2/fake-email.txt",
-    "Folder1/fake-memo.pdf",
-    "book-war-and-peace-1p.txt",
-    "list-item-example.pdf"
+    "fake-memo.pdf"
   ]
 }

test/integration/connectors/expected_results/sharepoint/file_data/01QKP26QZL5KBVQTQ3IRDYF72MRH2QKKR3.json

@@ -0,0 +1,33 @@
+{
+  "identifier": "01QKP26QZL5KBVQTQ3IRDYF72MRH2QKKR3",
+  "connector_type": "sharepoint",
+  "source_identifiers": {
+    "filename": "fake-memo.pdf",
+    "fullpath": "e2e-test-folder/fake-memo.pdf",
+    "rel_path": "fake-memo.pdf"
+  },
+  "metadata": {
+    "url": "/drive/root:/e2e-test-folder/fake-memo.pdf",
+    "version": "\"{5883EA2B-1B4E-4744-82FF-4C89F5052A3B},1\"",
+    "record_locator": {
+      "user_pname": "[email protected]",
+      "server_relative_path": "e2e-test-folder/fake-memo.pdf"
+    },
+    "date_created": "1738129296.0",
+    "date_modified": "1738129296.0",
+    "date_processed": "1739549929.526217",
+    "permissions_data": null,
+    "filesize_bytes": null
+  },
+  "additional_metadata": {
+    "eTag": "\"{5883EA2B-1B4E-4744-82FF-4C89F5052A3B},1\"",
+    "id": "01QKP26QZL5KBVQTQ3IRDYF72MRH2QKKR3",
+    "name": "fake-memo.pdf",
+    "webUrl": "https://unstructuredio.sharepoint.com/Shared%20Documents/e2e-test-folder/fake-memo.pdf",
+    "cTag": "\"c:{5883EA2B-1B4E-4744-82FF-4C89F5052A3B},1\"",
+    "size": 13374
+  },
+  "reprocess": false,
+  "local_download_path": "/private/var/folders/85/f389rtdn2c971nv4r3d31d740000gn/T/tmpxs1_nhhe/fake-memo.pdf",
+  "display_name": null
+}

test/integration/connectors/expected_results/sharepoint1/directory_structure.json

@@ -0,0 +1,8 @@
+{
+  "directory_structure": [
+    "Folder1/Folder2/fake-email.txt",
+    "Folder1/fake-memo.pdf",
+    "book-war-and-peace-1p.txt",
+    "list-item-example.pdf"
+  ]
+}

test/integration/connectors/expected_results/sharepoint1/file_data/0153RHRSABDC6JJIUHKJF2C5ZFGJF3C6XJ.json

@@ -0,0 +1,33 @@
+{
+  "identifier": "0153RHRSABDC6JJIUHKJF2C5ZFGJF3C6XJ",
+  "connector_type": "sharepoint",
+  "source_identifiers": {
+    "filename": "list-item-example.pdf",
+    "fullpath": "/list-item-example.pdf",
+    "rel_path": "list-item-example.pdf"
+  },
+  "metadata": {
+    "url": "/drive/root:/list-item-example.pdf",
+    "version": "\"{94BC1801-87A2-4B52-A177-25324BB17AE9},1\"",
+    "record_locator": {
+      "user_pname": "[email protected]",
+      "server_relative_path": "/list-item-example.pdf"
+    },
+    "date_created": "1738364795.0",
+    "date_modified": "1738364795.0",
+    "date_processed": "1739552937.759603",
+    "permissions_data": null,
+    "filesize_bytes": null
+  },
+  "additional_metadata": {
+    "eTag": "\"{94BC1801-87A2-4B52-A177-25324BB17AE9},1\"",
+    "id": "0153RHRSABDC6JJIUHKJF2C5ZFGJF3C6XJ",
+    "name": "list-item-example.pdf",
+    "webUrl": "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source/Shared%20Documents/list-item-example.pdf",
+    "cTag": "\"c:{94BC1801-87A2-4B52-A177-25324BB17AE9},1\"",
+    "size": 48981
+  },
+  "reprocess": false,
+  "local_download_path": "/private/var/folders/85/f389rtdn2c971nv4r3d31d740000gn/T/tmpf70nr_lt/list-item-example.pdf",
+  "display_name": null
+}

test/integration/connectors/expected_results/sharepoint1/file_data/0153RHRSAVBNSXPKVIBZC3ZD53KENINVGG.json

@@ -0,0 +1,33 @@
+{
+  "identifier": "0153RHRSAVBNSXPKVIBZC3ZD53KENINVGG",
+  "connector_type": "sharepoint",
+  "source_identifiers": {
+    "filename": "book-war-and-peace-1p.txt",
+    "fullpath": "/book-war-and-peace-1p.txt",
+    "rel_path": "book-war-and-peace-1p.txt"
+  },
+  "metadata": {
+    "url": "/drive/root:/book-war-and-peace-1p.txt",
+    "version": "\"{77650B15-A8AA-450E-BC8F-BB511A86D4C6},1\"",
+    "record_locator": {
+      "user_pname": "[email protected]",
+      "server_relative_path": "/book-war-and-peace-1p.txt"
+    },
+    "date_created": "1738885678.0",
+    "date_modified": "1738885678.0",
+    "date_processed": "1739552932.2096388",
+    "permissions_data": null,
+    "filesize_bytes": null
+  },
+  "additional_metadata": {
+    "eTag": "\"{77650B15-A8AA-450E-BC8F-BB511A86D4C6},1\"",
+    "id": "0153RHRSAVBNSXPKVIBZC3ZD53KENINVGG",
+    "name": "book-war-and-peace-1p.txt",
+    "webUrl": "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source/Shared%20Documents/book-war-and-peace-1p.txt",
+    "cTag": "\"c:{77650B15-A8AA-450E-BC8F-BB511A86D4C6},1\"",
+    "size": 3045
+  },
+  "reprocess": false,
+  "local_download_path": "/private/var/folders/85/f389rtdn2c971nv4r3d31d740000gn/T/tmpf70nr_lt/book-war-and-peace-1p.txt",
+  "display_name": null
+}

test/integration/connectors/expected_results/sharepoint1/file_data/0153RHRSEXY3G5U3B5GBFY7TZKAY236XAL.json

@@ -0,0 +1,33 @@
+{
+  "identifier": "0153RHRSEXY3G5U3B5GBFY7TZKAY236XAL",
+  "connector_type": "sharepoint",
+  "source_identifiers": {
+    "filename": "fake-memo.pdf",
+    "fullpath": "Folder1/fake-memo.pdf",
+    "rel_path": "Folder1/fake-memo.pdf"
+  },
+  "metadata": {
+    "url": "/drive/root:/Folder1/fake-memo.pdf",
+    "version": "\"{DACDC697-3D6C-4B30-8FCF-2A0635BF5C0B},1\"",
+    "record_locator": {
+      "user_pname": "[email protected]",
+      "server_relative_path": "Folder1/fake-memo.pdf"
+    },
+    "date_created": "1738364779.0",
+    "date_modified": "1738364779.0",
+    "date_processed": "1739552943.296567",
+    "permissions_data": null,
+    "filesize_bytes": null
+  },
+  "additional_metadata": {
+    "eTag": "\"{DACDC697-3D6C-4B30-8FCF-2A0635BF5C0B},1\"",
+    "id": "0153RHRSEXY3G5U3B5GBFY7TZKAY236XAL",
+    "name": "fake-memo.pdf",
+    "webUrl": "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source/Shared%20Documents/Folder1/fake-memo.pdf",
+    "cTag": "\"c:{DACDC697-3D6C-4B30-8FCF-2A0635BF5C0B},1\"",
+    "size": 13374
+  },
+  "reprocess": false,
+  "local_download_path": "/private/var/folders/85/f389rtdn2c971nv4r3d31d740000gn/T/tmpf70nr_lt/Folder1/fake-memo.pdf",
+  "display_name": null
+}

test/integration/connectors/expected_results/sharepoint1/file_data/0153RHRSFFVW43WTVG5ZFY5WL7ZH3GUOGT.json

@@ -0,0 +1,33 @@
+{
+  "identifier": "0153RHRSFFVW43WTVG5ZFY5WL7ZH3GUOGT",
+  "connector_type": "sharepoint",
+  "source_identifiers": {
+    "filename": "fake-email.txt",
+    "fullpath": "Folder1/Folder2/fake-email.txt",
+    "rel_path": "Folder1/Folder2/fake-email.txt"
+  },
+  "metadata": {
+    "url": "/drive/root:/Folder1/Folder2/fake-email.txt",
+    "version": "\"{BBB9ADA5-A64E-4BEE-8ED9-7FC9F66A38D3},1\"",
+    "record_locator": {
+      "user_pname": "[email protected]",
+      "server_relative_path": "Folder1/Folder2/fake-email.txt"
+    },
+    "date_created": "1738364377.0",
+    "date_modified": "1738364377.0",
+    "date_processed": "1739552950.261651",
+    "permissions_data": null,
+    "filesize_bytes": null
+  },
+  "additional_metadata": {
+    "eTag": "\"{BBB9ADA5-A64E-4BEE-8ED9-7FC9F66A38D3},1\"",
+    "id": "0153RHRSFFVW43WTVG5ZFY5WL7ZH3GUOGT",
+    "name": "fake-email.txt",
+    "webUrl": "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source/Shared%20Documents/Folder1/Folder2/fake-email.txt",
+    "cTag": "\"c:{BBB9ADA5-A64E-4BEE-8ED9-7FC9F66A38D3},1\"",
+    "size": 836
+  },
+  "reprocess": false,
+  "local_download_path": "/private/var/folders/85/f389rtdn2c971nv4r3d31d740000gn/T/tmpf70nr_lt/Folder1/Folder2/fake-email.txt",
+  "display_name": null
+}

test/integration/connectors/expected_results/sharepoint2/directory_structure.json

@@ -0,0 +1,6 @@
+{
+  "directory_structure": [
+    "Folder2/fake-email.txt",
+    "fake-memo.pdf"
+  ]
+}

test/integration/connectors/expected_results/sharepoint2/downloads/Folder2/fake-email.txt

@@ -0,0 +1,25 @@
+MIME-Version: 1.0
+Date: Fri, 16 Dec 2022 17:04:16 -0500
+Message-ID: <CADc-_xaLB2FeVQ7mNsoX+NJb_7hAJhBKa_zet-rtgPGenj0uVw@mail.gmail.com>
+Subject: Test Email
+From: Matthew Robinson <[email protected]>
+To: Matthew Robinson <[email protected]>
+Content-Type: multipart/alternative; boundary="00000000000095c9b205eff92630"
+
+--00000000000095c9b205eff92630
+Content-Type: text/plain; charset="UTF-8"
+
+This is a test email to use for unit tests.
+
+Important points:
+
+   - Roses are red
+   - Violets are blue
+   - 
+
+--00000000000095c9b205eff92630
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr"><div>This is a test email to use for unit tests.</div><div><br></div><div>Important points:</div><div><ul><li>Roses are red</li><li>Violets are blue</li></ul></div></div>
+
+--00000000000095c9b205eff92630--