[WIP] GitHub code: tar processing (#3050)

* WIP tar download and extraction * tar processing * Github tar processing and file / directory parsing * comment * feedback from review * fix * lint
dust-tt · Jan 4, 2024 · 4c530ba · 4c530ba
1 parent e772725
commit 4c530ba
Show file tree

Hide file tree

Showing 4 changed files with 437 additions and 0 deletions.
diff --git a/connectors/package-lock.json b/connectors/package-lock.json
diff --git a/connectors/package.json b/connectors/package.json
@@ -28,6 +28,7 @@
     "@types/uuid": "^9.0.2",
     "axios": "^1.5.1",
     "body-parser": "^1.20.2",
+    "blake3": "^2.1.7",
     "dd-trace": "^3.16.0",
     "eventsource-parser": "^1.0.0",
     "express": "^4.18.2",
@@ -54,13 +55,15 @@
     "redis": "^4.6.10",
     "sequelize": "^6.31.0",
     "talisman": "^1.1.4",
+    "tar": "^6.2.0",
     "uuid": "^9.0.0"
   },
   "devDependencies": {
     "@types/eslint": "^8.21.3",
     "@types/fast-levenshtein": "^0.0.2",
     "@types/node": "^18.15.5",
     "@types/p-queue": "^3.2.1",
+    "@types/tar": "^6.1.10",
     "@typescript-eslint/eslint-plugin": "^5.56.0",
     "@typescript-eslint/parser": "^5.56.0",
     "eslint": "^8.36.0",

diff --git a/connectors/src/admin/cli.ts b/connectors/src/admin/cli.ts
@@ -9,6 +9,10 @@ import {
   STOP_CONNECTOR_BY_TYPE,
   SYNC_CONNECTOR_BY_TYPE,
 } from "@connectors/connectors";
+import {
+  cleanUpProcessRepository,
+  processRepository,
+} from "@connectors/connectors/github/lib/github_api";
 import {
   getAuthObject,
   getDocumentId,
@@ -99,6 +103,64 @@ const connectors = async (command: string, args: parseArgs.ParsedArgs) => {
   }
 };
 
+const github = async (command: string, args: parseArgs.ParsedArgs) => {
+  switch (command) {
+    case "test-repo": {
+      if (!args.wId) {
+        throw new Error("Missing --wId argument");
+      }
+      if (!args.dataSourceName) {
+        throw new Error("Missing --dataSourceName argument");
+      }
+      if (!args.owner) {
+        throw new Error("Missing --owner argument");
+      }
+      if (!args.repo) {
+        throw new Error("Missing --repo argument");
+      }
+
+      const connector = await Connector.findOne({
+        where: {
+          type: "github",
+          workspaceId: args.wId,
+          dataSourceName: args.dataSourceName,
+        },
+      });
+
+      if (!connector) {
+        throw new Error(
+          `Could not find connector for workspace ${args.wId}, data source ${args.dataSourceName}`
+        );
+      }
+
+      const installationId = connector.connectionId;
+
+      const { tempDir, files, directories } = await processRepository(
+        installationId,
+        args.owner,
+        args.repo,
+        "999"
+      );
+
+      files.forEach((f) => {
+        console.log(f);
+      });
+      directories.forEach((d) => {
+        console.log(d);
+      });
+
+      console.log(
+        `Found ${files.length} files in ${directories.length} directories`
+      );
+      console.log(
+        `Files total size: ${files.reduce((acc, f) => acc + f.sizeBytes, 0)}`
+      );
+
+      await cleanUpProcessRepository(tempDir);
+    }
+  }
+};
+
 const notion = async (command: string, args: parseArgs.ParsedArgs) => {
   switch (command) {
     case "restart-all": {
@@ -568,6 +630,9 @@ const main = async () => {
     case "notion":
       await notion(command, argv);
       return;
+    case "github":
+      await github(command, argv);
+      return;
     case "google":
       await google(command, argv);
       return;