Fixing repochat action

flavienbwk · Sep 8, 2024 · 73bd159 · 73bd159
1 parent fa70983
commit 73bd159
Show file tree

Hide file tree

Showing 3 changed files with 96 additions and 36 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -12,6 +12,7 @@ jobs:
         steps:
             - uses: actions/checkout@v2
             - uses: ./
+              id: deploy_repochat
               with:
                 dirs_to_scan: "./example,README.md" # comma-separated glob dirs to analyze from this repo
                 openai_api_key: ${{ secrets.OPENAI_API_KEY }}
@@ -23,3 +24,6 @@ jobs:
                 provider_project_id: ${{ secrets.PROVIDER_PROJECT_ID }}
                 provider_default_region: 'fr-par'
                 provider_default_zone: 'fr-par-2'
+            - name: Get repochat domain
+              run: echo "DOMAIN=${{ steps.deploy_repochat.outputs.domain }}" >> $GITHUB_OUTPUT
+              id: repochat_domain
diff --git a/action/index.js b/action/index.js
@@ -24,9 +24,22 @@ async function sendFileToApi(filePath, apiUrl) {
   }
 }
 
-async function ingestFiles(directoryPath, apiUrl) {
-  if (!fs.existsSync(directoryPath) || !fs.statSync(directoryPath).isDirectory()) {
-    console.error(`Error: ${directoryPath} is not a valid directory.`);
+async function ingestFiles(directoryPath, apiUrl, excludeFiles = []) {
+  if (!fs.existsSync(directoryPath)) {
+    console.error(`Error: ${directoryPath} does not exist.`);
+    return;
+  }
+
+  const stats = fs.statSync(directoryPath);
+  if (stats.isFile()) {
+    if (isValidFile(directoryPath) && !isExcluded(directoryPath, excludeFiles)) {
+      await processFile(directoryPath, apiUrl);
+    }
+    return;
+  }
+
+  if (!stats.isDirectory()) {
+    console.error(`Error: ${directoryPath} is not a valid directory or file.`);
     return;
   }
 
@@ -35,21 +48,32 @@ async function ingestFiles(directoryPath, apiUrl) {
   for (const file of files) {
     const filePath = path.join(directoryPath, file.name);
     if (file.isDirectory()) {
-      await ingestFiles(filePath, apiUrl);
-    } else if (isValidFile(filePath)) {
-      console.log(`Sending file: ${filePath}`);
-      const response = await sendFileToApi(filePath, apiUrl);
-      if (response.status === 200) {
-        console.log(`Successfully ingested: ${filePath}`);
-      } else {
-        console.log(`Failed to ingest ${filePath}. Status code: ${response.status}`);
-      }
+      await ingestFiles(filePath, apiUrl, excludeFiles);
+    } else if (isValidFile(filePath) && !isExcluded(filePath, excludeFiles)) {
+      await processFile(filePath, apiUrl);
     } else {
-      console.log(`Skipping invalid or hidden file: ${filePath}`);
+      console.log(`Skipping invalid, hidden, or excluded file: ${filePath}`);
     }
   }
 }
 
+function isExcluded(filePath, excludeFiles) {
+  return excludeFiles.some(pattern => {
+    const regex = new RegExp('^' + pattern.replace(/\*/g, '.*').replace(/\?/g, '.') + '$');
+    return regex.test(filePath);
+  });
+}
+
+async function processFile(filePath, apiUrl) {
+  console.log(`Sending file: ${filePath}`);
+  const response = await sendFileToApi(filePath, apiUrl);
+  if (response.status === 200) {
+    console.log(`Successfully ingested: ${filePath}`);
+  } else {
+    console.log(`Failed to ingest ${filePath}. Status code: ${response.status}`);
+  }
+}
+
 try {
   const dirsToScan = core.getInput('dirs_to_scan');
   const openaiApiKey = core.getInput('openai_api_key');
@@ -169,7 +193,7 @@ try {
           const containers = await containerApi.listContainers({ namespaceId: namespace.id });
           container = containers.containers.find(c => c.name === containerName);
           console.log('Retrieved existing container:', container.id);
-          
+
           // Update the existing container with new configuration
           container = await containerApi.updateContainer({
             containerId: container.id,
@@ -222,16 +246,20 @@ try {
       // Feed RepoChat with repo data
       const containerEndpointApi = 'https://' + containerEndpoint + '/api/ingest';
       console.log(`Ingesting files at ${containerEndpointApi}...`);
-      ingestFiles('./', containerEndpointApi);
-
+      const dirsToScanArray = dirsToScan.split(',').map(dir => dir.trim());
+      for (const dir of dirsToScanArray) {
+        console.log(`Ingesting files inside ${dir}...`);
+        ingestFiles(dir, containerEndpointApi, ['node_modules', '.git', 'package-lock.json']);
+      }
+
       // Set outputs
       core.setOutput('domain', containerEndpoint);
     } catch (error) {
       console.error('Error deploying container:', error);
     }
-    
+
   }
-  
+
 } catch (error) {
   core.setFailed(error.message);
 }
diff --git a/lib/index.js b/lib/index.js
@@ -73539,9 +73539,22 @@ async function sendFileToApi(filePath, apiUrl) {
   }
 }
 
-async function ingestFiles(directoryPath, apiUrl) {
-  if (!fs__WEBPACK_IMPORTED_MODULE_0__.existsSync(directoryPath) || !fs__WEBPACK_IMPORTED_MODULE_0__.statSync(directoryPath).isDirectory()) {
-    console.error(`Error: ${directoryPath} is not a valid directory.`);
+async function ingestFiles(directoryPath, apiUrl, excludeFiles = []) {
+  if (!fs__WEBPACK_IMPORTED_MODULE_0__.existsSync(directoryPath)) {
+    console.error(`Error: ${directoryPath} does not exist.`);
+    return;
+  }
+
+  const stats = fs__WEBPACK_IMPORTED_MODULE_0__.statSync(directoryPath);
+  if (stats.isFile()) {
+    if (isValidFile(directoryPath) && !isExcluded(directoryPath, excludeFiles)) {
+      await processFile(directoryPath, apiUrl);
+    }
+    return;
+  }
+
+  if (!stats.isDirectory()) {
+    console.error(`Error: ${directoryPath} is not a valid directory or file.`);
     return;
   }
 
@@ -73550,21 +73563,32 @@ async function ingestFiles(directoryPath, apiUrl) {
   for (const file of files) {
     const filePath = path__WEBPACK_IMPORTED_MODULE_1__.join(directoryPath, file.name);
     if (file.isDirectory()) {
-      await ingestFiles(filePath, apiUrl);
-    } else if (isValidFile(filePath)) {
-      console.log(`Sending file: ${filePath}`);
-      const response = await sendFileToApi(filePath, apiUrl);
-      if (response.status === 200) {
-        console.log(`Successfully ingested: ${filePath}`);
-      } else {
-        console.log(`Failed to ingest ${filePath}. Status code: ${response.status}`);
-      }
+      await ingestFiles(filePath, apiUrl, excludeFiles);
+    } else if (isValidFile(filePath) && !isExcluded(filePath, excludeFiles)) {
+      await processFile(filePath, apiUrl);
     } else {
-      console.log(`Skipping invalid or hidden file: ${filePath}`);
+      console.log(`Skipping invalid, hidden, or excluded file: ${filePath}`);
     }
   }
 }
 
+function isExcluded(filePath, excludeFiles) {
+  return excludeFiles.some(pattern => {
+    const regex = new RegExp('^' + pattern.replace(/\*/g, '.*').replace(/\?/g, '.') + '$');
+    return regex.test(filePath);
+  });
+}
+
+async function processFile(filePath, apiUrl) {
+  console.log(`Sending file: ${filePath}`);
+  const response = await sendFileToApi(filePath, apiUrl);
+  if (response.status === 200) {
+    console.log(`Successfully ingested: ${filePath}`);
+  } else {
+    console.log(`Failed to ingest ${filePath}. Status code: ${response.status}`);
+  }
+}
+
 try {
   const dirsToScan = _actions_core__WEBPACK_IMPORTED_MODULE_2__.getInput('dirs_to_scan');
   const openaiApiKey = _actions_core__WEBPACK_IMPORTED_MODULE_2__.getInput('openai_api_key');
@@ -73684,7 +73708,7 @@ try {
           const containers = await containerApi.listContainers({ namespaceId: namespace.id });
           container = containers.containers.find(c => c.name === containerName);
           console.log('Retrieved existing container:', container.id);
-          
+
           // Update the existing container with new configuration
           container = await containerApi.updateContainer({
             containerId: container.id,
@@ -73737,16 +73761,20 @@ try {
       // Feed RepoChat with repo data
       const containerEndpointApi = 'https://' + containerEndpoint + '/api/ingest';
       console.log(`Ingesting files at ${containerEndpointApi}...`);
-      ingestFiles('./', containerEndpointApi);
-
+      const dirsToScanArray = dirsToScan.split(',').map(dir => dir.trim());
+      for (const dir of dirsToScanArray) {
+        console.log(`Ingesting files inside ${dir}...`);
+        ingestFiles(dir, containerEndpointApi, ['node_modules', '.git', 'package-lock.json']);
+      }
+
       // Set outputs
       _actions_core__WEBPACK_IMPORTED_MODULE_2__.setOutput('domain', containerEndpoint);
     } catch (error) {
       console.error('Error deploying container:', error);
     }
-    
+
   }
-  
+
 } catch (error) {
   _actions_core__WEBPACK_IMPORTED_MODULE_2__.setFailed(error.message);
 }