feat(core): add maxCacheSize field in nx.json

nrwl · Jan 16, 2025 · 86edfe6 · 86edfe6
1 parent a1aeb1e
commit 86edfe6
Show file tree

Hide file tree

Showing 9 changed files with 264 additions and 5 deletions.
diff --git a/docs/generated/devkit/NxJsonConfiguration.md b/docs/generated/devkit/NxJsonConfiguration.md
@@ -27,6 +27,7 @@ Nx.json configuration
 - [generators](../../devkit/documents/NxJsonConfiguration#generators): Object
 - [implicitDependencies](../../devkit/documents/NxJsonConfiguration#implicitdependencies): ImplicitDependencyEntry<T>
 - [installation](../../devkit/documents/NxJsonConfiguration#installation): NxInstallationConfiguration
+- [maxCacheSize](../../devkit/documents/NxJsonConfiguration#maxcachesize): string
 - [namedInputs](../../devkit/documents/NxJsonConfiguration#namedinputs): Object
 - [neverConnectToCloud](../../devkit/documents/NxJsonConfiguration#neverconnecttocloud): boolean
 - [nxCloudAccessToken](../../devkit/documents/NxJsonConfiguration#nxcloudaccesstoken): string
@@ -155,6 +156,14 @@ useful for workspaces that don't have a root package.json + node_modules.
 
 ---
 
+### maxCacheSize
+
+• `Optional` **maxCacheSize**: `string`
+
+Sets the maximum size of the local cache. Accepts a number followed by a unit (e.g. 100MB). Accepted units are B, KB, MB, and GB.
+
+---
+
 ### namedInputs
 
 • `Optional` **namedInputs**: `Object`

diff --git a/docs/generated/devkit/Workspace.md b/docs/generated/devkit/Workspace.md
@@ -25,6 +25,7 @@ use ProjectsConfigurations or NxJsonConfiguration
 - [generators](../../devkit/documents/Workspace#generators): Object
 - [implicitDependencies](../../devkit/documents/Workspace#implicitdependencies): ImplicitDependencyEntry<string[] | "\*">
 - [installation](../../devkit/documents/Workspace#installation): NxInstallationConfiguration
+- [maxCacheSize](../../devkit/documents/Workspace#maxcachesize): string
 - [namedInputs](../../devkit/documents/Workspace#namedinputs): Object
 - [neverConnectToCloud](../../devkit/documents/Workspace#neverconnecttocloud): boolean
 - [nxCloudAccessToken](../../devkit/documents/Workspace#nxcloudaccesstoken): string
@@ -191,6 +192,18 @@ useful for workspaces that don't have a root package.json + node_modules.
 
 ---
 
+### maxCacheSize
+
+• `Optional` **maxCacheSize**: `string`
+
+Sets the maximum size of the local cache. Accepts a number followed by a unit (e.g. 100MB). Accepted units are B, KB, MB, and GB.
+
+#### Inherited from
+
+[NxJsonConfiguration](../../devkit/documents/NxJsonConfiguration).[maxCacheSize](../../devkit/documents/NxJsonConfiguration#maxcachesize)
+
+---
+
 ### namedInputs
 
 • `Optional` **namedInputs**: `Object`

diff --git a/e2e/nx/src/cache.test.ts b/e2e/nx/src/cache.test.ts
@@ -11,6 +11,9 @@ import {
   updateFile,
   updateJson,
 } from '@nx/e2e/utils';
+
+import { readdir, stat } from 'fs/promises';
+
 import { join } from 'path';
 
 describe('cache', () => {
@@ -361,6 +364,66 @@ describe('cache', () => {
     expect(fourthRun).toContain('read the output from the cache');
   }, 120000);
 
+  it('should evict cache if larger than max cache size', async () => {
+    runCLI('reset');
+    updateJson(`nx.json`, (c) => {
+      c.maxCacheSize = '1MB';
+      return c;
+    });
+
+    const lib = uniq('cache-size');
+
+    updateFile(
+      `tools/copy.js`,
+      'require("fs").cpSync(process.argv[2], process.argv[3], { recursive: true, force: true });'
+    );
+    updateFile(
+      `libs/${lib}/project.json`,
+      JSON.stringify({
+        name: lib,
+        targets: {
+          write: {
+            cache: true,
+            command: 'node tools/copy.js {projectRoot}/src dist/{projectRoot}',
+            inputs: ['{projectRoot}/hash.txt'],
+            outputs: ['{workspaceRoot}/dist/{projectRoot}'],
+          },
+        },
+      })
+    );
+    // 100KB file
+    updateFile(`libs/${lib}/src/data.txt`, 'a'.repeat(100 * 1024));
+    for (let i = 0; i < 50; ++i) {
+      updateFile(`libs/${lib}/hash.txt`, i.toString());
+      runCLI(`write ${lib}`);
+    }
+
+    // Expect that size of cache artifacts in cacheDir is less than 1MB
+    const cacheDir = tmpProjPath('.nx/cache');
+    const cacheFiles = listFiles('.nx/cache');
+    let cacheEntries = 0;
+    let cacheEntriesSize = 0;
+    for (const file of cacheFiles) {
+      if (file.match(/^[0-9]+$/)) {
+        cacheEntries += 1;
+        cacheEntriesSize += await dirSize(join(cacheDir, file));
+        console.log(
+          'Checked cache entry',
+          file,
+          'size',
+          cacheEntriesSize,
+          'total entries',
+          cacheEntries
+        );
+      }
+    }
+    console.log('Cache entries:', cacheEntries);
+    console.log('Cache size:', cacheEntriesSize);
+    expect(cacheEntries).toBeGreaterThan(1);
+    expect(cacheEntries).toBeLessThan(21);
+    expect(cacheEntriesSize).toBeLessThan(1024 * 1024);
+  });
+
   function expectCached(
     actualOutput: string,
     expectedCachedProjects: string[]
@@ -404,3 +467,27 @@ describe('cache', () => {
     expect(matchingProjects).toEqual(expectedProjects);
   }
 });
+
+const dirSize = async (dir) => {
+  const files = await readdir(dir, { withFileTypes: true });
+
+  const paths = files.map(async (file) => {
+    const path = join(dir, file.name);
+
+    if (file.isDirectory()) return await dirSize(path);
+
+    if (file.isFile()) {
+      const { size } = await stat(path);
+
+      return size;
+    }
+
+    console.log('Unknown file type', path);
+
+    return 0;
+  });
+
+  return (await Promise.all(paths))
+    .flat(Infinity)
+    .reduce((i, size) => i + size, 0);
+};
diff --git a/packages/nx/src/adapter/compat.ts b/packages/nx/src/adapter/compat.ts
@@ -81,6 +81,7 @@ export const allowedWorkspaceExtensions = [
   'neverConnectToCloud',
   'sync',
   'useLegacyCache',
+  'maxCacheSize',
 ] as const;
 
 if (!patched) {

diff --git a/packages/nx/src/config/nx-json.ts b/packages/nx/src/config/nx-json.ts
@@ -524,6 +524,11 @@ export interface NxJsonConfiguration<T = '*' | string[]> {
    * Use the legacy file system cache instead of the db cache
    */
   useLegacyCache?: boolean;
+
+  /**
+   * Sets the maximum size of the local cache. Accepts a number followed by a unit (e.g. 100MB). Accepted units are B, KB, MB, and GB.
+   */
+  maxCacheSize?: string;
 }
 
 export type PluginConfiguration = string | ExpandedPluginConfiguration;

diff --git a/packages/nx/src/native/cache/cache.rs b/packages/nx/src/native/cache/cache.rs
@@ -2,7 +2,7 @@ use std::fs::{create_dir_all, read_to_string, write};
 use std::path::{Path, PathBuf};
 use std::time::Instant;
 
-use fs_extra::remove_items;
+use fs_extra::{dir::get_size, remove_items};
 use napi::bindgen_prelude::*;
 use regex::Regex;
 use rusqlite::params;
@@ -29,6 +29,7 @@ pub struct NxCache {
     cache_path: PathBuf,
     db: External<NxDbConnection>,
     link_task_details: bool,
+    max_cache_size: Option<i64>,
 }
 
 #[napi]
@@ -39,6 +40,7 @@ impl NxCache {
         cache_path: String,
         db_connection: External<NxDbConnection>,
         link_task_details: Option<bool>,
+        max_cache_size: Option<i64>,
     ) -> anyhow::Result<Self> {
         let cache_path = PathBuf::from(&cache_path);
 
@@ -51,6 +53,7 @@ impl NxCache {
             cache_directory: cache_path.to_normalized_string(),
             cache_path,
             link_task_details: link_task_details.unwrap_or(true),
+            max_cache_size,
         };
 
         r.setup()?;
@@ -113,7 +116,7 @@ impl NxCache {
                         code,
                         terminal_output,
                         outputs_path: task_dir.to_normalized_string(),
-                        size: Some(size)
+                        size: Some(size),
                     })
                 },
             )
@@ -176,10 +179,17 @@ impl NxCache {
             &result.outputs_path
         );
         let terminal_output = result.terminal_output;
+        let size = if result.size.is_some() {
+            result.size
+        } else if Path::new(&result.outputs_path).exists() {
+            Some(get_size(&result.outputs_path).unwrap_or(0) as i64)
+        } else {
+            None
+        };
         write(self.get_task_outputs_path(hash.clone()), terminal_output)?;
 
         let code: i16 = result.code;
-        self.record_to_cache(hash, code, result.size)?;
+        self.record_to_cache(hash, code, size)?;
         Ok(())
     }
 
@@ -200,6 +210,46 @@ impl NxCache {
             "INSERT OR REPLACE INTO cache_outputs (hash, code, size) VALUES (?1, ?2, ?3)",
             params![hash, code, size],
         )?;
+        if self.max_cache_size.is_some() {
+            self.ensure_cache_size_within_limit()?
+        }
+        Ok(())
+    }
+
+    fn ensure_cache_size_within_limit(&self) -> anyhow::Result<()> {
+        if let Some(max_cache_size) = self.max_cache_size {
+            let full_cache_size = self
+                .db
+                .query_row("SELECT SUM(size) FROM cache_outputs", [], |row| {
+                    row.get::<_, i64>(0)
+                })?
+                .unwrap_or(0);
+            if max_cache_size < full_cache_size {
+                let mut cache_size = full_cache_size;
+                let mut hashes_to_delete = vec![];
+                let mut stmt = self.db.prepare(
+                    "SELECT hash, size FROM cache_outputs ORDER BY accessed_at ASC LIMIT 100",
+                )?;
+                while cache_size > max_cache_size {
+                    let mut rows = stmt.query([])?;
+                    while let Some(row) = rows.next()? {
+                        let hash: String = row.get(0)?;
+                        let size: i64 = row.get(1)?;
+                        cache_size -= size;
+                        hashes_to_delete.push(hash);
+
+                        if (cache_size) < max_cache_size {
+                            break;
+                        }
+                    }
+                }
+                for hash in hashes_to_delete {
+                    self.db
+                        .execute("DELETE FROM cache_outputs WHERE hash = ?1", params![hash])?;
+                    remove_items(&[self.cache_path.join(&hash)])?;
+                }
+            }
+        }
         Ok(())
     }
 

diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts
@@ -28,7 +28,7 @@ export declare class ImportResult {
 
 export declare class NxCache {
   cacheDirectory: string
-  constructor(workspaceRoot: string, cachePath: string, dbConnection: ExternalObject<NxDbConnection>, linkTaskDetails?: boolean | undefined | null)
+  constructor(workspaceRoot: string, cachePath: string, dbConnection: ExternalObject<NxDbConnection>, linkTaskDetails?: boolean | undefined | null, maxCacheSize?: number | undefined | null)
   get(hash: string): CachedResult | null
   put(hash: string, terminalOutput: string, outputs: Array<string>, code: number): void
   applyRemoteCacheResults(hash: string, result: CachedResult): void

diff --git a/packages/nx/src/tasks-runner/cache.spec.ts b/packages/nx/src/tasks-runner/cache.spec.ts
@@ -0,0 +1,41 @@
+import { parseMaxCacheSize } from './cache';
+
+describe('cache', () => {
+  describe('parseMaxCacheSize', () => {
+    it('should parse KB', () => {
+      expect(parseMaxCacheSize('1KB')).toEqual(1024);
+    });
+
+    it('should parse MB', () => {
+      expect(parseMaxCacheSize('1MB')).toEqual(1024 * 1024);
+    });
+
+    it('should parse GB', () => {
+      expect(parseMaxCacheSize('1GB')).toEqual(1024 * 1024 * 1024);
+    });
+
+    it('should parse B', () => {
+      expect(parseMaxCacheSize('1B')).toEqual(1);
+    });
+
+    it('should parse as bytes by default', () => {
+      expect(parseMaxCacheSize('1')).toEqual(1);
+    });
+
+    it('should handle decimals', () => {
+      expect(parseMaxCacheSize('1.5KB')).toEqual(1024 * 1.5);
+    });
+
+    it('should error if invalid unit', () => {
+      expect(() => parseMaxCacheSize('1ZB')).toThrow();
+    });
+
+    it('should error if invalid number', () => {
+      expect(() => parseMaxCacheSize('abc')).toThrow();
+    });
+
+    it('should error if multiple decimal points', () => {
+      expect(() => parseMaxCacheSize('1.5.5KB')).toThrow;
+    });
+  });
+});
diff --git a/packages/nx/src/tasks-runner/cache.ts b/packages/nx/src/tasks-runner/cache.ts
@@ -89,7 +89,14 @@ export function getCache(options: DefaultTasksRunnerOptions): DbCache | Cache {
 }
 
 export class DbCache {
-  private cache = new NxCache(workspaceRoot, cacheDir, getDbConnection());
+  private nxJson = readNxJson();
+  private cache = new NxCache(
+    workspaceRoot,
+    cacheDir,
+    getDbConnection(),
+    undefined,
+    parseMaxCacheSize(this.nxJson.maxCacheSize)
+  );
 
   private remoteCache: RemoteCacheV2 | null;
   private remoteCachePromise: Promise<RemoteCacheV2>;
@@ -571,3 +578,49 @@ function tryAndRetry<T>(fn: () => Promise<T>): Promise<T> {
   };
   return _try();
 }
+
+/**
+ * Converts a string representation of a max cache size to a number.
+ *
+ * e.g. '1GB' -> 1024 * 1024 * 1024
+ *      '1MB' -> 1024 * 1024
+ *      '1KB' -> 1024
+ *
+ * @param maxCacheSize Max cache size as specified in nx.json
+ */
+export function parseMaxCacheSize(maxCacheSize: string): number | undefined {
+  if (!maxCacheSize) {
+    return undefined;
+  }
+  let regexResult = maxCacheSize.match(
+    /^(?<size>[\d|.]+)\s?((?<unit>[KMG]?B)?)$/
+  );
+  if (!regexResult) {
+    throw new Error(
+      `Invalid max cache size specified in nx.json: ${maxCacheSize}. Must be a number followed by an optional unit (KB, MB, GB)`
+    );
+  }
+  let sizeString = regexResult.groups.size;
+  let unit = regexResult.groups.unit;
+  if ([...sizeString].filter((c) => c === '.').length > 1) {
+    throw new Error(
+      `Invalid max cache size specified in nx.json: ${maxCacheSize} (multiple decimal points in size)`
+    );
+  }
+  let size = parseFloat(sizeString);
+  if (isNaN(size)) {
+    throw new Error(
+      `Invalid max cache size specified in nx.json: ${maxCacheSize} (${sizeString} is not a number)`
+    );
+  }
+  switch (unit) {
+    case 'KB':
+      return size * 1024;
+    case 'MB':
+      return size * 1024 * 1024;
+    case 'GB':
+      return size * 1024 * 1024 * 1024;
+    default:
+      return size;
+  }
+}