From 5f203b0571188c3d84ef8dde8febf389e62cd721 Mon Sep 17 00:00:00 2001 From: g122622 <3300925806@qq.com> Date: Fri, 16 Feb 2024 16:55:08 +0800 Subject: [PATCH] v3.4.0 --- src/api/File.ts | 2 +- src/api/FileWatcher.ts | 2 +- src/api/core/KVPEngines/KVPEngineHybrid.ts | 42 +++- src/api/core/KVPEngines/KVPEngineJson.ts | 11 +- src/api/core/adapters/gcryptV1/adapter.ts | 22 +- .../encryptionEngines/EncryptionEngineNoop.ts | 2 +- src/api/core/types/AdapterBase.ts | 3 +- src/api/hash/getDigest.ts | 2 +- src/components/AceEditor/AceEditor.vue | 2 +- src/components/FileMgr/DialogMgr.vue | 6 +- src/components/FileMgr/FileItem.vue | 9 +- src/components/FileMgr/FileMgr.vue | 6 +- src/components/SystemBar.vue | 11 + src/components/shared/AdvancedList.vue | 48 ++-- src/main.ts | 4 +- src/test/KVPEngineHybridUnit.ts | 225 ++++++++++++++++++ src/utils/{ => file}/getExtName.ts | 0 src/utils/{ => file}/getFileName.ts | 0 src/utils/{ => file}/getFileType.ts | 4 +- src/utils/{ => file}/isFileOccupied.ts | 0 src/utils/helpers/Lock.ts | 36 +++ 21 files changed, 378 insertions(+), 59 deletions(-) create mode 100644 src/test/KVPEngineHybridUnit.ts rename src/utils/{ => file}/getExtName.ts (100%) rename src/utils/{ => file}/getFileName.ts (100%) rename src/utils/{ => file}/getFileType.ts (89%) rename src/utils/{ => file}/isFileOccupied.ts (100%) create mode 100644 src/utils/helpers/Lock.ts diff --git a/src/api/File.ts b/src/api/File.ts index 2c0fa0c..36d1e0d 100644 --- a/src/api/File.ts +++ b/src/api/File.ts @@ -2,7 +2,7 @@ import os from "os"; import path from "path"; import fs from "fs-extra"; -import getExtName from "@/utils/getExtName"; +import getExtName from "@/utils/file/getExtName"; import sharedUtils from "@/utils/sharedUtils"; import FileWatcher from "./FileWatcher"; import AdapterBase from "./core/types/AdapterBase"; diff --git a/src/api/FileWatcher.ts b/src/api/FileWatcher.ts index bfec470..0d27f53 100644 --- a/src/api/FileWatcher.ts +++ b/src/api/FileWatcher.ts @@ -1,6 +1,6 @@ import fs from "fs-extra" import path from "path" -import isFileOccupied from '@/utils/isFileOccupied' +import isFileOccupied from '@/utils/file/isFileOccupied' interface Options { minUpdateIntervalMs: number, diff --git a/src/api/core/KVPEngines/KVPEngineHybrid.ts b/src/api/core/KVPEngines/KVPEngineHybrid.ts index 1f7d393..b0b14b1 100644 --- a/src/api/core/KVPEngines/KVPEngineHybrid.ts +++ b/src/api/core/KVPEngines/KVPEngineHybrid.ts @@ -16,6 +16,7 @@ import KVPEngineFolder from "./KVPEngineFolder"; import calcBufSize from "@/utils/calcBufSize"; import ASSERT from "@/utils/ASSERT"; import sharedUtils from "@/utils/sharedUtils"; +import Lock from "@/utils/helpers/Lock" const config = { blockInclusionThreshold: 12 * 1024, // 小于这个值的value将被放在block内 @@ -28,6 +29,8 @@ class KVPEngineHybrid extends KVPEngineFolder { private keyReferenceMap = {} // blockUsageMap用于快速查询某个block的存在情况和空间使用情况 private blockUsageMap = {} + // 用于确保操作的原子性 + private opLock: Lock /** * 初始化jsonStorage @@ -35,6 +38,7 @@ class KVPEngineHybrid extends KVPEngineFolder { * @param pwd 密码 */ public async init(storeEntryJsonSrc: string, pwd: string, encryptionEngine) { + this.opLock = new Lock() await super.init(storeEntryJsonSrc, pwd, encryptionEngine) if (await super.hasData('@keyReferenceMap')) { this.keyReferenceMap = JSON.parse((await super.getData('@keyReferenceMap')).toString()) @@ -44,6 +48,11 @@ class KVPEngineHybrid extends KVPEngineFolder { } } + private async saveMaps() { + await super.setData('@keyReferenceMap', Buffer.from(JSON.stringify(this.keyReferenceMap))) + await super.setData('@blockUsageMap', Buffer.from(JSON.stringify(this.blockUsageMap))) + } + private async hasDataInBlock(blockKey: string, masterKey: string) { // TS2339: Property 'hasOwn' does not exist on type 'ObjectConstructor'. // eslint-disable-next-line dot-notation @@ -51,10 +60,15 @@ class KVPEngineHybrid extends KVPEngineFolder { } private async getDataInBlock(blockKey: string, masterKey: string): Promise { - const obj = JSON.parse((await super.getData(blockKey)).toString('utf-8')) + // console.log(blockKey, masterKey, (await super.getData(blockKey)).toString()) + const obj = JSON.parse((await super.getData(blockKey)).toString()) if (!!obj && !!obj[masterKey]) { - return Buffer.from(obj[masterKey]) + return Buffer.from(obj[masterKey], 'binary') } else { + // 还需判断obj[masterKey]是否为空字符串 + if (obj[masterKey] === '') { + return Buffer.from('') + } return null } } @@ -74,10 +88,9 @@ class KVPEngineHybrid extends KVPEngineFolder { } // 修改对象，增加或改写键值对 - obj[masterKey] = value.toString('utf-8') + obj[masterKey] = value.toString('binary') // 已修改的对象写回本地磁盘 await super.setData(blockKey, Buffer.from(JSON.stringify(obj))) - // console.log('写入数据完毕', blockKey, masterKey, obj, JSON.stringify(obj)) // 更新使用量数据 if (!this.blockUsageMap[blockKey]) { this.blockUsageMap[blockKey] = 0 @@ -85,22 +98,23 @@ class KVPEngineHybrid extends KVPEngineFolder { this.blockUsageMap[blockKey] += calcBufSize(value) // 更新keyReferenceMap this.keyReferenceMap[masterKey] = blockKey + await this.saveMaps() } private async deleteDataInBlock(blockKey: string, masterKey: string): Promise { const obj = JSON.parse((await super.getData(blockKey)).toString()) if (obj) { - const size = calcBufSize(Buffer.from(obj[masterKey])) + const size = calcBufSize(Buffer.from(obj[masterKey], 'binary')) // 修改对象，删除键值对 delete obj[masterKey] // 已修改的对象写回本地磁盘 await super.setData(blockKey, Buffer.from(JSON.stringify(obj))) - // console.log('删除数据完毕', blockKey, masterKey, obj, JSON.stringify(obj)) // 更新使用量数据 ASSERT(this.blockUsageMap[blockKey] - size >= 0) // 断言：删除之后，block大小不应该为负数 this.blockUsageMap[blockKey] -= size // 更新keyReferenceMap delete this.keyReferenceMap[masterKey] + await this.saveMaps() } else { throw new Error("KVPEngineHybrid::deleteDataInBlock::noSuchKey: " + masterKey + "in block " + blockKey) } @@ -135,14 +149,20 @@ class KVPEngineHybrid extends KVPEngineFolder { * @param key */ public async getData(key: string): Promise { + await this.opLock.lock() if (!(await this.hasData(key))) { + this.opLock.unlock() return null } // 若属于存在block内的小文件 if (this.keyReferenceMap[key]) { - return await this.getDataInBlock(this.keyReferenceMap[key], key) + const res = await this.getDataInBlock(this.keyReferenceMap[key], key) + this.opLock.unlock() + return res } else { - return await super.getData(key) + const res = await super.getData(key) + this.opLock.unlock() + return res } } @@ -152,6 +172,8 @@ class KVPEngineHybrid extends KVPEngineFolder { * @param buf */ public async setData(key: string, buf: Buffer) { + await this.opLock.lock() + // 小文件，从单独存放转移，再存block if (calcBufSize(buf) <= config.blockInclusionThreshold) { if (this.keyReferenceMap[key]) { @@ -182,6 +204,8 @@ class KVPEngineHybrid extends KVPEngineFolder { await super.setData(key, buf) } } + + this.opLock.unlock() } /** @@ -190,11 +214,13 @@ class KVPEngineHybrid extends KVPEngineFolder { * @param buf */ public async deleteData(key: string) { + await this.opLock.lock() if (this.keyReferenceMap[key]) { await this.deleteDataInBlock(this.keyReferenceMap[key], key) } else { await super.deleteData(key) } + this.opLock.unlock() } } diff --git a/src/api/core/KVPEngines/KVPEngineJson.ts b/src/api/core/KVPEngines/KVPEngineJson.ts index 99382c8..cf47633 100644 --- a/src/api/core/KVPEngines/KVPEngineJson.ts +++ b/src/api/core/KVPEngines/KVPEngineJson.ts @@ -4,7 +4,7 @@ * Created Date: 2023-11-26 17:14:30 * Author: Guoyi * ----- - * Last Modified: 2024-02-15 11:13:37 + * Last Modified: 2024-02-15 22:49:30 * Modified By: Guoyi * ----- * Copyright (c) 2024 Guoyi Inc. @@ -90,13 +90,13 @@ class KVPEngineJson implements KVPEngineBase { * 获取数据 * @param hash */ - public getData = async (hash: string): Promise => { + public getData = async (hash: string): Promise => { // eslint-disable-next-line dot-notation if (Object['hasOwn'](this.currentJson.data, hash)) { const data: string = this.currentJson.data[hash] return await this.encryptionEngine.decrypt(Buffer.from(data, 'base64')) } else { - console.error(`这个哈希key不存在`, hash, this.currentJson) + return null } } @@ -112,10 +112,13 @@ class KVPEngineJson implements KVPEngineBase { /** * - * @param hash 根据已有键去set数据 + * @param hash 根据已有键去delete数据 * @param buf */ public deleteData = async (hash: string) => { + if (!(await this.hasData(hash))) { + throw new Error('要删除的key不存在：' + hash) + } delete this.currentJson.data[hash] await this.sync() } diff --git a/src/api/core/adapters/gcryptV1/adapter.ts b/src/api/core/adapters/gcryptV1/adapter.ts index 13d94a9..7e30c52 100644 --- a/src/api/core/adapters/gcryptV1/adapter.ts +++ b/src/api/core/adapters/gcryptV1/adapter.ts @@ -4,7 +4,7 @@ * Created Date: 2023-11-26 17:14:30 * Author: Guoyi * ----- - * Last Modified: 2024-02-15 12:36:17 + * Last Modified: 2024-02-16 11:01:43 * Modified By: Guoyi * ----- * Copyright (c) 2024 Guoyi Inc. @@ -382,7 +382,7 @@ class GcryptV1Adapter implements AdapterBase { throw new Error("GcryptV1Adapter::InvalidKey") } // 取数据 - return (await this.KVPEngine.getData(fileKey + metaKey)) ?? null + return (await this.KVPEngine.getData(metaKey + '-' + fileKey)) ?? null } /** @@ -395,12 +395,12 @@ class GcryptV1Adapter implements AdapterBase { if (!fileKey || !metaKey) { throw new Error("GcryptV1Adapter::InvalidKey") } - await this.KVPEngine.setData(fileKey + metaKey, value) + await this.KVPEngine.setData(metaKey + '-' + fileKey, value) // await this.KVPEngine.setData('[ExtraMetaKeyList]' + fileKey, Buffer.from(metaKey)) } /** - * 删除额外元数据 + * 删除额外元数据 * @param fileKey 不是文件名 * @param metaKey 长度随意的任意字符串 */ @@ -408,7 +408,19 @@ class GcryptV1Adapter implements AdapterBase { if (!fileKey || !metaKey) { throw new Error("GcryptV1Adapter::InvalidKey") } - await this.KVPEngine.deleteData(fileKey + metaKey) + await this.KVPEngine.deleteData(metaKey + '-' + fileKey) + } + + /** + * 是否存在额外元数据 + * @param fileKey 不是文件名 + * @param metaKey 长度随意的任意字符串 + */ + public async hasExtraMeta(fileKey: string, metaKey: string): Promise { + if (!fileKey || !metaKey) { + throw new Error("GcryptV1Adapter::InvalidKey") + } + return await this.KVPEngine.hasData(metaKey + '-' + fileKey) } /** diff --git a/src/api/core/encryptionEngines/EncryptionEngineNoop.ts b/src/api/core/encryptionEngines/EncryptionEngineNoop.ts index 4360816..ae40376 100644 --- a/src/api/core/encryptionEngines/EncryptionEngineNoop.ts +++ b/src/api/core/encryptionEngines/EncryptionEngineNoop.ts @@ -4,7 +4,7 @@ * Created Date: 2024-02-15 16:53:24 * Author: Guoyi * ----- - * Last Modified: 2024-02-15 17:48:55 + * Last Modified: 2024-02-16 15:40:03 * Modified By: Guoyi * ----- * Copyright (c) 2024 Guoyi Inc. diff --git a/src/api/core/types/AdapterBase.ts b/src/api/core/types/AdapterBase.ts index 75f0429..35c02f2 100644 --- a/src/api/core/types/AdapterBase.ts +++ b/src/api/core/types/AdapterBase.ts @@ -4,7 +4,7 @@ * Created Date: 2023-11-26 17:14:30 * Author: Guoyi * ----- - * Last Modified: 2024-02-15 11:13:37 + * Last Modified: 2024-02-16 10:54:14 * Modified By: Guoyi * ----- * Copyright (c) 2024 Guoyi Inc. @@ -41,6 +41,7 @@ abstract class AdapterBase { public abstract getExtraMeta?: (fileKey: string, metaKey: string) => Promise public abstract setExtraMeta?: (fileKey: string, metaKey: string, value: Buffer) => Promise public abstract deleteExtraMeta?: (fileKey: string, metaKey: string) => Promise + public abstract hasExtraMeta?: (fileKey: string, metaKey: string) => Promise } export default AdapterBase diff --git a/src/api/hash/getDigest.ts b/src/api/hash/getDigest.ts index b09df77..40ebbbd 100644 --- a/src/api/hash/getDigest.ts +++ b/src/api/hash/getDigest.ts @@ -1,6 +1,6 @@ import crypto from "crypto" -export default function getDigest(data: Buffer, digestType: string) { +export default function getDigest(data: Buffer, digestType = 'md5') { const hash = crypto.createHash(digestType).update(data).digest("hex"); return hash } diff --git a/src/components/AceEditor/AceEditor.vue b/src/components/AceEditor/AceEditor.vue index 8fa54ad..255730d 100644 --- a/src/components/AceEditor/AceEditor.vue +++ b/src/components/AceEditor/AceEditor.vue @@ -10,7 +10,7 @@ import { ref, onMounted, watch } from "vue" import sharedUtils from "@/utils/sharedUtils" import { VAceEditor } from 'vue3-ace-editor'; import File from "@/api/File"; -import getExtName from "@/utils/getExtName"; +import getExtName from "@/utils/file/getExtName"; import { useSettingsStore } from "@/store/settings" const settingsStore = useSettingsStore() diff --git a/src/components/FileMgr/DialogMgr.vue b/src/components/FileMgr/DialogMgr.vue index 59ab5e5..bd7a82b 100644 --- a/src/components/FileMgr/DialogMgr.vue +++ b/src/components/FileMgr/DialogMgr.vue @@ -128,15 +128,15 @@ export default defineComponent({ }, async handleSaveFile() { await this.adapter.writeFile(this.fileName, Buffer.from('')) - this.refresh() + await this.refresh() }, async handleSaveFolder() { await this.adapter.mkdir(this.folderName) - this.refresh() + await this.refresh() }, async handleRenameFile() { await this.adapter.renameFile(this.oldFileName, this.newFileName) - this.refresh() + await this.refresh() } } // mounted() { diff --git a/src/components/FileMgr/FileItem.vue b/src/components/FileMgr/FileItem.vue index 05a5c5d..09693c2 100644 --- a/src/components/FileMgr/FileItem.vue +++ b/src/components/FileMgr/FileItem.vue @@ -52,7 +52,7 @@