Skip to content

Commit

Permalink
v3.4.0
Browse files Browse the repository at this point in the history
  • Loading branch information
g122622 committed Feb 16, 2024
1 parent dcb924b commit 5f203b0
Show file tree
Hide file tree
Showing 21 changed files with 378 additions and 59 deletions.
2 changes: 1 addition & 1 deletion src/api/File.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import os from "os";
import path from "path";
import fs from "fs-extra";
import getExtName from "@/utils/getExtName";
import getExtName from "@/utils/file/getExtName";
import sharedUtils from "@/utils/sharedUtils";
import FileWatcher from "./FileWatcher";
import AdapterBase from "./core/types/AdapterBase";
Expand Down
2 changes: 1 addition & 1 deletion src/api/FileWatcher.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import fs from "fs-extra"
import path from "path"
import isFileOccupied from '@/utils/isFileOccupied'
import isFileOccupied from '@/utils/file/isFileOccupied'

interface Options {
minUpdateIntervalMs: number,
Expand Down
42 changes: 34 additions & 8 deletions src/api/core/KVPEngines/KVPEngineHybrid.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import KVPEngineFolder from "./KVPEngineFolder";
import calcBufSize from "@/utils/calcBufSize";
import ASSERT from "@/utils/ASSERT";
import sharedUtils from "@/utils/sharedUtils";
import Lock from "@/utils/helpers/Lock"

const config = {
blockInclusionThreshold: 12 * 1024, // 小于这个值的value将被放在block内
Expand All @@ -28,13 +29,16 @@ class KVPEngineHybrid extends KVPEngineFolder {
private keyReferenceMap = {}
// blockUsageMap用于快速查询某个block的存在情况和空间使用情况
private blockUsageMap = {}
// 用于确保操作的原子性
private opLock: Lock

/**
* 初始化jsonStorage
* @param storeEntryJsonSrc 入口json文件的绝对路径,不是data.json的路径!!!
* @param pwd 密码
*/
public async init(storeEntryJsonSrc: string, pwd: string, encryptionEngine) {
this.opLock = new Lock()
await super.init(storeEntryJsonSrc, pwd, encryptionEngine)
if (await super.hasData('@keyReferenceMap')) {
this.keyReferenceMap = JSON.parse((await super.getData('@keyReferenceMap')).toString())
Expand All @@ -44,17 +48,27 @@ class KVPEngineHybrid extends KVPEngineFolder {
}
}

private async saveMaps() {
await super.setData('@keyReferenceMap', Buffer.from(JSON.stringify(this.keyReferenceMap)))
await super.setData('@blockUsageMap', Buffer.from(JSON.stringify(this.blockUsageMap)))
}

private async hasDataInBlock(blockKey: string, masterKey: string) {
// TS2339: Property 'hasOwn' does not exist on type 'ObjectConstructor'.
// eslint-disable-next-line dot-notation
return Object['hasOwn'](JSON.parse((await super.getData(blockKey)).toString()), masterKey)
}

private async getDataInBlock(blockKey: string, masterKey: string): Promise<Buffer | null> {
const obj = JSON.parse((await super.getData(blockKey)).toString('utf-8'))
// console.log(blockKey, masterKey, (await super.getData(blockKey)).toString())
const obj = JSON.parse((await super.getData(blockKey)).toString())
if (!!obj && !!obj[masterKey]) {
return Buffer.from(obj[masterKey])
return Buffer.from(obj[masterKey], 'binary')
} else {
// 还需判断obj[masterKey]是否为空字符串
if (obj[masterKey] === '') {
return Buffer.from('')
}
return null
}
}
Expand All @@ -74,33 +88,33 @@ class KVPEngineHybrid extends KVPEngineFolder {
}

// 修改对象,增加或改写键值对
obj[masterKey] = value.toString('utf-8')
obj[masterKey] = value.toString('binary')
// 已修改的对象写回本地磁盘
await super.setData(blockKey, Buffer.from(JSON.stringify(obj)))
// console.log('写入数据完毕', blockKey, masterKey, obj, JSON.stringify(obj))
// 更新使用量数据
if (!this.blockUsageMap[blockKey]) {
this.blockUsageMap[blockKey] = 0
}
this.blockUsageMap[blockKey] += calcBufSize(value)
// 更新keyReferenceMap
this.keyReferenceMap[masterKey] = blockKey
await this.saveMaps()
}

private async deleteDataInBlock(blockKey: string, masterKey: string): Promise<void> {
const obj = JSON.parse((await super.getData(blockKey)).toString())
if (obj) {
const size = calcBufSize(Buffer.from(obj[masterKey]))
const size = calcBufSize(Buffer.from(obj[masterKey], 'binary'))
// 修改对象,删除键值对
delete obj[masterKey]
// 已修改的对象写回本地磁盘
await super.setData(blockKey, Buffer.from(JSON.stringify(obj)))
// console.log('删除数据完毕', blockKey, masterKey, obj, JSON.stringify(obj))
// 更新使用量数据
ASSERT(this.blockUsageMap[blockKey] - size >= 0) // 断言:删除之后,block大小不应该为负数
this.blockUsageMap[blockKey] -= size
// 更新keyReferenceMap
delete this.keyReferenceMap[masterKey]
await this.saveMaps()
} else {
throw new Error("KVPEngineHybrid::deleteDataInBlock::noSuchKey: " + masterKey + "in block " + blockKey)
}
Expand Down Expand Up @@ -135,14 +149,20 @@ class KVPEngineHybrid extends KVPEngineFolder {
* @param key
*/
public async getData(key: string): Promise<Buffer | null> {
await this.opLock.lock()
if (!(await this.hasData(key))) {
this.opLock.unlock()
return null
}
// 若属于存在block内的小文件
if (this.keyReferenceMap[key]) {
return await this.getDataInBlock(this.keyReferenceMap[key], key)
const res = await this.getDataInBlock(this.keyReferenceMap[key], key)
this.opLock.unlock()
return res
} else {
return await super.getData(key)
const res = await super.getData(key)
this.opLock.unlock()
return res
}
}

Expand All @@ -152,6 +172,8 @@ class KVPEngineHybrid extends KVPEngineFolder {
* @param buf
*/
public async setData(key: string, buf: Buffer) {
await this.opLock.lock()

// 小文件,从单独存放转移,再存block
if (calcBufSize(buf) <= config.blockInclusionThreshold) {
if (this.keyReferenceMap[key]) {
Expand Down Expand Up @@ -182,6 +204,8 @@ class KVPEngineHybrid extends KVPEngineFolder {
await super.setData(key, buf)
}
}

this.opLock.unlock()
}

/**
Expand All @@ -190,11 +214,13 @@ class KVPEngineHybrid extends KVPEngineFolder {
* @param buf
*/
public async deleteData(key: string) {
await this.opLock.lock()
if (this.keyReferenceMap[key]) {
await this.deleteDataInBlock(this.keyReferenceMap[key], key)
} else {
await super.deleteData(key)
}
this.opLock.unlock()
}
}

Expand Down
11 changes: 7 additions & 4 deletions src/api/core/KVPEngines/KVPEngineJson.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* Created Date: 2023-11-26 17:14:30
* Author: Guoyi
* -----
* Last Modified: 2024-02-15 11:13:37
* Last Modified: 2024-02-15 22:49:30
* Modified By: Guoyi
* -----
* Copyright (c) 2024 Guoyi Inc.
Expand Down Expand Up @@ -90,13 +90,13 @@ class KVPEngineJson implements KVPEngineBase {
* 获取数据
* @param hash
*/
public getData = async (hash: string): Promise<Buffer> => {
public getData = async (hash: string): Promise<Buffer | null> => {
// eslint-disable-next-line dot-notation
if (Object['hasOwn'](this.currentJson.data, hash)) {
const data: string = this.currentJson.data[hash]
return await this.encryptionEngine.decrypt(Buffer.from(data, 'base64'))
} else {
console.error(`这个哈希key不存在`, hash, this.currentJson)
return null
}
}

Expand All @@ -112,10 +112,13 @@ class KVPEngineJson implements KVPEngineBase {

/**
*
* @param hash 根据已有键去set数据
* @param hash 根据已有键去delete数据
* @param buf
*/
public deleteData = async (hash: string) => {
if (!(await this.hasData(hash))) {
throw new Error('要删除的key不存在:' + hash)
}
delete this.currentJson.data[hash]
await this.sync()
}
Expand Down
22 changes: 17 additions & 5 deletions src/api/core/adapters/gcryptV1/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* Created Date: 2023-11-26 17:14:30
* Author: Guoyi
* -----
* Last Modified: 2024-02-15 12:36:17
* Last Modified: 2024-02-16 11:01:43
* Modified By: Guoyi
* -----
* Copyright (c) 2024 Guoyi Inc.
Expand Down Expand Up @@ -382,7 +382,7 @@ class GcryptV1Adapter implements AdapterBase {
throw new Error("GcryptV1Adapter::InvalidKey")
}
// 取数据
return (await this.KVPEngine.getData(fileKey + metaKey)) ?? null
return (await this.KVPEngine.getData(metaKey + '-' + fileKey)) ?? null
}

/**
Expand All @@ -395,20 +395,32 @@ class GcryptV1Adapter implements AdapterBase {
if (!fileKey || !metaKey) {
throw new Error("GcryptV1Adapter::InvalidKey")
}
await this.KVPEngine.setData(fileKey + metaKey, value)
await this.KVPEngine.setData(metaKey + '-' + fileKey, value)
// await this.KVPEngine.setData('[ExtraMetaKeyList]' + fileKey, Buffer.from(metaKey))
}

/**
* 删除额外元数据
* 删除额外元数据
* @param fileKey 不是文件名
* @param metaKey 长度随意的任意字符串
*/
public async deleteExtraMeta(fileKey: string, metaKey: string): Promise<void> {
if (!fileKey || !metaKey) {
throw new Error("GcryptV1Adapter::InvalidKey")
}
await this.KVPEngine.deleteData(fileKey + metaKey)
await this.KVPEngine.deleteData(metaKey + '-' + fileKey)
}

/**
* 是否存在额外元数据
* @param fileKey 不是文件名
* @param metaKey 长度随意的任意字符串
*/
public async hasExtraMeta(fileKey: string, metaKey: string): Promise<boolean> {
if (!fileKey || !metaKey) {
throw new Error("GcryptV1Adapter::InvalidKey")
}
return await this.KVPEngine.hasData(metaKey + '-' + fileKey)
}

/**
Expand Down
2 changes: 1 addition & 1 deletion src/api/core/encryptionEngines/EncryptionEngineNoop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* Created Date: 2024-02-15 16:53:24
* Author: Guoyi
* -----
* Last Modified: 2024-02-15 17:48:55
* Last Modified: 2024-02-16 15:40:03
* Modified By: Guoyi
* -----
* Copyright (c) 2024 Guoyi Inc.
Expand Down
3 changes: 2 additions & 1 deletion src/api/core/types/AdapterBase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* Created Date: 2023-11-26 17:14:30
* Author: Guoyi
* -----
* Last Modified: 2024-02-15 11:13:37
* Last Modified: 2024-02-16 10:54:14
* Modified By: Guoyi
* -----
* Copyright (c) 2024 Guoyi Inc.
Expand Down Expand Up @@ -41,6 +41,7 @@ abstract class AdapterBase {
public abstract getExtraMeta?: (fileKey: string, metaKey: string) => Promise<Buffer | null>
public abstract setExtraMeta?: (fileKey: string, metaKey: string, value: Buffer) => Promise<void>
public abstract deleteExtraMeta?: (fileKey: string, metaKey: string) => Promise<void>
public abstract hasExtraMeta?: (fileKey: string, metaKey: string) => Promise<boolean>
}

export default AdapterBase
2 changes: 1 addition & 1 deletion src/api/hash/getDigest.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import crypto from "crypto"

export default function getDigest(data: Buffer, digestType: string) {
export default function getDigest(data: Buffer, digestType = 'md5') {
const hash = crypto.createHash(digestType).update(data).digest("hex");
return hash
}
2 changes: 1 addition & 1 deletion src/components/AceEditor/AceEditor.vue
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { ref, onMounted, watch } from "vue"
import sharedUtils from "@/utils/sharedUtils"
import { VAceEditor } from 'vue3-ace-editor';
import File from "@/api/File";
import getExtName from "@/utils/getExtName";
import getExtName from "@/utils/file/getExtName";
import { useSettingsStore } from "@/store/settings"
const settingsStore = useSettingsStore()
Expand Down
6 changes: 3 additions & 3 deletions src/components/FileMgr/DialogMgr.vue
Original file line number Diff line number Diff line change
Expand Up @@ -128,15 +128,15 @@ export default defineComponent({
},
async handleSaveFile() {
await this.adapter.writeFile(this.fileName, Buffer.from(''))
this.refresh()
await this.refresh()
},
async handleSaveFolder() {
await this.adapter.mkdir(this.folderName)
this.refresh()
await this.refresh()
},
async handleRenameFile() {
await this.adapter.renameFile(this.oldFileName, this.newFileName)
this.refresh()
await this.refresh()
}
}
// mounted() {
Expand Down
9 changes: 7 additions & 2 deletions src/components/FileMgr/FileItem.vue
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@

<script setup lang="ts">
import DirSingleItem from "@/api/core/types/DirSingleItem";
import getFileType from "@/utils/getFileType";
import getFileType from "@/utils/file/getFileType";
import { computed, ref, onMounted, onUnmounted } from "vue";
import { useMainStore } from "@/store/main"
import prettyBytes from "@/utils/prettyBytes";
Expand Down Expand Up @@ -145,12 +145,17 @@ onMounted(async () => {
// 当thumbnailBuf为null
// 尝试生成缩略图
idleCallbackId = requestIdleCallback(async () => {
if (!(await props.adapter.hasExtraMeta(props.singleFileItem.key, 'fileOriginalPath'))) {
return
}
let fileOriginalPath = null
try {
const fileOriginalPath = (await props.adapter.getExtraMeta(props.singleFileItem.key, 'fileOriginalPath')).toString()
fileOriginalPath = (await props.adapter.getExtraMeta(props.singleFileItem.key, 'fileOriginalPath')).toString()
currentThumbnail.value = await getThumbnailFromSystem(fileOriginalPath, { height: 256, width: 256, quality: 90 })
await props.adapter.setExtraMeta(props.singleFileItem.key, 'thumbnail', Buffer.from(currentThumbnail.value))
} catch (e) {
await props.adapter.setExtraMeta(props.singleFileItem.key, 'thumbnail', Buffer.from('n/a'))
warn(`从系统获取缩略图失败,错误:${e.toString()},路径:${fileOriginalPath}`)
} finally {
// 收尾工作,清理掉fileOriginalPath
await props.adapter.deleteExtraMeta(props.singleFileItem.key, 'fileOriginalPath')
Expand Down
6 changes: 3 additions & 3 deletions src/components/FileMgr/FileMgr.vue
Original file line number Diff line number Diff line change
Expand Up @@ -142,14 +142,14 @@
<script setup lang="ts">
import { ref, computed, watch, reactive, onMounted } from "vue";
import Addr from "@/api/core/common/Addr";
import getFileName from "@/utils/getFileName";
import getFileName from "@/utils/file/getFileName";
import lodash from "lodash";
import DirSingleItem from "@/api/core/types/DirSingleItem";
import FileTable from "@/api/core/types/FileTable";
import emitter from "@/eventBus";
import AdapterBase from "@/api/core/types/AdapterBase";
import File from "@/api/File";
import getExtName from "@/utils/getExtName";
import getExtName from "@/utils/file/getExtName";
import { useTaskStore } from '@/store/task'
import Task from "@/api/Task";
import sharedUtils from "@/utils/sharedUtils";
Expand Down Expand Up @@ -317,7 +317,7 @@ const importFile = async (files: FileList) => {
fileKey = await props.adapter.writeFile(getFileName(file.path, true), dataBuf)
// 处理缩略图相关逻辑
if (options.useThumbnails && props.adapter.setExtraMeta) {
props.adapter.setExtraMeta(fileKey, 'fileOriginalPath', Buffer.from(file.path))
await props.adapter.setExtraMeta(fileKey, 'fileOriginalPath', Buffer.from(file.path))
}
}
reader.readAsArrayBuffer(file);
Expand Down
11 changes: 11 additions & 0 deletions src/components/SystemBar.vue
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,17 @@ const taskStore = useTaskStore()
const pendingOrRunningTaskAmount = computed<number>(() => taskStore.getPendingOrRunningTaskAmount())
const itemList = computed(() => {
return [
{
name: 'test',
tooltip: '运行测试函数',
onClick: () => {
// eslint-disable-next-line dot-notation
window['runTest']()
},
icon: 'mdi-test-tube',
class: 'system-bar-item-normal',
hide: sharedUtils.env === 'production'
},
{
name: 'devtools',
tooltip: '切换开发者工具',
Expand Down
Loading

0 comments on commit 5f203b0

Please sign in to comment.