From b14a1db2f95ca4888b6cf4ba4dafd0302af3d540 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Mon, 18 Dec 2023 20:35:17 +0800 Subject: [PATCH] fix: worker un exit (#624) --- packages/global/common/string/textSplitter.ts | 2 +- packages/service/common/string/markdown.ts | 3 + pnpm-lock.yaml | 35 -------- .../src/components/Markdown/chat/Guide.tsx | 2 +- .../app/src/components/SideTabs/index.tsx | 2 +- .../Flow/components/modules/ButtonEdge.tsx | 8 +- .../detail/components/Import/Provider.tsx | 84 +++++-------------- projects/app/src/web/styles/theme.ts | 20 ++--- worker/html2md.js | 32 +++---- 9 files changed, 58 insertions(+), 130 deletions(-) diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts index 29d4ad8e3ad8..fca35471b532 100644 --- a/packages/global/common/string/textSplitter.ts +++ b/packages/global/common/string/textSplitter.ts @@ -232,7 +232,7 @@ export const splitText2Chunks = (props: { step: 0, lastText: '', mdTitle: '' - }).map((chunk) => chunk.replaceAll(codeBlockMarker, '\n')); // restore code block + }).map((chunk) => chunk?.replaceAll(codeBlockMarker, '\n') || ''); // restore code block const tokens = countTokens ? chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0) diff --git a/packages/service/common/string/markdown.ts b/packages/service/common/string/markdown.ts index f444d1f7fa9e..fd292d86120d 100644 --- a/packages/service/common/string/markdown.ts +++ b/packages/service/common/string/markdown.ts @@ -13,9 +13,12 @@ export const htmlToMarkdown = (html?: string | null) => const worker = new Worker(getWorkerPath('html2md')); worker.on('message', (md: string) => { + worker.terminate(); + resolve(simpleMarkdownText(md)); }); worker.on('error', (err) => { + worker.terminate(); reject(err); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bae5f97d0318..c85d34ac7698 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -84,15 +84,9 @@ importers: dayjs: specifier: ^1.11.7 version: registry.npmmirror.com/dayjs@1.11.7 - domino-ext: - specifier: ^2.1.4 - version: registry.npmmirror.com/domino-ext@2.1.4 encoding: specifier: ^0.1.13 version: registry.npmmirror.com/encoding@0.1.13 - joplin-turndown-plugin-gfm: - specifier: ^1.0.12 - version: registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12 jsonwebtoken: specifier: ^9.0.2 version: registry.npmmirror.com/jsonwebtoken@9.0.2 @@ -117,9 +111,6 @@ importers: tunnel: specifier: ^0.0.6 version: registry.npmmirror.com/tunnel@0.0.6 - turndown: - specifier: ^7.1.2 - version: registry.npmmirror.com/turndown@7.1.2 devDependencies: '@types/cookie': specifier: ^0.5.2 @@ -6765,18 +6756,6 @@ packages: domelementtype: registry.npmmirror.com/domelementtype@2.3.0 dev: false - registry.npmmirror.com/domino-ext@2.1.4: - resolution: {integrity: sha512-t8piRI9Qahd4V/NqnCcqdBWsQ4OYeOvcTuoHl38Pzk9OJJ/UiCYHA2jX2fACmBtDlSMiWa0uR524KuLEAMc/3Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domino-ext/-/domino-ext-2.1.4.tgz} - name: domino-ext - version: 2.1.4 - dev: false - - registry.npmmirror.com/domino@2.1.6: - resolution: {integrity: sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domino/-/domino-2.1.6.tgz} - name: domino - version: 2.1.6 - dev: false - registry.npmmirror.com/dompurify@3.0.3: resolution: {integrity: sha512-axQ9zieHLnAnHh0sfAamKYiqXMJAVwu+LM/alQ7WDagoWessyWvMSFyW65CqF3owufNu8HBcE4cM2Vflu7YWcQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dompurify/-/dompurify-3.0.3.tgz} name: dompurify @@ -8793,12 +8772,6 @@ packages: set-function-name: registry.npmmirror.com/set-function-name@2.0.1 dev: true - registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12: - resolution: {integrity: sha512-qL4+1iycQjZ1fs8zk3jSRk7cg3ROBUHk7GKtiLAQLFzLPKErnILUvz5DLszSQvz3s1sTjPbywLDISVUtBY6HaA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/joplin-turndown-plugin-gfm/-/joplin-turndown-plugin-gfm-1.0.12.tgz} - name: joplin-turndown-plugin-gfm - version: 1.0.12 - dev: false - registry.npmmirror.com/js-sdsl@4.4.2: resolution: {integrity: sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/js-sdsl/-/js-sdsl-4.4.2.tgz} name: js-sdsl @@ -12193,14 +12166,6 @@ packages: engines: {node: '>=0.6.11 <=0.7.0 || >=0.7.3'} dev: false - registry.npmmirror.com/turndown@7.1.2: - resolution: {integrity: sha512-ntI9R7fcUKjqBP6QU8rBK2Ehyt8LAzt3UBT9JR9tgo6GtuKvyUzpayWmeMKJw1DPdXzktvtIT8m2mVXz+bL/Qg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/turndown/-/turndown-7.1.2.tgz} - name: turndown - version: 7.1.2 - dependencies: - domino: registry.npmmirror.com/domino@2.1.6 - dev: false - registry.npmmirror.com/type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/type-check/-/type-check-0.4.0.tgz} name: type-check diff --git a/projects/app/src/components/Markdown/chat/Guide.tsx b/projects/app/src/components/Markdown/chat/Guide.tsx index 3c6740662378..558d298e362e 100644 --- a/projects/app/src/components/Markdown/chat/Guide.tsx +++ b/projects/app/src/components/Markdown/chat/Guide.tsx @@ -23,7 +23,7 @@ function MyLink(e: any) { { diff --git a/projects/app/src/components/SideTabs/index.tsx b/projects/app/src/components/SideTabs/index.tsx index 81946e0cdaa7..278ed6fe5c37 100644 --- a/projects/app/src/components/SideTabs/index.tsx +++ b/projects/app/src/components/SideTabs/index.tsx @@ -44,7 +44,7 @@ const SideTabs = ({ list, size = 'md', activeId, onChange, ...props }: Props) => alignItems={'center'} {...(activeId === item.id ? { - bg: ' blue.200 !important', + bg: ' blue.100 !important', fontWeight: 'bold', color: 'blue.600 ', cursor: 'default' diff --git a/projects/app/src/components/core/module/Flow/components/modules/ButtonEdge.tsx b/projects/app/src/components/core/module/Flow/components/modules/ButtonEdge.tsx index a3d6ccfe3e71..4139819ad46d 100644 --- a/projects/app/src/components/core/module/Flow/components/modules/ButtonEdge.tsx +++ b/projects/app/src/components/core/module/Flow/components/modules/ButtonEdge.tsx @@ -1,11 +1,5 @@ import React from 'react'; -import { - SmoothStepEdge, - EdgeLabelRenderer, - EdgeProps, - getSmoothStepPath, - MarkerType -} from 'reactflow'; +import { SmoothStepEdge, EdgeLabelRenderer, EdgeProps, getSmoothStepPath } from 'reactflow'; import { Flex } from '@chakra-ui/react'; import MyIcon from '@/components/Icon'; diff --git a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx index 0e512fd91515..6a7693fe5ff5 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx @@ -289,28 +289,30 @@ export const PreviewFileOrChunk = () => { overflow={'overlay'} px={[4, 8]} my={4} - contentEditable - dangerouslySetInnerHTML={{ __html: previewFile.rawText }} + // contentEditable + // dangerouslySetInnerHTML={{ __html: previewFile.rawText }} fontSize={'sm'} whiteSpace={'pre-wrap'} wordBreak={'break-all'} - onBlur={(e) => { - // @ts-ignore - const val = e.target.innerText; - setReShowRePreview(true); + // onBlur={(e) => { + // // @ts-ignore + // const val = e.target.innerText; + // setReShowRePreview(true); - setFiles((state) => - state.map((file) => - file.id === previewFile.id - ? { - ...file, - text: val - } - : file - ) - ); - }} - /> + // setFiles((state) => + // state.map((file) => + // file.id === previewFile.id + // ? { + // ...file, + // text: val + // } + // : file + // ) + // ); + // }} + > + {previewFile.rawText} + ) : ( @@ -365,49 +367,9 @@ export const PreviewFileOrChunk = () => { }} /> - { - // @ts-ignore - const val = e.target.innerText; - - /* delete file */ - if (val === '') { - setFiles((state) => - state.map((stateFile) => - stateFile.id === file.id - ? { - ...file, - chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)] - } - : stateFile - ) - ); - } else { - // update chunk - setFiles((stateFiles) => - stateFiles.map((stateFile) => - file.id === stateFile.id - ? { - ...stateFile, - chunks: stateFile.chunks.map((chunk, index) => ({ - ...chunk, - index: i === index ? val : chunk.q - })) - } - : stateFile - ) - ); - } - }} - /> + + {chunk.a ? `q:${chunk.q}\na:${chunk.a}` : chunk.q} + )) )} diff --git a/projects/app/src/web/styles/theme.ts b/projects/app/src/web/styles/theme.ts index 6a158bc5321a..1f59f94d8d79 100644 --- a/projects/app/src/web/styles/theme.ts +++ b/projects/app/src/web/styles/theme.ts @@ -284,16 +284,16 @@ export const theme = extendTheme({ 1000: '#121416' }, blue: { - 50: '#f0f7ff', - 100: '#EBF7FD', - 200: '#d6e8ff', - 300: '#adceff', - 400: '#85b1ff', - 500: '#4e83fd', - 600: '#3370ff', - 700: '#2152d9', - 800: '#1237b3', - 900: '#07228c' + 50: '#F0F4FF', + 100: '#E1EAFF', + 200: '#C5D7FF', + 300: '#94B5FF', + 400: '#5E8FFF', + 500: '#487FFF', + 600: '#3370FF', + 700: '#2B5FD9', + 800: '#2450B5', + 900: '#1D4091' }, myRead: { 600: '#ff4d4f' diff --git a/worker/html2md.js b/worker/html2md.js index 545f0b6e70c9..73798c855260 100644 --- a/worker/html2md.js +++ b/worker/html2md.js @@ -20,22 +20,26 @@ parentPort?.on('message', (html) => { }); const html2md = (html) => { - const window = domino.createWindow(html); - const document = window.document; + try { + const window = domino.createWindow(html); + const document = window.document; - turndownService.remove(['i', 'script', 'iframe']); - turndownService.addRule('codeBlock', { - filter: 'pre', - replacement(_, node) { - const content = node.textContent?.trim() || ''; - // @ts-ignore - const codeName = node?._attrsByQName?.class?.data?.trim() || ''; + turndownService.remove(['i', 'script', 'iframe']); + turndownService.addRule('codeBlock', { + filter: 'pre', + replacement(_, node) { + const content = node.textContent?.trim() || ''; + // @ts-ignore + const codeName = node?._attrsByQName?.class?.data?.trim() || ''; - return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`; - } - }); + return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`; + } + }); - turndownService.use(turndownPluginGfm.gfm); + turndownService.use(turndownPluginGfm.gfm); - return turndownService.turndown(document); + return turndownService.turndown(document); + } catch (error) { + return ''; + } };