From ab25bd76c1617abd41bc3b73a39f6d35a9ee71f0 Mon Sep 17 00:00:00 2001 From: Lucas Jackson Date: Mon, 13 May 2024 14:32:14 -0700 Subject: [PATCH] Update UI for extraction graph updates (#583) * fix indexes loading policy name * update indexify + update extraction policy graph + improve search index page * add task count in extraction graph table * version bump --- Cargo.lock | 2 +- Cargo.toml | 2 +- ui/package-lock.json | 29 +++++++++-- ui/package.json | 2 +- ui/src/components/ExtractionGraphs.tsx | 53 ++++++++++++++------ ui/src/components/ExtractionPolicyItem.tsx | 13 +++-- ui/src/components/SearchResultCard.tsx | 20 +++++--- ui/src/components/TasksTable.tsx | 12 ++++- ui/src/components/tables/ContentTable.tsx | 16 ++++-- ui/src/components/tables/ExtractorsTable.tsx | 13 ++++- ui/src/components/tables/IndexTable.tsx | 39 ++++++++++---- ui/src/components/tables/SchemasTable.tsx | 36 +++++++++---- ui/src/index.tsx | 2 +- ui/src/routes/Namespace/content.tsx | 8 +-- ui/src/routes/Namespace/extractionPolicy.tsx | 11 ++-- ui/src/routes/Namespace/index.tsx | 34 ++++++++++--- ui/src/types.ts | 1 + 17 files changed, 216 insertions(+), 77 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 674dd21c2..60b9891de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3339,7 +3339,7 @@ dependencies = [ [[package]] name = "indexify" -version = "0.0.39" +version = "0.0.40" dependencies = [ "anyerror", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 9586a4b41..fed111ff6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "indexify" -version = "0.0.39" +version = "0.0.40" edition = "2021" authors = ["Diptanu Gon Choudhury "] build = "build.rs" diff --git a/ui/package-lock.json b/ui/package-lock.json index 5284898de..8c2db3a45 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -24,7 +24,7 @@ "@types/react-router-dom": "^5.3.3", "axios": "^1.6.7", "crypto": "^1.0.1", - "getindexify": "^0.0.25", + "getindexify": "^0.0.42", "moment": "^2.30.1", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -7115,6 +7115,11 @@ "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==", "deprecated": "This package is no longer supported. It's now a built-in Node module. If you've depended on crypto, you should switch to the one that's built-in." }, + "node_modules/crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" + }, "node_modules/crypto-random-string": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", @@ -9723,11 +9728,25 @@ } }, "node_modules/getindexify": { - "version": "0.0.25", - "resolved": "https://registry.npmjs.org/getindexify/-/getindexify-0.0.25.tgz", - "integrity": "sha512-NDnVk04G+HBUHbK5OzP8tCHUa6Q5FoCTGxjYe++JFEq6CT5+z68IKGbcXgz68n6HRLaI90kVN3OyySiHSRqnjQ==", + "version": "0.0.42", + "resolved": "https://registry.npmjs.org/getindexify/-/getindexify-0.0.42.tgz", + "integrity": "sha512-s40Bh79iu3Vxr5J2Rs/WCiaZb8RZWf5sBpbZ5AjEuENUVkkOLPN4xzUA+XyJCYufeQz40xjPU7uaqb7b1vB5eQ==", "dependencies": { - "axios": "^1.6.7" + "axios": "^1.6.7", + "crypto-js": "^4.2.0", + "uuid": "^9.0.1" + } + }, + "node_modules/getindexify/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" } }, "node_modules/glob": { diff --git a/ui/package.json b/ui/package.json index 9d24fd9b0..7b89b5842 100644 --- a/ui/package.json +++ b/ui/package.json @@ -20,7 +20,7 @@ "@types/react-router-dom": "^5.3.3", "axios": "^1.6.7", "crypto": "^1.0.1", - "getindexify": "^0.0.25", + "getindexify": "^0.0.42", "moment": "^2.30.1", "react": "^18.2.0", "react-dom": "^18.2.0", diff --git a/ui/src/components/ExtractionGraphs.tsx b/ui/src/components/ExtractionGraphs.tsx index d01a77388..4c85f4aa1 100644 --- a/ui/src/components/ExtractionGraphs.tsx +++ b/ui/src/components/ExtractionGraphs.tsx @@ -1,28 +1,37 @@ -import { IExtractionPolicy, IExtractor } from "getindexify"; -import { Alert, Typography } from "@mui/material"; +import { + IExtractionGraph, + IExtractionPolicy, + IExtractor, + ITask, +} from "getindexify"; +import { Alert, IconButton, Typography } from "@mui/material"; import { Box, Stack } from "@mui/system"; import React, { ReactElement } from "react"; import GavelIcon from "@mui/icons-material/Gavel"; +import InfoIcon from "@mui/icons-material/Info"; import ExtractionPolicyItem from "./ExtractionPolicyItem"; import { IExtractionGraphCol, IExtractionGraphColumns } from "../types"; const ExtractionGraphs = ({ - extractionPolicies, + extractionGraphs, namespace, extractors, + tasks, }: { - extractionPolicies: IExtractionPolicy[]; + extractionGraphs: IExtractionGraph[]; namespace: string; - extractors:IExtractor[] + extractors: IExtractor[]; + tasks: ITask[]; }) => { const itemheight = 60; const cols: IExtractionGraphColumns = { name: { displayName: "Name", width: 350 }, - extractor: { displayName: "Extractor", width: 250 }, - mimeTypes: { displayName: "Input MimeTypes", width: 250 }, - inputParams: { displayName: "Input Parameters", width: 250 }, + extractor: { displayName: "Extractor", width: 225 }, + mimeTypes: { displayName: "Input MimeTypes", width: 225}, + inputParams: { displayName: "Input Parameters", width: 225 }, + taskCount: { displayName: "Tasks", width: 75 }, }; - + const renderHeader = () => { return ( { - if (extractionPolicies.length === 0) { + if (extractionGraphs.length === 0) { return ( - No Policies Found + No Graphs Found ); @@ -97,9 +107,14 @@ const ExtractionGraphs = ({ }} >
{renderHeader()}
- - {renderGraphItems(extractionPolicies, "ingestion")} - + {extractionGraphs.map((graph) => { + return ( + + {graph.name} + {renderGraphItems(graph.extraction_policies, "")} + + ); + })} ); }; @@ -113,7 +128,15 @@ const ExtractionGraphs = ({ spacing={2} > - Extraction Graphs + + Extraction Graphs + + + + {renderContent()} diff --git a/ui/src/components/ExtractionPolicyItem.tsx b/ui/src/components/ExtractionPolicyItem.tsx index d805d3974..73503b1c6 100644 --- a/ui/src/components/ExtractionPolicyItem.tsx +++ b/ui/src/components/ExtractionPolicyItem.tsx @@ -1,5 +1,5 @@ import { Box, Chip, Stack, Typography } from "@mui/material"; -import { IExtractionPolicy, IExtractor, IIndex } from "getindexify"; +import { IExtractionPolicy, IExtractor, IIndex, ITask } from "getindexify"; import { IExtractionGraphColumns } from "../types"; import { Link } from "react-router-dom"; @@ -11,6 +11,7 @@ const ExtractionPolicyItem = ({ depth, itemHeight, extractors, + tasks, index, }: { extractionPolicy: IExtractionPolicy; @@ -20,11 +21,12 @@ const ExtractionPolicyItem = ({ depth: number; itemHeight: number; extractors: IExtractor[]; + tasks: ITask[]; index?: IIndex; }) => { const renderInputParams = () => { if ( - extractionPolicy.input_params === undefined || + !extractionPolicy.input_params || Object.keys(extractionPolicy.input_params).length === 0 ) { return ; @@ -48,7 +50,7 @@ const ExtractionPolicyItem = ({ if (!extractor) return null; return ( - + {(extractor.input_mime_types ?? []).map((val: string) => { return ( @@ -107,7 +109,7 @@ const ExtractionPolicyItem = ({ > {depth > 0 && } {extractionPolicy.name} @@ -119,6 +121,9 @@ const ExtractionPolicyItem = ({ {renderInputParams()} + + {tasks.filter(task => task.extraction_policy_id === extractionPolicy.id).length} + ); diff --git a/ui/src/components/SearchResultCard.tsx b/ui/src/components/SearchResultCard.tsx index 65077322f..1275e62f0 100644 --- a/ui/src/components/SearchResultCard.tsx +++ b/ui/src/components/SearchResultCard.tsx @@ -31,16 +31,24 @@ const SearchResultCard = ({ return ( - Content ID: {data.content_id} - - - - {data.text} + Content ID:{" "} + + {data.content_id} + + + {data.text ? ( + <> + + + {data.text} + + + ) : null} {Object.keys(data.labels).length !== 0 && ( - + {Object.keys(data.labels).map((val: string) => { return ; })} diff --git a/ui/src/components/TasksTable.tsx b/ui/src/components/TasksTable.tsx index f0ec64a7c..bcba748a2 100644 --- a/ui/src/components/TasksTable.tsx +++ b/ui/src/components/TasksTable.tsx @@ -6,6 +6,7 @@ import { Box, Stack } from "@mui/system"; import TaskIcon from '@mui/icons-material/Task'; import moment from "moment"; import { Link } from "react-router-dom"; +import {TaskStatus} from "getindexify" const TasksTable = ({ namespace, @@ -40,7 +41,7 @@ const TasksTable = ({ headerName: "Extraction Policy", renderCell: (params) => { const policy = policies.find(policy => policy.id === params.value) - return policy ? + return policy ? {policy?.name} : null }, @@ -49,6 +50,15 @@ const TasksTable = ({ { field: "outcome", headerName: "Outcome", + valueGetter: (params) => { + if (params.value === TaskStatus.Failure) { + return "Failure" + } else if (params.value === TaskStatus.Success) { + return "Success" + } else { + return "Unknown" + } + }, width: 100, }, { diff --git a/ui/src/components/tables/ContentTable.tsx b/ui/src/components/tables/ContentTable.tsx index 4b848b897..605515c31 100644 --- a/ui/src/components/tables/ContentTable.tsx +++ b/ui/src/components/tables/ContentTable.tsx @@ -11,9 +11,11 @@ import { Tabs, TextField, Typography, + IconButton, } from "@mui/material"; import { Box, Stack } from "@mui/system"; import ArticleIcon from "@mui/icons-material/Article"; +import InfoIcon from "@mui/icons-material/Info"; import React, { useEffect, useState } from "react"; import moment from "moment"; import { Link } from "react-router-dom"; @@ -61,7 +63,7 @@ const ContentTable = ({ }>({ contentId: "", policyName: "Any" }); const [filteredContent, setFilteredContent] = useState( - content.filter((c) => c.source === "ingestion") + content.filter((c) => c.source === "") ); const [currentTab, setCurrentTab] = useState("ingested"); @@ -112,7 +114,7 @@ const ContentTable = ({ } else if (currentTab === "ingested") { // go back to root node of graph tab setGraphTabIds([]); - setFilteredContent([...content.filter((c) => c.source === "ingestion")]); + setFilteredContent([...content.filter((c) => c.source === "")]); } else { // current tab is now a content id // remove tabs after id: selectedValue if possible @@ -265,7 +267,15 @@ const ContentTable = ({ spacing={2} > - Content + + Content + + + + { @@ -125,7 +126,15 @@ const ExtractorsTable = ({ extractors }: { extractors: Extractor[] }) => { spacing={2} > - Extractors + + Extractors + + + + {renderContent()} diff --git a/ui/src/components/tables/IndexTable.tsx b/ui/src/components/tables/IndexTable.tsx index 8b382d2ae..64554c700 100644 --- a/ui/src/components/tables/IndexTable.tsx +++ b/ui/src/components/tables/IndexTable.tsx @@ -1,29 +1,34 @@ import { DataGrid, GridColDef } from "@mui/x-data-grid"; import { IExtractionPolicy, IIndex } from "getindexify"; -import { Alert, Typography } from "@mui/material"; +import { Alert, IconButton, Typography } from "@mui/material"; import { Box, Stack } from "@mui/system"; import ManageSearchIcon from "@mui/icons-material/ManageSearch"; +import InfoIcon from "@mui/icons-material/Info"; import React from "react"; import { Link } from "react-router-dom"; const IndexTable = ({ indexes, namespace, - extractionPolicies + extractionPolicies, }: { indexes: IIndex[]; namespace: string; - extractionPolicies: IExtractionPolicy[] + extractionPolicies: IExtractionPolicy[]; }) => { - const getPolicyFromIndexname = (indexName:string):IExtractionPolicy | undefined => { - return extractionPolicies.find(policy => String(indexName).startsWith(policy.name)) - } + const getPolicyFromIndexname = ( + indexName: string + ): IExtractionPolicy | undefined => { + return extractionPolicies.find((policy) => + String(indexName).startsWith(`${policy.graph_name}.${policy.name}`) + ); + }; const columns: GridColDef[] = [ { field: "name", headerName: "Name", - width: 300, + width: 500, renderCell: (params) => { return ( @@ -37,12 +42,16 @@ const IndexTable = ({ headerName: "Policy Name", width: 300, renderCell: (params) => { - const policy = getPolicyFromIndexname(params.row.name) + const policy = getPolicyFromIndexname(params.row.name); if (!policy) { - return null + return null; } return ( - {policy.name} + + {policy.name} + ); }, }, @@ -96,7 +105,15 @@ const IndexTable = ({ spacing={2} > - Indexes + + Indexes + + + + {renderContent()} diff --git a/ui/src/components/tables/SchemasTable.tsx b/ui/src/components/tables/SchemasTable.tsx index aef77b094..cf9a4750b 100644 --- a/ui/src/components/tables/SchemasTable.tsx +++ b/ui/src/components/tables/SchemasTable.tsx @@ -1,17 +1,27 @@ import React from "react"; import { DataGrid, GridColDef } from "@mui/x-data-grid"; -import { Alert, Chip, Typography } from "@mui/material"; +import { Alert, Chip, IconButton, Typography } from "@mui/material"; import { Box, Stack } from "@mui/system"; import StorageIcon from "@mui/icons-material/Storage"; +import InfoIcon from "@mui/icons-material/Info"; import { ISchema } from "getindexify"; const SchemasTable = ({ schemas }: { schemas: ISchema[] }) => { + const getRowId = (row: ISchema) => { + return row.id; + }; + const columns: GridColDef[] = [ - { field: "content_source", headerName: "Content Source", width: 300 }, + { field: "namespace", headerName: "namespace", width: 200 }, + { + field: "extraction_graph_name", + headerName: "Extraction Graph", + width: 250, + }, { field: "columns", headerName: "Columns", - width: 250, + width: 500, renderCell: (params) => { if (!params.value) { return None; @@ -23,7 +33,7 @@ const SchemasTable = ({ schemas }: { schemas: ISchema[] }) => { ))} @@ -33,10 +43,6 @@ const SchemasTable = ({ schemas }: { schemas: ISchema[] }) => { }, ]; - const getRowId = (row: ISchema) => { - return row.content_source; - }; - const renderContent = () => { const filteredSchemas = schemas.filter( (schema) => Object.keys(schema.columns).length > 0 @@ -45,7 +51,7 @@ const SchemasTable = ({ schemas }: { schemas: ISchema[] }) => { return ( - No Extractors Found + No Schemas Found ); @@ -59,9 +65,9 @@ const SchemasTable = ({ schemas }: { schemas: ISchema[] }) => { { spacing={2} > - SQL Tables + + SQL Tables + + + + {renderContent()} diff --git a/ui/src/index.tsx b/ui/src/index.tsx index dd75367e1..60af18800 100644 --- a/ui/src/index.tsx +++ b/ui/src/index.tsx @@ -32,7 +32,7 @@ const router = createBrowserRouter( errorElement: , }, { - path: "/:namespace/extraction-policies/:policyname", + path: "/:namespace/extraction-policies/:graphname/:policyname", element: , loader: ExtractionPolicyLoader, errorElement: , diff --git a/ui/src/routes/Namespace/content.tsx b/ui/src/routes/Namespace/content.tsx index 7732d8fe3..458b49733 100644 --- a/ui/src/routes/Namespace/content.tsx +++ b/ui/src/routes/Namespace/content.tsx @@ -40,9 +40,9 @@ export async function loader({ params }: LoaderFunctionArgs) { } return null; }); - const contentMetadata = await client.getContentById(contentId); + const contentMetadata = await client.getContentMetadata(contentId); const extractedMetadataList = await client - .getExtractedMetadata(contentId) + .getStructuredMetadata(contentId) .catch((e) => { if (isAxiosError(e)) { errors.push( @@ -51,7 +51,7 @@ export async function loader({ params }: LoaderFunctionArgs) { }` ); } - return []; + return [] as IExtractedMetadata[]; }); return { client, @@ -216,7 +216,7 @@ const ContentPage = () => { ); })} graph.extraction_policies).flat()} namespace={namespace} tasks={tasks} hideContentId diff --git a/ui/src/routes/Namespace/extractionPolicy.tsx b/ui/src/routes/Namespace/extractionPolicy.tsx index 82804c2cf..6f317a685 100644 --- a/ui/src/routes/Namespace/extractionPolicy.tsx +++ b/ui/src/routes/Namespace/extractionPolicy.tsx @@ -9,6 +9,7 @@ import { getIndexifyServiceURL } from "../../utils/helpers"; export async function loader({ params }: LoaderFunctionArgs) { const namespace = params.namespace; const policyname = params.policyname; + const graphname = params.graphname; if (!namespace || !policyname) return redirect("/"); const client = await IndexifyClient.createClient({ @@ -16,9 +17,10 @@ export async function loader({ params }: LoaderFunctionArgs) { namespace, }); - const policy = client.extractionPolicies.find( - (policy) => policy.name === policyname - ); + const policy = client.extractionGraphs + .map((graph) => graph.extraction_policies) + .flat() + .find((policy) => policy.name === policyname && policy.graph_name === graphname); const tasks = (await client.getTasks()).filter( (task) => task.extraction_policy_id === policy?.id ); @@ -41,6 +43,7 @@ const ExtractionPolicyPage = () => { {namespace} Extraction Policies + {policy.graph_name} {policy.name} @@ -49,7 +52,7 @@ const ExtractionPolicyPage = () => { graph.extraction_policies).flat()} namespace={namespace} tasks={tasks} hideExtractionPolicy diff --git a/ui/src/routes/Namespace/index.tsx b/ui/src/routes/Namespace/index.tsx index 2f30c0c80..d3e6c3235 100644 --- a/ui/src/routes/Namespace/index.tsx +++ b/ui/src/routes/Namespace/index.tsx @@ -4,6 +4,7 @@ import { IContentMetadata, IIndex, ISchema, + ITask, } from "getindexify"; import { useLoaderData, LoaderFunctionArgs } from "react-router-dom"; import { Stack } from "@mui/material"; @@ -21,23 +22,33 @@ export async function loader({ params }: LoaderFunctionArgs) { serviceUrl: getIndexifyServiceURL(), namespace, }); - const [extractors, indexes, contentList, schemas] = await Promise.all([ + const [extractors, indexes, contentList, schemas, tasks] = await Promise.all([ client.extractors(), client.indexes(), - client.getContent(), + client.getExtractedContent(), client.getSchemas(), + client.getTasks(), ]); - return { client, extractors, indexes, contentList, schemas, namespace }; + return { + client, + extractors, + indexes, + contentList, + schemas, + tasks, + namespace, + }; } const NamespacePage = () => { - const { client, extractors, indexes, contentList, schemas, namespace } = + const { client, extractors, indexes, contentList, schemas, tasks, namespace } = useLoaderData() as { client: IndexifyClient; extractors: Extractor[]; indexes: IIndex[]; contentList: IContentMetadata[]; schemas: ISchema[]; + tasks: ITask[]; namespace: string; }; @@ -45,13 +56,22 @@ const NamespacePage = () => { + graph.extraction_policies) + .flat()} /> - graph.extraction_policies) + .flat()} content={contentList} /> diff --git a/ui/src/types.ts b/ui/src/types.ts index 5eeab575a..706adb212 100644 --- a/ui/src/types.ts +++ b/ui/src/types.ts @@ -8,4 +8,5 @@ export interface IExtractionGraphColumns { extractor: IExtractionGraphCol; inputParams: IExtractionGraphCol; mimeTypes: IExtractionGraphCol; + taskCount: IExtractionGraphCol; }