From 358703fa0d82c4ea492ba6abf9b12eee780c27a9 Mon Sep 17 00:00:00 2001 From: Tommaso Turchi Date: Fri, 27 Jan 2023 08:07:21 +0100 Subject: [PATCH] huge refactoring, performance improvements --- README.md | 3 +- examples/face-tracking/package.json | 6 +- examples/image-tracking/package.json | 6 +- package.json | 4 +- src/AR.js | 202 +++++++++++++++------------ 5 files changed, 119 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index b4a674c..7b4b99d 100644 --- a/README.md +++ b/README.md @@ -110,4 +110,5 @@ A Mesh Object representing a tracked face (see the original [MindAR example](htt ## 📮 TODO - [x] Add Showcase Video -- [ ] Fix CI Build +- [x] Fix Performance Issues +- [ ] Fix CI Build (Update to mind-ar 1.2) diff --git a/examples/face-tracking/package.json b/examples/face-tracking/package.json index 6daf5db..0ab1921 100644 --- a/examples/face-tracking/package.json +++ b/examples/face-tracking/package.json @@ -3,9 +3,9 @@ "version": "0.0.1", "private": true, "dependencies": { - "react-three-mind": "^0.1.7", - "react": "^18.2.0", - "react-dom": "^18.2.0", + "react-three-mind": "^0.1.8", + "react": "^18.0.0", + "react-dom": "^18.0.0", "react-scripts": "5.0.1" }, "scripts": { diff --git a/examples/image-tracking/package.json b/examples/image-tracking/package.json index 98632cd..02fef1c 100644 --- a/examples/image-tracking/package.json +++ b/examples/image-tracking/package.json @@ -3,9 +3,9 @@ "version": "0.0.1", "private": true, "dependencies": { - "react-three-mind": "^0.1.7", - "react": "^18.2.0", - "react-dom": "^18.2.0", + "react-three-mind": "^0.1.8", + "react": "^18.0.0", + "react-dom": "^18.0.0", "react-scripts": "5.0.1" }, "scripts": { diff --git a/package.json b/package.json index 3df64df..bc19ecf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-three-mind", - "version": "0.1.8", + "version": "0.1.9", "description": "MindAR components for @react-three/fiber", "main": "dist/index.js", "author": "Tommaso Turchi", @@ -38,7 +38,7 @@ "@react-three/drei": "^9.36.0", "@react-three/fiber": "^8.8.10", "jotai": "^1.8.6", - "mind-ar": "^1.1.5", + "mind-ar": "1.1.5", "react-webcam": "^7.0.1", "three": "^0.145.0" } diff --git a/src/AR.js b/src/AR.js index af90a89..1794b3f 100644 --- a/src/AR.js +++ b/src/AR.js @@ -1,4 +1,4 @@ -import { Canvas, useFrame, useThree } from "@react-three/fiber"; +import { Canvas, useThree } from "@react-three/fiber"; import { faces as FaceMeshFaces, uvs as FaceMeshUVs, @@ -14,17 +14,17 @@ import React, { useRef, useState, } from "react"; -import { atom, useAtom } from "jotai"; +import { atom, useAtomValue, useSetAtom } from "jotai"; import { Controller as FaceTargetController } from "mind-ar/src/face-target/controller"; import { Html } from "@react-three/drei"; import { Controller as ImageTargetController } from "mind-ar/src/image-target/controller"; import Webcam from "react-webcam"; -import { useUpdateAtom } from "jotai/utils"; import { useWindowSize } from "./hooks"; -const anchorsAtom = atom([]); -const faceMeshesAtom = atom([]); +const modeAtom = atom(); +const anchorsAtom = atom({}); +const faceMeshAtom = atom(); const ARProvider = forwardRef( ( @@ -47,13 +47,16 @@ const ARProvider = forwardRef( const [ready, setReady] = useState(false); const controllerRef = useRef(null); const { camera } = useThree(); - const [anchors] = useAtom(anchorsAtom); - const [faceMeshes] = useAtom(faceMeshesAtom); + const setMode = useSetAtom(modeAtom); + const setAnchors = useSetAtom(anchorsAtom); + const setFaceMesh = useSetAtom(faceMeshAtom); const { width, height } = useWindowSize(); useEffect(() => { if (controllerRef.current) { + setMode(Boolean(imageTargets)); + if (imageTargets) { const ARprojectionMatrix = controllerRef.current.getProjectionMatrix(); @@ -64,7 +67,14 @@ const ARProvider = forwardRef( camera.updateProjectionMatrix(); } } - }, [width, height, camera, imageTargets]); + + return () => { + if (controllerRef.current) { + controllerRef.current.stopProcessVideo(); + controllerRef.current = null; + } + }; + }, [width, height, camera, imageTargets, setMode]); const handleStream = useCallback(() => { if (webcamRef.current) { @@ -110,36 +120,18 @@ const ARProvider = forwardRef( camera.far = ARprojectionMatrix[14] / (ARprojectionMatrix[10] + 1.0); camera.updateProjectionMatrix(); - controller.onUpdate = (data) => { - if (data.type === "updateMatrix") { - const { targetIndex, worldMatrix } = data; - - anchors.forEach( - ({ anchor, target, onAnchorFound, onAnchorLost }) => { - if (target === targetIndex) { - if ( - !anchor.visible && - worldMatrix !== null && - onAnchorFound - ) - onAnchorFound(); - else if ( - anchor.visible && - worldMatrix === null && - onAnchorLost - ) - onAnchorLost(); - - anchor.visible = worldMatrix !== null; - - if (worldMatrix !== null) { - anchor.matrix = new Matrix4() + controller.onUpdate = ({ type, targetIndex, worldMatrix }) => { + if (type === "updateMatrix") { + setAnchors((anchors) => ({ + ...anchors, + [targetIndex]: + worldMatrix !== null + ? new Matrix4() .fromArray([...worldMatrix]) - .multiply(postMatrices[targetIndex]); - } - } - } - ); + .multiply(postMatrices[targetIndex]) + .toArray() + : null, + })); } }; } else { @@ -148,40 +140,13 @@ const ARProvider = forwardRef( filterBeta, }); - controller.onUpdate = ({ hasFace, estimateResult }) => { - faceMeshes.forEach(({ anchor, onFaceFound, onFaceLost }) => { - if (!anchor.visible && hasFace && onFaceFound) onFaceFound(); - else if (anchor.visible && !hasFace && onFaceLost) onFaceLost(); - - anchor.visible = hasFace; - }); - - anchors.forEach( - ({ anchor, target, onAnchorFound, onAnchorLost }) => { - if (!anchor.visible && hasFace && onAnchorFound) - onAnchorFound(); - else if (anchor.visible && !hasFace && onAnchorLost) - onAnchorLost(); - - anchor.visible = hasFace; - if (hasFace) - anchor.matrix.set(...controller.getLandmarkMatrix(target)); - } + controller.onUpdate = ({ + hasFace, + estimateResult: { faceMatrix, metricLandmarks, faceScale }, + }) => { + setFaceMesh( + hasFace ? { faceMatrix, metricLandmarks, faceScale } : null ); - - if (hasFace) - faceMeshes.forEach(({ anchor }) => { - anchor.matrix.set(...estimateResult.faceMatrix); - - for (let i = 0; i < FaceMeshUVs.length; i++) - anchor.geometry.attributes.position.set( - estimateResult.metricLandmarks[i], - i * 3 - ); - - anchor.geometry.attributes.position.needsUpdate = true; - anchor.geometry.computeVertexNormals(); - }); }; await controller.setup(webcamRef.current.video); @@ -205,14 +170,15 @@ const ARProvider = forwardRef( }, [ ready, imageTargets, + onReady, maxTrack, filterMinCF, filterBeta, missTolerance, warmupTolerance, camera, - anchors, - faceMeshes, + setAnchors, + setFaceMesh, ]); const stopTracking = useCallback(() => { @@ -221,13 +187,6 @@ const ARProvider = forwardRef( } }, [controllerRef]); - useFrame(() => { - if (controllerRef.current && !controllerRef.current.processingVideo) { - faceMeshes.forEach(({ anchor }) => (anchor.visible = false)); - anchors.forEach(({ anchor }) => (anchor.visible = false)); - } - }); - useImperativeHandle( ref, () => ({ @@ -357,15 +316,59 @@ const ARAnchor = ({ ...rest }) => { const ref = useRef(); - const setAnchors = useUpdateAtom(anchorsAtom); + const anchor = useAtomValue(anchorsAtom); + const mode = useAtomValue(modeAtom); + const faceMesh = useAtomValue(faceMeshAtom); useEffect(() => { - if (ref.current) - setAnchors((anchors) => [ - ...anchors, - { target, anchor: ref.current, onAnchorFound, onAnchorLost }, - ]); - }, [ref, setAnchors, target, onAnchorFound, onAnchorLost]); + if (ref.current) { + if (mode) { + if (anchor[target]) { + if (ref.current.visible !== true && onAnchorFound) onAnchorFound(); + ref.current.visible = true; + ref.current.matrix = new Matrix4().fromArray(anchor[target]); + } else { + if (ref.current.visible !== false && onAnchorLost) onAnchorLost(); + ref.current.visible = false; + } + } else { + if (faceMesh) { + if (ref.current.visible !== true && onAnchorFound) onAnchorFound(); + ref.current.visible = true; + const fm = faceMesh.faceMatrix; + const s = faceMesh.faceScale; + const t = [ + faceMesh.metricLandmarks[target][0], + faceMesh.metricLandmarks[target][1], + faceMesh.metricLandmarks[target][2], + ]; + ref.current.matrix.set( + ...[ + fm[0] * s, + fm[1] * s, + fm[2] * s, + fm[0] * t[0] + fm[1] * t[1] + fm[2] * t[2] + fm[3], + fm[4] * s, + fm[5] * s, + fm[6] * s, + fm[4] * t[0] + fm[5] * t[1] + fm[6] * t[2] + fm[7], + fm[8] * s, + fm[9] * s, + fm[10] * s, + fm[8] * t[0] + fm[9] * t[1] + fm[10] * t[2] + fm[11], + fm[12] * s, + fm[13] * s, + fm[14] * s, + fm[12] * t[0] + fm[13] * t[1] + fm[14] * t[2] + fm[15], + ] + ); + } else { + if (ref.current.visible !== false && onAnchorLost) onAnchorLost(); + ref.current.visible = false; + } + } + } + }, [anchor, target, onAnchorFound, onAnchorLost, mode, faceMesh]); return ( @@ -376,7 +379,7 @@ const ARAnchor = ({ const ARFaceMesh = ({ children, onFaceFound, onFaceLost, ...rest }) => { const ref = useRef(); - const setFaceMeshes = useUpdateAtom(faceMeshesAtom); + const faceMesh = useAtomValue(faceMeshAtom); const [positions, uvs, indexes] = useMemo(() => { const positions = new Float32Array(FaceMeshUVs.length * 3); @@ -391,12 +394,25 @@ const ARFaceMesh = ({ children, onFaceFound, onFaceLost, ...rest }) => { }, []); useEffect(() => { - if (ref.current) - setFaceMeshes((faceMeshes) => [ - ...faceMeshes, - { anchor: ref.current, onFaceFound, onFaceLost }, - ]); - }, [ref, setFaceMeshes, onFaceFound, onFaceLost]); + if (ref.current) { + if (faceMesh) { + if (ref.current.visible !== true && onFaceFound) onFaceFound(); + ref.current.visible = true; + ref.current.matrix.set(...faceMesh.faceMatrix); + for (let i = 0; i < FaceMeshUVs.length; i++) + ref.current.geometry.attributes.position.set( + faceMesh.metricLandmarks[i], + i * 3 + ); + + ref.current.geometry.attributes.position.needsUpdate = true; + ref.current.geometry.computeVertexNormals(); + } else { + if (ref.current.visible !== false && onFaceLost) onFaceLost(); + ref.current.visible = false; + } + } + }, [onFaceFound, onFaceLost, faceMesh]); return (