diff --git a/angular.json b/angular.json index 5ef63895..6da9cf42 100644 --- a/angular.json +++ b/angular.json @@ -40,6 +40,11 @@ "input": "./node_modules/flag-icons/flags/1x1/", "output": "./assets/flags/1x1/" }, + { + "glob": "**/*", + "input": "./node_modules/@mediapipe/holistic/", + "output": "./" + }, { "glob": "*.ttf", "input": "./node_modules/@sutton-signwriting/font-ttf/font", diff --git a/src/app/components/video/video.component.ts b/src/app/components/video/video.component.ts index 46f1ffb6..fae75dfc 100644 --- a/src/app/components/video/video.component.ts +++ b/src/app/components/video/video.component.ts @@ -1,7 +1,7 @@ import {AfterViewInit, Component, ElementRef, HostBinding, Input, ViewChild} from '@angular/core'; import {Store} from '@ngxs/store'; import {combineLatest, firstValueFrom} from 'rxjs'; -import {VideoSettings, VideoStateModel} from '../../core/modules/ngxs/store/video/video.state'; +import {VideoStateModel} from '../../core/modules/ngxs/store/video/video.state'; import Stats from 'stats.js'; import {distinctUntilChanged, filter, map, takeUntil, tap} from 'rxjs/operators'; import {BaseComponent} from '../base/base.component'; @@ -112,14 +112,15 @@ export class VideoComponent extends BaseComponent implements AfterViewInit { .pipe( map(state => state.videoSettings), filter(Boolean), - tap(({width, height}) => { + tap(({width, height, aspectRatio}) => { + this.aspectRatio = 'aspect-' + aspectRatio; + this.canvasEl.nativeElement.width = width; this.canvasEl.nativeElement.height = height; // It is required to wait for next frame, as grid element might still be resizing requestAnimationFrame(this.scaleCanvas.bind(this)); }), - tap((settings: VideoSettings) => (this.aspectRatio = 'aspect-' + settings.aspectRatio)), takeUntil(this.ngUnsubscribe) ) .subscribe(); diff --git a/src/app/modules/animation/animation.service.ts b/src/app/modules/animation/animation.service.ts index 7efa1901..be221fbb 100644 --- a/src/app/modules/animation/animation.service.ts +++ b/src/app/modules/animation/animation.service.ts @@ -4,6 +4,7 @@ import {LayersModel} from '@tensorflow/tfjs-layers'; import {Injectable} from '@angular/core'; import {TensorflowService} from '../../core/services/tfjs/tfjs.service'; import {MediapipeHolisticService} from '../../core/services/holistic.service'; +import {POSE_LANDMARKS} from '@mediapipe/holistic'; const ANIMATION_KEYS = [ 'mixamorigHead.quaternion', @@ -80,8 +81,7 @@ export class AnimationService { } normalizePose(pose: Pose): Tensor { - const bodyLandmarks = - pose.poseLandmarks || new Array(Object.keys(this.holistic.POSE_LANDMARKS).length).fill(EMPTY_LANDMARK); + const bodyLandmarks = pose.poseLandmarks || new Array(Object.keys(POSE_LANDMARKS).length).fill(EMPTY_LANDMARK); const leftHandLandmarks = pose.leftHandLandmarks || new Array(21).fill(EMPTY_LANDMARK); const rightHandLandmarks = pose.rightHandLandmarks || new Array(21).fill(EMPTY_LANDMARK); const landmarks = bodyLandmarks.concat(leftHandLandmarks, rightHandLandmarks); @@ -90,8 +90,8 @@ export class AnimationService { .tensor(landmarks.map(l => [l.x, l.y, l.z])) .mul(this.tf.tensor([pose.image.width, pose.image.height, pose.image.width])); - const p1 = tensor.slice(this.holistic.POSE_LANDMARKS.LEFT_SHOULDER, 1); - const p2 = tensor.slice(this.holistic.POSE_LANDMARKS.RIGHT_SHOULDER, 1); + const p1 = tensor.slice(POSE_LANDMARKS.LEFT_SHOULDER, 1); + const p2 = tensor.slice(POSE_LANDMARKS.RIGHT_SHOULDER, 1); const d = this.tf.sqrt(this.tf.pow(p2.sub(p1), 2).sum()); let normTensor = this.tf.sub(tensor, p1.add(p2).div(2)).div(d); diff --git a/src/app/modules/detector/detector.service.ts b/src/app/modules/detector/detector.service.ts index 7ca09248..a329615c 100644 --- a/src/app/modules/detector/detector.service.ts +++ b/src/app/modules/detector/detector.service.ts @@ -2,8 +2,8 @@ import {Tensor} from '@tensorflow/tfjs'; import {EMPTY_LANDMARK, Pose, PoseLandmark} from '../pose/pose.state'; import {LayersModel} from '@tensorflow/tfjs-layers'; import {Injectable} from '@angular/core'; +import {POSE_LANDMARKS} from '@mediapipe/holistic'; import {TensorflowService} from '../../core/services/tfjs/tfjs.service'; -import {MediapipeHolisticService} from '../../core/services/holistic.service'; const WINDOW_SIZE = 20; @@ -19,16 +19,11 @@ export class DetectorService { sequentialModel: LayersModel; - constructor(private tf: TensorflowService, private holistic: MediapipeHolisticService) {} + constructor(private tf: TensorflowService) {} async loadModel() { - return Promise.all([ - this.holistic.load(), - this.tf - .load() - .then(() => this.tf.loadLayersModel('assets/models/sign-detector/model.json')) - .then(model => (this.sequentialModel = model as unknown as LayersModel)), - ]); + await this.tf.load(); + this.sequentialModel = await this.tf.loadLayersModel('assets/models/sign-detector/model.json'); } distance(p1: PoseLandmark, p2: PoseLandmark): number { @@ -38,16 +33,15 @@ export class DetectorService { } normalizePose(pose: Pose): PoseLandmark[] { - const bodyLandmarks = - pose.poseLandmarks || new Array(Object.keys(this.holistic.POSE_LANDMARKS).length).fill(EMPTY_LANDMARK); + const bodyLandmarks = pose.poseLandmarks || new Array(Object.keys(POSE_LANDMARKS).length).fill(EMPTY_LANDMARK); const leftHandLandmarks = pose.leftHandLandmarks || new Array(21).fill(EMPTY_LANDMARK); const rightHandLandmarks = pose.leftHandLandmarks || new Array(21).fill(EMPTY_LANDMARK); const landmarks = bodyLandmarks .concat(leftHandLandmarks, rightHandLandmarks) .map(l => (this.isValidLandmark(l) ? l : EMPTY_LANDMARK)); - const p1 = landmarks[this.holistic.POSE_LANDMARKS.LEFT_SHOULDER]; - const p2 = landmarks[this.holistic.POSE_LANDMARKS.RIGHT_SHOULDER]; + const p1 = landmarks[POSE_LANDMARKS.LEFT_SHOULDER]; + const p2 = landmarks[POSE_LANDMARKS.RIGHT_SHOULDER]; if (p1.x > 0 && p2.x > 0) { this.shoulderWidth[this.shoulderWidthIndex % WINDOW_SIZE] = this.distance(p1, p2); @@ -69,25 +63,18 @@ export class DetectorService { // TODO remove, this is to be compliant with openpose const neck = { - x: - (newPose[this.holistic.POSE_LANDMARKS.LEFT_SHOULDER].x + - newPose[this.holistic.POSE_LANDMARKS.RIGHT_SHOULDER].x) / - 2, - y: - (newPose[this.holistic.POSE_LANDMARKS.LEFT_SHOULDER].y + - newPose[this.holistic.POSE_LANDMARKS.RIGHT_SHOULDER].y) / - 2, + x: (newPose[POSE_LANDMARKS.LEFT_SHOULDER].x + newPose[POSE_LANDMARKS.RIGHT_SHOULDER].x) / 2, + y: (newPose[POSE_LANDMARKS.LEFT_SHOULDER].y + newPose[POSE_LANDMARKS.RIGHT_SHOULDER].y) / 2, }; - return [ - newPose[this.holistic.POSE_LANDMARKS.NOSE], + newPose[POSE_LANDMARKS.NOSE], neck, - newPose[this.holistic.POSE_LANDMARKS.RIGHT_SHOULDER], - newPose[this.holistic.POSE_LANDMARKS.RIGHT_ELBOW], - newPose[this.holistic.POSE_LANDMARKS.RIGHT_WRIST], - newPose[this.holistic.POSE_LANDMARKS.LEFT_SHOULDER], - newPose[this.holistic.POSE_LANDMARKS.LEFT_ELBOW], - newPose[this.holistic.POSE_LANDMARKS.LEFT_WRIST], + newPose[POSE_LANDMARKS.RIGHT_SHOULDER], + newPose[POSE_LANDMARKS.RIGHT_ELBOW], + newPose[POSE_LANDMARKS.RIGHT_WRIST], + newPose[POSE_LANDMARKS.LEFT_SHOULDER], + newPose[POSE_LANDMARKS.LEFT_ELBOW], + newPose[POSE_LANDMARKS.LEFT_WRIST], EMPTY_LANDMARK, EMPTY_LANDMARK, EMPTY_LANDMARK, @@ -95,10 +82,10 @@ export class DetectorService { EMPTY_LANDMARK, EMPTY_LANDMARK, EMPTY_LANDMARK, - newPose[this.holistic.POSE_LANDMARKS.RIGHT_EYE], - newPose[this.holistic.POSE_LANDMARKS.LEFT_EYE], - newPose[this.holistic.POSE_LANDMARKS.RIGHT_EAR], - newPose[this.holistic.POSE_LANDMARKS.LEFT_EAR], + newPose[POSE_LANDMARKS.RIGHT_EYE], + newPose[POSE_LANDMARKS.LEFT_EYE], + newPose[POSE_LANDMARKS.RIGHT_EAR], + newPose[POSE_LANDMARKS.LEFT_EAR], EMPTY_LANDMARK, EMPTY_LANDMARK, EMPTY_LANDMARK, diff --git a/src/app/modules/pose/pose.actions.ts b/src/app/modules/pose/pose.actions.ts index 43d2de9a..2bb57229 100644 --- a/src/app/modules/pose/pose.actions.ts +++ b/src/app/modules/pose/pose.actions.ts @@ -1,5 +1,9 @@ import {Pose} from './pose.state'; +export class LoadPoseModel { + static readonly type = '[Pose] Load Pose Model'; +} + export class PoseVideoFrame { static readonly type = '[Pose] Pose Video Frame'; diff --git a/src/app/modules/pose/pose.service.ts b/src/app/modules/pose/pose.service.ts index a1a331fd..f14db8bf 100644 --- a/src/app/modules/pose/pose.service.ts +++ b/src/app/modules/pose/pose.service.ts @@ -1,8 +1,21 @@ import {Injectable} from '@angular/core'; +import { + FACEMESH_FACE_OVAL, + FACEMESH_LEFT_EYE, + FACEMESH_LEFT_EYEBROW, + FACEMESH_LIPS, + FACEMESH_RIGHT_EYE, + FACEMESH_RIGHT_EYEBROW, + FACEMESH_TESSELATION, + HAND_CONNECTIONS, + POSE_CONNECTIONS, + POSE_LANDMARKS, +} from '@mediapipe/holistic'; import * as drawing from '@mediapipe/drawing_utils/drawing_utils.js'; import {Pose, PoseLandmark} from './pose.state'; import {GoogleAnalyticsService} from '../../core/modules/google-analytics/google-analytics.service'; -import {MediapipeHolisticService} from '../../core/services/holistic.service'; +import * as comlink from 'comlink'; +import {transferableImage} from '../../core/helpers/image/transferable'; const IGNORED_BODY_LANDMARKS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18, 19, 20, 21, 22]; @@ -10,46 +23,50 @@ const IGNORED_BODY_LANDMARKS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18 providedIn: 'root', }) export class PoseService { - model?: any; isFirstFrame = true; - onResultsCallbacks = []; - constructor(private ga: GoogleAnalyticsService, private holistic: MediapipeHolisticService) {} + worker: comlink.Remote<{ + loadModel: () => Promise; + pose: (imageBitmap: ImageBitmap | ImageData) => Promise; + }>; - onResults(onResultsCallback) { - this.onResultsCallbacks.push(onResultsCallback); - } + constructor(private ga: GoogleAnalyticsService) {} async load(): Promise { - if (this.model) { + if (this.worker) { return; } - await this.holistic.load(); - - await this.ga.trace('pose', 'load', () => { - this.model = new this.holistic.Holistic({locateFile: file => `assets/models/holistic/${file}`}); - - this.model.setOptions({ - upperBodyOnly: false, - modelComplexity: 1, - }); - - this.model.onResults(results => { - for (const callback of this.onResultsCallbacks) { - callback(results); - } - }); + await this.ga.trace('pose', 'load', async () => { + this.worker = comlink.wrap(new Worker(new URL('./pose.worker', import.meta.url), {type: 'module'})); + await this.worker.loadModel(); }); } - async predict(video: HTMLVideoElement | HTMLImageElement): Promise { - await this.load(); + async predict(video: HTMLVideoElement | HTMLImageElement): Promise { + const width = (video as HTMLVideoElement).videoWidth ?? video.width; + if (!this.worker || width === 0) { + return null; + } const frameType = this.isFirstFrame ? 'first-frame' : 'frame'; - await this.ga.trace('pose', frameType, () => { + const image = await transferableImage(video); + + return this.ga.trace('pose', frameType, async () => { this.isFirstFrame = false; - return this.model.send({image: video}); + const result: Pose = await this.worker.pose(image); + if (!result) { + return null; + } + + // TODO not sure if this is needed + // const newImage = document.createElement('canvas'); + // newImage.width = image.width; + // newImage.height = image.height; + // const ctx = newImage.getContext('2d'); + // ctx.drawImage(image as any, 0, 0); + // result.image = newImage; + return result; }); } @@ -59,7 +76,7 @@ export class PoseService { delete filteredLandmarks[l]; } - drawing.drawConnectors(ctx, filteredLandmarks, this.holistic.POSE_CONNECTIONS, {color: '#00FF00'}); + drawing.drawConnectors(ctx, filteredLandmarks, POSE_CONNECTIONS, {color: '#00FF00'}); drawing.drawLandmarks(ctx, filteredLandmarks, {color: '#00FF00', fillColor: '#FF0000'}); } @@ -70,7 +87,7 @@ export class PoseService { dotColor: string, dotFillColor: string ): void { - drawing.drawConnectors(ctx, landmarks, this.holistic.HAND_CONNECTIONS, {color: lineColor}); + drawing.drawConnectors(ctx, landmarks, HAND_CONNECTIONS, {color: lineColor}); drawing.drawLandmarks(ctx, landmarks, { color: dotColor, fillColor: dotFillColor, @@ -82,13 +99,13 @@ export class PoseService { } drawFace(landmarks: PoseLandmark[], ctx: CanvasRenderingContext2D): void { - drawing.drawConnectors(ctx, landmarks, this.holistic.FACEMESH_TESSELATION, {color: '#C0C0C070', lineWidth: 1}); - drawing.drawConnectors(ctx, landmarks, this.holistic.FACEMESH_RIGHT_EYE, {color: '#FF3030'}); - drawing.drawConnectors(ctx, landmarks, this.holistic.FACEMESH_RIGHT_EYEBROW, {color: '#FF3030'}); - drawing.drawConnectors(ctx, landmarks, this.holistic.FACEMESH_LEFT_EYE, {color: '#30FF30'}); - drawing.drawConnectors(ctx, landmarks, this.holistic.FACEMESH_LEFT_EYEBROW, {color: '#30FF30'}); - drawing.drawConnectors(ctx, landmarks, this.holistic.FACEMESH_FACE_OVAL, {color: '#E0E0E0'}); - drawing.drawConnectors(ctx, landmarks, this.holistic.FACEMESH_LIPS, {color: '#E0E0E0'}); + drawing.drawConnectors(ctx, landmarks, FACEMESH_TESSELATION, {color: '#C0C0C070', lineWidth: 1}); + drawing.drawConnectors(ctx, landmarks, FACEMESH_RIGHT_EYE, {color: '#FF3030'}); + drawing.drawConnectors(ctx, landmarks, FACEMESH_RIGHT_EYEBROW, {color: '#FF3030'}); + drawing.drawConnectors(ctx, landmarks, FACEMESH_LEFT_EYE, {color: '#30FF30'}); + drawing.drawConnectors(ctx, landmarks, FACEMESH_LEFT_EYEBROW, {color: '#30FF30'}); + drawing.drawConnectors(ctx, landmarks, FACEMESH_FACE_OVAL, {color: '#E0E0E0'}); + drawing.drawConnectors(ctx, landmarks, FACEMESH_LIPS, {color: '#E0E0E0'}); } drawConnect(connectors: PoseLandmark[][], ctx: CanvasRenderingContext2D): void { @@ -112,15 +129,12 @@ export class PoseService { if (pose.rightHandLandmarks) { ctx.strokeStyle = '#00FF00'; - this.drawConnect( - [[pose.poseLandmarks[this.holistic.POSE_LANDMARKS.RIGHT_ELBOW], pose.rightHandLandmarks[0]]], - ctx - ); + this.drawConnect([[pose.poseLandmarks[POSE_LANDMARKS.RIGHT_ELBOW], pose.rightHandLandmarks[0]]], ctx); } if (pose.leftHandLandmarks) { ctx.strokeStyle = '#FF0000'; - this.drawConnect([[pose.poseLandmarks[this.holistic.POSE_LANDMARKS.LEFT_ELBOW], pose.leftHandLandmarks[0]]], ctx); + this.drawConnect([[pose.poseLandmarks[POSE_LANDMARKS.LEFT_ELBOW], pose.leftHandLandmarks[0]]], ctx); } } diff --git a/src/app/modules/pose/pose.state.ts b/src/app/modules/pose/pose.state.ts index 72346b79..446a3bed 100644 --- a/src/app/modules/pose/pose.state.ts +++ b/src/app/modules/pose/pose.state.ts @@ -1,7 +1,8 @@ import {Injectable} from '@angular/core'; import {Action, NgxsOnInit, State, StateContext, Store} from '@ngxs/store'; import {PoseService} from './pose.service'; -import {PoseVideoFrame, StoreFramePose} from './pose.actions'; +import {LoadPoseModel, PoseVideoFrame, StoreFramePose} from './pose.actions'; +import {filter, first, tap} from 'rxjs/operators'; export interface PoseLandmark { x: number; @@ -36,34 +37,51 @@ const initialState: PoseStateModel = { defaults: initialState, }) export class PoseState implements NgxsOnInit { + poseSetting$ = this.store.select(state => state.settings.pose); + constructor(private poseService: PoseService, private store: Store) {} - ngxsOnInit(): void { - this.poseService.onResults(results => { - // TODO: passing the `image` canvas through NGXS bugs the pose. - // https://github.com/google/mediapipe/issues/2422 - const fakeImage = document.createElement('canvas'); - fakeImage.width = results.image.width; - fakeImage.height = results.image.height; - const ctx = fakeImage.getContext('2d'); - ctx.drawImage(results.image, 0, 0, fakeImage.width, fakeImage.height); + ngxsOnInit({dispatch}: StateContext): void { + this.poseSetting$ + .pipe( + filter(Boolean), + first(), + tap(() => dispatch(LoadPoseModel)) + ) + .subscribe(); + } - // Since v0.4, "results" include additional parameters - this.store.dispatch( - new StoreFramePose({ - faceLandmarks: results.faceLandmarks, - poseLandmarks: results.poseLandmarks, - leftHandLandmarks: results.leftHandLandmarks, - rightHandLandmarks: results.rightHandLandmarks, - image: fakeImage, - }) - ); - }); + @Action(LoadPoseModel) + async load({patchState}: StateContext): Promise { + patchState({isLoaded: false}); + await this.poseService.load(); + // isLoaded is set to true once the first frame is processed. } @Action(PoseVideoFrame) async poseFrame({patchState, dispatch}: StateContext, {video}: PoseVideoFrame): Promise { - await this.poseService.predict(video); + const result = await this.poseService.predict(video); + // TODO: passing the `image` canvas through NGXS bugs the pose. + // https://github.com/google/mediapipe/issues/2422 + // const fakeImage = document.createElement('canvas'); + // fakeImage.width = results.image.width; + // fakeImage.height = results.image.height; + // const ctx = fakeImage.getContext('2d'); + // ctx.drawImage(results.image, 0, 0, fakeImage.width, fakeImage.height); + // + // // Since v0.4, "results" include additional parameters + // this.store.dispatch( + // new StoreFramePose({ + // faceLandmarks: results.faceLandmarks, + // poseLandmarks: results.poseLandmarks, + // leftHandLandmarks: results.leftHandLandmarks, + // rightHandLandmarks: results.rightHandLandmarks, + // image: fakeImage, + // }) + // ); + + // Since v0.4, "results" include additional parameters + dispatch(new StoreFramePose(result)); } @Action(StoreFramePose) diff --git a/src/app/modules/pose/pose.worker.ts b/src/app/modules/pose/pose.worker.ts new file mode 100644 index 00000000..7f462e80 --- /dev/null +++ b/src/app/modules/pose/pose.worker.ts @@ -0,0 +1,82 @@ +/// + +import * as comlink from 'comlink'; +import {Holistic} from '@mediapipe/holistic'; +import {firstValueFrom, Observable} from 'rxjs'; + +// Fake document to satisfy `"ontouchend" in document` +globalThis.document = {} as any; + +const POSE_CONFIG = { + // angular.json copies `@mediapipe/holistic` to `assets/mediapipe/holistic` + locateFile: file => new URL(`/assets/models/holistic/${file}`, globalThis.location.origin).toString(), +}; + +// Solution taken from https://github.com/google/mediapipe/issues/2506#issuecomment-1386616165 +(self as any).createMediapipeSolutionsWasm = POSE_CONFIG; +(self as any).createMediapipeSolutionsPackedAssets = POSE_CONFIG; + +importScripts( + '/assets/models/holistic/holistic_solution_packed_assets_loader.js', + '/assets/models/holistic/holistic_solution_simd_wasm_bin.js' +); + +// let model: Holistic; +let model: Holistic; +let results: Observable; + +async function loadModel(): Promise { + model = new Holistic(POSE_CONFIG); + model.setOptions({ + // TODO use our preferred settings: + // modelComplexity: 1, + // smoothLandmarks: false + + selfieMode: false, + modelComplexity: 2, + smoothLandmarks: false, + }); + + const solution = (model as any).g; + const solutionConfig = solution.g; + solutionConfig.files = () => []; // disable default import files behavior + await model.initialize(); + solution.D = solution.h.GL.currentContext.GLctx; // set gl ctx + + // load data files + const files = solution.F; + files['pose_landmark_heavy.tflite'] = ( + await fetch(POSE_CONFIG.locateFile('pose_landmark_heavy.tflite')) + ).arrayBuffer(); + files['holistic.binarypb'] = (await fetch(POSE_CONFIG.locateFile('holistic.binarypb'))).arrayBuffer(); + // + // // To be on the safe side, we load the files in the order they are listed in the manifest. TODO: remove this + // for (const file of ['pose_landmark_lite.tflite', 'pose_landmark_full.tflite', 'pose_landmark_heavy.tflite']) { + // files[file] = fetch(POSE_CONFIG.locateFile(file)).then(res => res.arrayBuffer()); + // } + + results = new Observable(subscriber => { + model.onResults(results => { + console.log(results); // Currently prints: {image: ImageBitmap, multiFaceGeometry: Array(0)} + subscriber.next({ + faceLandmarks: results.faceLandmarks, + poseLandmarks: results.poseLandmarks, + leftHandLandmarks: results.leftHandLandmarks, + rightHandLandmarks: results.rightHandLandmarks, + image: results.image, // TODO reconsider if sending this is expensive + }); + }); + }); +} + +async function pose(imageBitmap: ImageBitmap | ImageData): Promise { + if (!results) { + return null; + } + + const result = firstValueFrom(results); + await model.send({image: imageBitmap as any}); + return result; +} + +comlink.expose({loadModel, pose}); diff --git a/src/app/modules/settings/settings.state.ts b/src/app/modules/settings/settings.state.ts index 1c468cd1..a388f1ac 100644 --- a/src/app/modules/settings/settings.state.ts +++ b/src/app/modules/settings/settings.state.ts @@ -12,6 +12,7 @@ export interface SettingsStateModel { animatePose: boolean; drawVideo: boolean; + pose: boolean; drawPose: boolean; drawSignWriting: boolean; @@ -28,6 +29,7 @@ const initialState: SettingsStateModel = { animatePose: false, drawVideo: true, + pose: false, drawPose: true, drawSignWriting: false, diff --git a/src/app/modules/settings/settings/settings.component.ts b/src/app/modules/settings/settings/settings.component.ts index 15192a4a..da2f1fd8 100644 --- a/src/app/modules/settings/settings/settings.component.ts +++ b/src/app/modules/settings/settings/settings.component.ts @@ -13,6 +13,7 @@ export class SettingsComponent extends BaseSettingsComponent implements OnInit { availableSettings: Array = [ 'detectSign', 'drawVideo', + 'pose', 'drawPose', 'drawSignWriting', 'animatePose', diff --git a/src/app/modules/sign-writing/body.service.ts b/src/app/modules/sign-writing/body.service.ts index ae0a5b8b..40164322 100644 --- a/src/app/modules/sign-writing/body.service.ts +++ b/src/app/modules/sign-writing/body.service.ts @@ -1,9 +1,9 @@ import {Injectable} from '@angular/core'; import {SignWritingService} from './sign-writing.service'; import {PoseLandmark} from '../pose/pose.state'; +import {POSE_LANDMARKS} from '@mediapipe/holistic'; import {ThreeService} from '../../core/services/three.service'; import {Vector2} from 'three'; -import {MediapipeHolisticService} from '../../core/services/holistic.service'; export interface BodyShoulders { center: Vector2; @@ -20,11 +20,11 @@ export interface BodyStateModel { providedIn: 'root', }) export class BodyService { - constructor(private three: ThreeService, private holistic: MediapipeHolisticService) {} + constructor(private three: ThreeService) {} shoulders(landmarks: PoseLandmark[]): BodyShoulders { - const p1 = landmarks[this.holistic.POSE_LANDMARKS.LEFT_SHOULDER]; - const p2 = landmarks[this.holistic.POSE_LANDMARKS.RIGHT_SHOULDER]; + const p1 = landmarks[POSE_LANDMARKS.LEFT_SHOULDER]; + const p2 = landmarks[POSE_LANDMARKS.RIGHT_SHOULDER]; return { center: new this.three.Vector2((p1.x + p2.x) / 2, (p1.y + p2.y) / 2), diff --git a/src/app/modules/sign-writing/sign-writing.state.ts b/src/app/modules/sign-writing/sign-writing.state.ts index 22cc670c..2ac46971 100644 --- a/src/app/modules/sign-writing/sign-writing.state.ts +++ b/src/app/modules/sign-writing/sign-writing.state.ts @@ -5,6 +5,7 @@ import {filter, first, tap} from 'rxjs/operators'; import {HandsService, HandStateModel} from './hands.service'; import {CalculateBodyFactors, EstimateFaceShape, EstimateHandShape} from './sign-writing.actions'; import {BodyService, BodyStateModel} from './body.service'; +import {POSE_LANDMARKS} from '@mediapipe/holistic'; import {FaceService, FaceStateModel} from './face.service'; import {ThreeService} from '../../core/services/three.service'; import {MediapipeHolisticService} from '../../core/services/holistic.service'; @@ -102,14 +103,8 @@ export class SignWritingState implements NgxsOnInit { patchState({ body: { shoulders: this.bodyService.shoulders(pose.poseLandmarks), - elbows: [ - pose.poseLandmarks[this.holistic.POSE_LANDMARKS.LEFT_ELBOW], - pose.poseLandmarks[this.holistic.POSE_LANDMARKS.RIGHT_ELBOW], - ], - wrists: [ - pose.poseLandmarks[this.holistic.POSE_LANDMARKS.LEFT_WRIST], - pose.poseLandmarks[this.holistic.POSE_LANDMARKS.RIGHT_WRIST], - ], + elbows: [pose.poseLandmarks[POSE_LANDMARKS.LEFT_ELBOW], pose.poseLandmarks[POSE_LANDMARKS.RIGHT_ELBOW]], + wrists: [pose.poseLandmarks[POSE_LANDMARKS.LEFT_WRIST], pose.poseLandmarks[POSE_LANDMARKS.RIGHT_WRIST]], }, }); } diff --git a/src/app/modules/translate/translate.service.ts b/src/app/modules/translate/translate.service.ts index 2a00f3ff..919dd485 100644 --- a/src/app/modules/translate/translate.service.ts +++ b/src/app/modules/translate/translate.service.ts @@ -190,7 +190,8 @@ export class TranslationService { } translateSpokenToSigned(text: string, spokenLanguage: string, signedLanguage: string): string { - const api = 'https://spoken-to-signed-sxie2r74ua-uc.a.run.app/'; - return `${api}?slang=${spokenLanguage}&dlang=${signedLanguage}&sentence=${encodeURIComponent(text)}`; + // const api = 'https://spoken-to-signed-sxie2r74ua-uc.a.run.app/'; + // return `${api}?slang=${spokenLanguage}&dlang=${signedLanguage}&sentence=${encodeURIComponent(text)}`; + return 'assets/tmp/example.pose'; } } diff --git a/src/app/pages/landing/about/about-offline/about-offline.component.html b/src/app/pages/landing/about/about-offline/about-offline.component.html index 766597cf..76cda7dd 100644 --- a/src/app/pages/landing/about/about-offline/about-offline.component.html +++ b/src/app/pages/landing/about/about-offline/about-offline.component.html @@ -1,5 +1,8 @@
-

airplanemode_active Offline Support

+

+ airplanemode_active + Offline Support +

Sign Translate works ; diff --git a/src/app/pages/playground/playground.component.ts b/src/app/pages/playground/playground.component.ts index 880a17bf..21bccf7d 100644 --- a/src/app/pages/playground/playground.component.ts +++ b/src/app/pages/playground/playground.component.ts @@ -4,6 +4,7 @@ import {BaseComponent} from '../../components/base/base.component'; import {filter, takeUntil, tap} from 'rxjs/operators'; import {SetVideo, StartCamera} from '../../core/modules/ngxs/store/video/video.actions'; import {TranslocoService} from '@ngneat/transloco'; +import {SetSetting} from '../../modules/settings/settings.actions'; @Component({ selector: 'app-playground', @@ -33,6 +34,7 @@ export class PlaygroundComponent extends BaseComponent implements OnInit { ) .subscribe(); + this.store.dispatch(new SetSetting('pose', true)); this.store.dispatch(new SetVideo('assets/tmp/example-sentence.mp4')); } } diff --git a/src/app/pages/settings/settings-offline/settings-offline.component.html b/src/app/pages/settings/settings-offline/settings-offline.component.html index e0d7bf1e..6a01f2bc 100644 --- a/src/app/pages/settings/settings-offline/settings-offline.component.html +++ b/src/app/pages/settings/settings-offline/settings-offline.component.html @@ -51,7 +51,7 @@ mat-icon-button matTreeNodeToggle [disabled]="node.children.length === 0" - [attr.aria-label]="'settings.other.offline.actions.toggle' | transloco : {label: node.label}"> + [attr.aria-label]="'settings.other.offline.actions.toggle' | transloco: {label: node.label}"> {{ treeControl.isExpanded(node) ? 'expand_more' : 'chevron_right' }} diff --git a/src/app/pages/translate/translate.component.ts b/src/app/pages/translate/translate.component.ts index 6ed06d3c..0c27c3b3 100644 --- a/src/app/pages/translate/translate.component.ts +++ b/src/app/pages/translate/translate.component.ts @@ -42,6 +42,7 @@ export class TranslateComponent extends BaseComponent implements OnInit { new SetSetting('receiveVideo', true), new SetSetting('detectSign', false), new SetSetting('drawSignWriting', false), // This setting currently also controls loading the SignWriting models. + new SetSetting('pose', false), new SetSetting('drawPose', true), new SetSetting('poseViewer', 'pose'), ]); @@ -69,7 +70,7 @@ export class TranslateComponent extends BaseComponent implements OnInit { tap(spokenToSigned => { this.spokenToSigned = spokenToSigned; if (!this.spokenToSigned) { - this.store.dispatch(new SetSetting('drawSignWriting', true)); + this.store.dispatch([new SetSetting('pose', true), new SetSetting('drawSignWriting', true)]); } }), takeUntil(this.ngUnsubscribe) diff --git a/src/assets/promotional/stores/google-play-badge.svg b/src/assets/promotional/stores/google-play-badge.svg index a01a8d87..9e2c5d7c 100644 --- a/src/assets/promotional/stores/google-play-badge.svg +++ b/src/assets/promotional/stores/google-play-badge.svg @@ -1,44 +1,100 @@ - - - - - - - + + + + + + + + ]> - + - - + + - + - - + - - + - - - - - - - + " /> + + + + + + + - - - - - - + + + + + + - - - - + + + + - - - - - - - + + + + + + + - - - - + + + + diff --git a/src/assets/tmp/example.pose b/src/assets/tmp/example.pose new file mode 100644 index 00000000..17968448 Binary files /dev/null and b/src/assets/tmp/example.pose differ