From 20393cddd8d00a49d752e32ef7c63704bf5ccaa9 Mon Sep 17 00:00:00 2001
From: Amit <amitmoryossef@gmail.com>
Date: Mon, 6 Sep 2021 13:34:54 +0300
Subject: [PATCH 1/8] feat(holistic): move pose estimation to web worker

---
 angular.json                                  |  5 ++
 src/app/components/video/video.component.ts   |  5 +-
 .../modules/animation/animation.service.ts    |  8 +--
 src/app/modules/detector/detector.service.ts  | 34 +++++------
 src/app/modules/pix2pix/pix2pix.service.ts    |  1 -
 src/app/modules/pix2pix/pix2pix.worker.ts     |  8 +--
 src/app/modules/pose/pose.service.ts          | 60 ++++++++++++-------
 src/app/modules/pose/pose.state.ts            | 39 +++++-------
 src/app/modules/pose/pose.worker.ts           | 44 ++++++++++++++
 src/app/modules/settings/settings.state.ts    |  2 +
 src/app/modules/sign-writing/body.service.ts  |  2 +-
 .../sign-writing/sign-writing.state.ts        |  6 +-
 .../pages/translate/translate.component.ts    |  6 +-
 13 files changed, 140 insertions(+), 80 deletions(-)
 create mode 100644 src/app/modules/pose/pose.worker.ts

diff --git a/angular.json b/angular.json
index bdce9b79..6ff9ecb3 100644
--- a/angular.json
+++ b/angular.json
@@ -25,6 +25,11 @@
                 "input": "./node_modules/@mediapipe/holistic/",
                 "output": "./assets/models/holistic/"
               },
+              {
+                "glob": "**/*",
+                "input": "./node_modules/@mediapipe/holistic/",
+                "output": "./"
+              },
               {
                 "glob": "*.ttf",
                 "input": "./node_modules/@sutton-signwriting/font-ttf/font",
diff --git a/src/app/components/video/video.component.ts b/src/app/components/video/video.component.ts
index 316e6f97..a6ac72ed 100644
--- a/src/app/components/video/video.component.ts
+++ b/src/app/components/video/video.component.ts
@@ -106,14 +106,15 @@ export class VideoComponent extends BaseComponent implements AfterViewInit {
     this.videoState$.pipe(
       map(state => state.videoSettings),
       filter(Boolean),
-      tap(({width, height}) => {
+      tap(({width, height, aspectRatio}) => {
+        this.aspectRatio = 'aspect-' + aspectRatio;
+
         this.canvasEl.nativeElement.width = width;
         this.canvasEl.nativeElement.height = height;
 
         // It is required to wait for next frame, as grid element might still be resizing
         requestAnimationFrame(this.scaleCanvas.bind(this));
       }),
-      tap((settings: VideoSettings) => this.aspectRatio = 'aspect-' + settings.aspectRatio),
       takeUntil(this.ngUnsubscribe)
     ).subscribe();
   }
diff --git a/src/app/modules/animation/animation.service.ts b/src/app/modules/animation/animation.service.ts
index 31a18ad7..573d26be 100644
--- a/src/app/modules/animation/animation.service.ts
+++ b/src/app/modules/animation/animation.service.ts
@@ -2,7 +2,7 @@ import {Tensor} from '@tensorflow/tfjs';
 import {EMPTY_LANDMARK, Pose} from '../pose/pose.state';
 import {LayersModel} from '@tensorflow/tfjs-layers';
 import {Injectable} from '@angular/core';
-import * as holistic from '@mediapipe/holistic/holistic.js';
+import {POSE_LANDMARKS} from '@mediapipe/holistic';
 import {TensorflowService} from '../../core/services/tfjs.service';
 
 const ANIMATION_KEYS = [
@@ -57,7 +57,7 @@ export class AnimationService {
   }
 
   normalizePose(pose: Pose): Tensor {
-    const bodyLandmarks = pose.poseLandmarks || new Array(Object.keys(holistic.POSE_LANDMARKS).length).fill(EMPTY_LANDMARK);
+    const bodyLandmarks = pose.poseLandmarks || new Array(Object.keys(POSE_LANDMARKS).length).fill(EMPTY_LANDMARK);
     const leftHandLandmarks = pose.leftHandLandmarks || new Array(21).fill(EMPTY_LANDMARK);
     const rightHandLandmarks = pose.rightHandLandmarks || new Array(21).fill(EMPTY_LANDMARK);
     const landmarks = bodyLandmarks.concat(leftHandLandmarks, rightHandLandmarks);
@@ -65,8 +65,8 @@ export class AnimationService {
     const tensor = this.tf.tensor(landmarks.map(l => [l.x, l.y, l.z]))
       .mul(this.tf.tensor([pose.image.width, pose.image.height, pose.image.width]));
 
-    const p1 = tensor.slice(holistic.POSE_LANDMARKS.LEFT_SHOULDER, 1);
-    const p2 = tensor.slice(holistic.POSE_LANDMARKS.RIGHT_SHOULDER, 1);
+    const p1 = tensor.slice(POSE_LANDMARKS.LEFT_SHOULDER, 1);
+    const p2 = tensor.slice(POSE_LANDMARKS.RIGHT_SHOULDER, 1);
 
     const d = this.tf.sqrt(this.tf.pow(p2.sub(p1), 2).sum());
     let normTensor = this.tf.sub(tensor, p1.add(p2).div(2)).div(d);
diff --git a/src/app/modules/detector/detector.service.ts b/src/app/modules/detector/detector.service.ts
index 65f1bdc4..aad9484b 100644
--- a/src/app/modules/detector/detector.service.ts
+++ b/src/app/modules/detector/detector.service.ts
@@ -2,7 +2,7 @@ import {Tensor} from '@tensorflow/tfjs';
 import {EMPTY_LANDMARK, Pose, PoseLandmark} from '../pose/pose.state';
 import {LayersModel} from '@tensorflow/tfjs-layers';
 import {Injectable} from '@angular/core';
-import * as holistic from '@mediapipe/holistic/holistic.js';
+import {POSE_LANDMARKS} from '@mediapipe/holistic';
 import {TensorflowService} from '../../core/services/tfjs.service';
 
 const WINDOW_SIZE = 20;
@@ -35,13 +35,13 @@ export class DetectorService {
   }
 
   normalizePose(pose: Pose): PoseLandmark[] {
-    const bodyLandmarks = pose.poseLandmarks || new Array(Object.keys(holistic.POSE_LANDMARKS).length).fill(EMPTY_LANDMARK);
+    const bodyLandmarks = pose.poseLandmarks || new Array(Object.keys(POSE_LANDMARKS).length).fill(EMPTY_LANDMARK);
     const leftHandLandmarks = pose.leftHandLandmarks || new Array(21).fill(EMPTY_LANDMARK);
     const rightHandLandmarks = pose.leftHandLandmarks || new Array(21).fill(EMPTY_LANDMARK);
     const landmarks = bodyLandmarks.concat(leftHandLandmarks, rightHandLandmarks).map(l => this.isValidLandmark(l) ? l : EMPTY_LANDMARK);
 
-    const p1 = landmarks[holistic.POSE_LANDMARKS.LEFT_SHOULDER];
-    const p2 = landmarks[holistic.POSE_LANDMARKS.RIGHT_SHOULDER];
+    const p1 = landmarks[POSE_LANDMARKS.LEFT_SHOULDER];
+    const p2 = landmarks[POSE_LANDMARKS.RIGHT_SHOULDER];
 
     if (p1.x > 0 && p2.x > 0) {
       this.shoulderWidth[this.shoulderWidthIndex % WINDOW_SIZE] = this.distance(p1, p2);
@@ -63,18 +63,18 @@ export class DetectorService {
 
     // TODO remove, this is to be compliant with openpose
     const neck = {
-      x: (newPose[holistic.POSE_LANDMARKS.LEFT_SHOULDER].x + newPose[holistic.POSE_LANDMARKS.RIGHT_SHOULDER].x) / 2,
-      y: (newPose[holistic.POSE_LANDMARKS.LEFT_SHOULDER].y + newPose[holistic.POSE_LANDMARKS.RIGHT_SHOULDER].y) / 2,
+      x: (newPose[POSE_LANDMARKS.LEFT_SHOULDER].x + newPose[POSE_LANDMARKS.RIGHT_SHOULDER].x) / 2,
+      y: (newPose[POSE_LANDMARKS.LEFT_SHOULDER].y + newPose[POSE_LANDMARKS.RIGHT_SHOULDER].y) / 2,
     };
     const newFakePose = [
-      newPose[holistic.POSE_LANDMARKS.NOSE],
+      newPose[POSE_LANDMARKS.NOSE],
       neck,
-      newPose[holistic.POSE_LANDMARKS.RIGHT_SHOULDER],
-      newPose[holistic.POSE_LANDMARKS.RIGHT_ELBOW],
-      newPose[holistic.POSE_LANDMARKS.RIGHT_WRIST],
-      newPose[holistic.POSE_LANDMARKS.LEFT_SHOULDER],
-      newPose[holistic.POSE_LANDMARKS.LEFT_ELBOW],
-      newPose[holistic.POSE_LANDMARKS.LEFT_WRIST],
+      newPose[POSE_LANDMARKS.RIGHT_SHOULDER],
+      newPose[POSE_LANDMARKS.RIGHT_ELBOW],
+      newPose[POSE_LANDMARKS.RIGHT_WRIST],
+      newPose[POSE_LANDMARKS.LEFT_SHOULDER],
+      newPose[POSE_LANDMARKS.LEFT_ELBOW],
+      newPose[POSE_LANDMARKS.LEFT_WRIST],
       EMPTY_LANDMARK,
       EMPTY_LANDMARK,
       EMPTY_LANDMARK,
@@ -82,10 +82,10 @@ export class DetectorService {
       EMPTY_LANDMARK,
       EMPTY_LANDMARK,
       EMPTY_LANDMARK,
-      newPose[holistic.POSE_LANDMARKS.RIGHT_EYE],
-      newPose[holistic.POSE_LANDMARKS.LEFT_EYE],
-      newPose[holistic.POSE_LANDMARKS.RIGHT_EAR],
-      newPose[holistic.POSE_LANDMARKS.LEFT_EAR],
+      newPose[POSE_LANDMARKS.RIGHT_EYE],
+      newPose[POSE_LANDMARKS.LEFT_EYE],
+      newPose[POSE_LANDMARKS.RIGHT_EAR],
+      newPose[POSE_LANDMARKS.LEFT_EAR],
       EMPTY_LANDMARK,
       EMPTY_LANDMARK,
       EMPTY_LANDMARK,
diff --git a/src/app/modules/pix2pix/pix2pix.service.ts b/src/app/modules/pix2pix/pix2pix.service.ts
index 0a8e3994..a32888b7 100644
--- a/src/app/modules/pix2pix/pix2pix.service.ts
+++ b/src/app/modules/pix2pix/pix2pix.service.ts
@@ -24,7 +24,6 @@ export class Pix2PixService {
 
     await this.tf.load();
 
-    // eslint-disable-next-line
     this.worker = comlink.wrap(new Worker(new URL('./pix2pix.worker', import.meta.url)));
     await this.worker.loadModel();
   }
diff --git a/src/app/modules/pix2pix/pix2pix.worker.ts b/src/app/modules/pix2pix/pix2pix.worker.ts
index 1a433d36..87bb2069 100644
--- a/src/app/modules/pix2pix/pix2pix.worker.ts
+++ b/src/app/modules/pix2pix/pix2pix.worker.ts
@@ -4,12 +4,6 @@ import * as comlink from 'comlink';
 import {Tensor, Tensor3D, TensorLike} from '@tensorflow/tfjs';
 import {LayersModel} from '@tensorflow/tfjs-layers';
 
-class ModelNotLoadedError extends Error {
-  constructor() {
-    super('Model not loaded');
-  }
-}
-
 const tfPromise = import(/* webpackChunkName: "@tensorflow/tfjs" */ '@tensorflow/tfjs');
 let model: LayersModel;
 
@@ -40,7 +34,7 @@ function removeGreenScreen(data: Uint8ClampedArray): Uint8ClampedArray {
 
 async function translate(width: number, height: number, pixels: TensorLike): Promise<Uint8ClampedArray> {
   if (!model) {
-    throw new ModelNotLoadedError();
+    throw new Error('Model not loaded');
   }
   const tf = await tfPromise;
 
diff --git a/src/app/modules/pose/pose.service.ts b/src/app/modules/pose/pose.service.ts
index 02d8e92e..b34b79cb 100644
--- a/src/app/modules/pose/pose.service.ts
+++ b/src/app/modules/pose/pose.service.ts
@@ -1,7 +1,9 @@
 import {Injectable} from '@angular/core';
-import * as holistic from '@mediapipe/holistic/holistic.js';
+import {FACEMESH_FACE_OVAL, FACEMESH_LEFT_EYE, FACEMESH_LEFT_EYEBROW, FACEMESH_LIPS, FACEMESH_RIGHT_EYE, FACEMESH_RIGHT_EYEBROW, FACEMESH_TESSELATION, HAND_CONNECTIONS, POSE_CONNECTIONS, POSE_LANDMARKS} from '@mediapipe/holistic';
 import * as drawing from '@mediapipe/drawing_utils/drawing_utils.js';
 import {Pose, PoseLandmark} from './pose.state';
+import * as comlink from 'comlink';
+
 
 const IGNORED_BODY_LANDMARKS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18, 19, 20, 21, 22];
 
@@ -10,22 +12,38 @@ const IGNORED_BODY_LANDMARKS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18
 })
 export class PoseService {
 
-  model?: any;
+  worker: comlink.Remote<{
+    loadModel: () => Promise<void>,
+    pose: (imageData: ImageData) => Promise<Pose>,
+  }>;
 
   async load(): Promise<void> {
-    this.model = new holistic.Holistic({locateFile: (file) => `assets/models/holistic/${file}`});
+    if (this.worker) {
+      return;
+    }
 
-    this.model.setOptions({
-      upperBodyOnly: false,
-      modelComplexity: 1
-    });
+    this.worker = comlink.wrap(new Worker(new URL('./pose.worker', import.meta.url)));
+    await this.worker.loadModel();
   }
 
-  async predict(video: HTMLVideoElement): Promise<void> {
-    if (!this.model) {
+  async predict(video: HTMLVideoElement): Promise<Pose> {
+    const [width, height] = [video.videoWidth, video.videoHeight];
+    if (!this.worker || width === 0) {
       return Promise.resolve(null);
     }
-    await this.model.send({image: video}); // This is void
+
+    // Create a canvas
+    const fakeImage = document.createElement('canvas');
+    fakeImage.width = width;
+    fakeImage.height = height;
+    const ctx = fakeImage.getContext('2d');
+    ctx.drawImage(video, 0, 0, width, height);
+    const imageData = ctx.getImageData(0, 0, width, height);
+
+    const result: Pose = await this.worker.pose(imageData);
+    result.image = fakeImage;
+
+    return result;
   }
 
   drawBody(landmarks: PoseLandmark[], ctx: CanvasRenderingContext2D): void {
@@ -34,12 +52,12 @@ export class PoseService {
       delete filteredLandmarks[l];
     }
 
-    drawing.drawConnectors(ctx, filteredLandmarks, holistic.POSE_CONNECTIONS, {color: '#00FF00'});
+    drawing.drawConnectors(ctx, filteredLandmarks, POSE_CONNECTIONS, {color: '#00FF00'});
     drawing.drawLandmarks(ctx, filteredLandmarks, {color: '#00FF00', fillColor: '#FF0000'});
   }
 
   drawHand(landmarks: PoseLandmark[], ctx: CanvasRenderingContext2D, lineColor: string, dotColor: string, dotFillColor: string): void {
-    drawing.drawConnectors(ctx, landmarks, holistic.HAND_CONNECTIONS, {color: lineColor});
+    drawing.drawConnectors(ctx, landmarks, HAND_CONNECTIONS, {color: lineColor});
     drawing.drawLandmarks(ctx, landmarks, {
       color: dotColor,
       fillColor: dotFillColor,
@@ -51,13 +69,13 @@ export class PoseService {
   }
 
   drawFace(landmarks: PoseLandmark[], ctx: CanvasRenderingContext2D): void {
-    drawing.drawConnectors(ctx, landmarks, holistic.FACEMESH_TESSELATION, {color: '#C0C0C070', lineWidth: 1});
-    drawing.drawConnectors(ctx, landmarks, holistic.FACEMESH_RIGHT_EYE, {color: '#FF3030'});
-    drawing.drawConnectors(ctx, landmarks, holistic.FACEMESH_RIGHT_EYEBROW, {color: '#FF3030'});
-    drawing.drawConnectors(ctx, landmarks, holistic.FACEMESH_LEFT_EYE, {color: '#30FF30'});
-    drawing.drawConnectors(ctx, landmarks, holistic.FACEMESH_LEFT_EYEBROW, {color: '#30FF30'});
-    drawing.drawConnectors(ctx, landmarks, holistic.FACEMESH_FACE_OVAL, {color: '#E0E0E0'});
-    drawing.drawConnectors(ctx, landmarks, holistic.FACEMESH_LIPS, {color: '#E0E0E0'});
+    drawing.drawConnectors(ctx, landmarks, FACEMESH_TESSELATION, {color: '#C0C0C070', lineWidth: 1});
+    drawing.drawConnectors(ctx, landmarks, FACEMESH_RIGHT_EYE, {color: '#FF3030'});
+    drawing.drawConnectors(ctx, landmarks, FACEMESH_RIGHT_EYEBROW, {color: '#FF3030'});
+    drawing.drawConnectors(ctx, landmarks, FACEMESH_LEFT_EYE, {color: '#30FF30'});
+    drawing.drawConnectors(ctx, landmarks, FACEMESH_LEFT_EYEBROW, {color: '#30FF30'});
+    drawing.drawConnectors(ctx, landmarks, FACEMESH_FACE_OVAL, {color: '#E0E0E0'});
+    drawing.drawConnectors(ctx, landmarks, FACEMESH_LIPS, {color: '#E0E0E0'});
   }
 
   drawConnect(connectors: PoseLandmark[][], ctx: CanvasRenderingContext2D): void {
@@ -82,12 +100,12 @@ export class PoseService {
 
     if (pose.rightHandLandmarks) {
       ctx.strokeStyle = '#00FF00';
-      this.drawConnect([[pose.poseLandmarks[holistic.POSE_LANDMARKS.RIGHT_ELBOW], pose.rightHandLandmarks[0]]], ctx);
+      this.drawConnect([[pose.poseLandmarks[POSE_LANDMARKS.RIGHT_ELBOW], pose.rightHandLandmarks[0]]], ctx);
     }
 
     if (pose.leftHandLandmarks) {
       ctx.strokeStyle = '#FF0000';
-      this.drawConnect([[pose.poseLandmarks[holistic.POSE_LANDMARKS.LEFT_ELBOW], pose.leftHandLandmarks[0]]], ctx);
+      this.drawConnect([[pose.poseLandmarks[POSE_LANDMARKS.LEFT_ELBOW], pose.leftHandLandmarks[0]]], ctx);
     }
   }
 
diff --git a/src/app/modules/pose/pose.state.ts b/src/app/modules/pose/pose.state.ts
index 2a10b7b7..4e2b27b2 100644
--- a/src/app/modules/pose/pose.state.ts
+++ b/src/app/modules/pose/pose.state.ts
@@ -1,7 +1,9 @@
 import {Injectable} from '@angular/core';
-import {Action, NgxsOnInit, State, StateContext, Store} from '@ngxs/store';
+import {Action, NgxsOnInit, Select, State, StateContext} from '@ngxs/store';
 import {PoseService} from './pose.service';
 import {LoadPoseModel, PoseVideoFrame, StoreFramePose} from './pose.actions';
+import {Observable} from 'rxjs';
+import {filter, first, tap} from 'rxjs/operators';
 
 export interface PoseLandmark {
   x: number;
@@ -37,40 +39,31 @@ const initialState: PoseStateModel = {
   defaults: initialState
 })
 export class PoseState implements NgxsOnInit {
-  constructor(private poseService: PoseService, private store: Store) {
+  @Select(state => state.settings.pose) poseSetting$: Observable<boolean>;
+
+  constructor(private poseService: PoseService) {
   }
 
-  ngxsOnInit(ctx?: StateContext<any>): void {
-    this.store.dispatch(LoadPoseModel);
+  ngxsOnInit({dispatch}: StateContext<any>): void {
+    this.poseSetting$.pipe(
+      filter(Boolean),
+      first(),
+      tap(() => dispatch(LoadPoseModel))
+    ).subscribe();
   }
 
   @Action(LoadPoseModel)
   async load({patchState, dispatch}: StateContext<PoseStateModel>): Promise<void> {
     patchState({isLoaded: false});
     await this.poseService.load();
-    this.poseService.model.onResults((results) => {
-      // TODO: passing the `image` canvas through NGXS bugs the pose.
-      // https://github.com/google/mediapipe/issues/2422
-      const fakeImage = document.createElement('canvas');
-      fakeImage.width = results.image.width;
-      fakeImage.height = results.image.height;
-      const ctx = fakeImage.getContext('2d');
-      ctx.drawImage(results.image, 0, 0, fakeImage.width, fakeImage.height);
-
-      // Since v0.4, "results" include additional parameters
-      this.store.dispatch(new StoreFramePose({
-        faceLandmarks: results.faceLandmarks,
-        poseLandmarks: results.poseLandmarks,
-        leftHandLandmarks: results.leftHandLandmarks,
-        rightHandLandmarks: results.rightHandLandmarks,
-        image: fakeImage
-      }));
-    });
   }
 
   @Action(PoseVideoFrame)
   async poseFrame({patchState, dispatch}: StateContext<PoseStateModel>, {video}: PoseVideoFrame): Promise<void> {
-    await this.poseService.predict(video);
+    const result = await this.poseService.predict(video);
+
+    // Since v0.4, "results" include additional parameters
+    dispatch(new StoreFramePose(result));
   }
 
   @Action(StoreFramePose)
diff --git a/src/app/modules/pose/pose.worker.ts b/src/app/modules/pose/pose.worker.ts
new file mode 100644
index 00000000..0bd8784c
--- /dev/null
+++ b/src/app/modules/pose/pose.worker.ts
@@ -0,0 +1,44 @@
+/// <reference lib="webworker" />
+
+import * as comlink from 'comlink';
+import {Holistic} from '@mediapipe/holistic';
+import {Observable} from 'rxjs';
+import {first} from 'rxjs/operators';
+
+let model: Holistic;
+let results: Observable<any>;
+
+async function loadModel(): Promise<void> {
+  model = new Holistic({locateFile: (file) => `/assets/models/holistic/${file}`});
+
+  model.setOptions({modelComplexity: 1});
+  await model.initialize();
+
+  results = new Observable(subscriber => {
+    model.onResults((results) => {
+      subscriber.next({
+        faceLandmarks: results.faceLandmarks,
+        poseLandmarks: results.poseLandmarks,
+        leftHandLandmarks: results.leftHandLandmarks,
+        rightHandLandmarks: results.rightHandLandmarks
+      });
+    });
+  });
+}
+
+async function pose(imageData: ImageData): Promise<any> {
+  if (!model) {
+    throw new Error('Model not loaded');
+  }
+
+  const image = new OffscreenCanvas(imageData.width, imageData.height);
+  const ctx = image.getContext('2d');
+  ctx.putImageData(imageData, 0, 0);
+
+  const result = results.pipe(first()).toPromise();
+  await model.send({image: image as any});
+  return result;
+}
+
+
+comlink.expose({loadModel, pose});
diff --git a/src/app/modules/settings/settings.state.ts b/src/app/modules/settings/settings.state.ts
index ebeb878c..9eccd9d5 100644
--- a/src/app/modules/settings/settings.state.ts
+++ b/src/app/modules/settings/settings.state.ts
@@ -12,6 +12,7 @@ export interface SettingsStateModel {
   animatePose: boolean;
 
   drawVideo: boolean;
+  pose: boolean;
   drawPose: boolean;
   drawSignWriting: boolean;
 
@@ -26,6 +27,7 @@ const initialState: SettingsStateModel = {
   animatePose: false,
 
   drawVideo: true,
+  pose: false,
   drawPose: true,
   drawSignWriting: false,
 
diff --git a/src/app/modules/sign-writing/body.service.ts b/src/app/modules/sign-writing/body.service.ts
index 2d5f797c..d442b4b9 100644
--- a/src/app/modules/sign-writing/body.service.ts
+++ b/src/app/modules/sign-writing/body.service.ts
@@ -1,7 +1,7 @@
 import {Injectable} from '@angular/core';
 import {SignWritingService} from './sign-writing.service';
 import {PoseLandmark} from '../pose/pose.state';
-import {POSE_LANDMARKS} from '@mediapipe/holistic/holistic.js';
+import {POSE_LANDMARKS} from '@mediapipe/holistic';
 import {ThreeService} from '../../core/services/three.service';
 import {Vector2} from 'three';
 
diff --git a/src/app/modules/sign-writing/sign-writing.state.ts b/src/app/modules/sign-writing/sign-writing.state.ts
index f31112f7..cfadbabe 100644
--- a/src/app/modules/sign-writing/sign-writing.state.ts
+++ b/src/app/modules/sign-writing/sign-writing.state.ts
@@ -6,7 +6,7 @@ import {filter, first, tap} from 'rxjs/operators';
 import {HandsService, HandStateModel} from './hands.service';
 import {CalculateBodyFactors, EstimateFaceShape, EstimateHandShape} from './sign-writing.actions';
 import {BodyService, BodyStateModel} from './body.service';
-import * as holistic from '@mediapipe/holistic/holistic';
+import {POSE_LANDMARKS} from '@mediapipe/holistic';
 import {FaceService, FaceStateModel} from './face.service';
 import {ThreeService} from '../../core/services/three.service';
 
@@ -90,8 +90,8 @@ export class SignWritingState implements NgxsOnInit {
     patchState({
       body: {
         shoulders: this.bodyService.shoulders(pose.poseLandmarks),
-        elbows: [pose.poseLandmarks[holistic.POSE_LANDMARKS.LEFT_ELBOW], pose.poseLandmarks[holistic.POSE_LANDMARKS.RIGHT_ELBOW]],
-        wrists: [pose.poseLandmarks[holistic.POSE_LANDMARKS.LEFT_WRIST], pose.poseLandmarks[holistic.POSE_LANDMARKS.RIGHT_WRIST]],
+        elbows: [pose.poseLandmarks[POSE_LANDMARKS.LEFT_ELBOW], pose.poseLandmarks[POSE_LANDMARKS.RIGHT_ELBOW]],
+        wrists: [pose.poseLandmarks[POSE_LANDMARKS.LEFT_WRIST], pose.poseLandmarks[POSE_LANDMARKS.RIGHT_WRIST]],
       }
     });
   }
diff --git a/src/app/pages/translate/translate.component.ts b/src/app/pages/translate/translate.component.ts
index 5d13d653..eb62f3bc 100644
--- a/src/app/pages/translate/translate.component.ts
+++ b/src/app/pages/translate/translate.component.ts
@@ -30,6 +30,7 @@ export class TranslateComponent extends BaseComponent implements OnInit {
       new SetSetting('receiveVideo', true),
       new SetSetting('detectSign', false),
       new SetSetting('drawSignWriting', false),
+      new SetSetting('pose', false),
       new SetSetting('drawPose', true),
       new SetSetting('poseViewer', 'pose'),
     ]);
@@ -52,7 +53,10 @@ export class TranslateComponent extends BaseComponent implements OnInit {
       tap((spokenToSigned) => {
         this.spokenToSigned = spokenToSigned;
         if (!this.spokenToSigned) {
-          this.store.dispatch(new SetSetting('drawSignWriting', true));
+          this.store.dispatch([
+            new SetSetting('pose', true),
+            new SetSetting('drawSignWriting', true)
+          ]);
         }
       }),
       takeUntil(this.ngUnsubscribe)

From c100d788c994e6ba3de27116a43deb574f431d0a Mon Sep 17 00:00:00 2001
From: Amit <amitmoryossef@gmail.com>
Date: Mon, 6 Sep 2021 13:53:00 +0300
Subject: [PATCH 2/8] feat(holistic): move pose estimation to web worker

---
 src/app/modules/pose/pose.service.ts | 7 +++++--
 src/app/modules/pose/pose.worker.ts  | 5 +++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/app/modules/pose/pose.service.ts b/src/app/modules/pose/pose.service.ts
index b34b79cb..a4212d4a 100644
--- a/src/app/modules/pose/pose.service.ts
+++ b/src/app/modules/pose/pose.service.ts
@@ -29,7 +29,7 @@ export class PoseService {
   async predict(video: HTMLVideoElement): Promise<Pose> {
     const [width, height] = [video.videoWidth, video.videoHeight];
     if (!this.worker || width === 0) {
-      return Promise.resolve(null);
+      return null;
     }
 
     // Create a canvas
@@ -41,8 +41,11 @@ export class PoseService {
     const imageData = ctx.getImageData(0, 0, width, height);
 
     const result: Pose = await this.worker.pose(imageData);
-    result.image = fakeImage;
+    if (!result) {
+      return null;
+    }
 
+    result.image = fakeImage;
     return result;
   }
 
diff --git a/src/app/modules/pose/pose.worker.ts b/src/app/modules/pose/pose.worker.ts
index 0bd8784c..600daa34 100644
--- a/src/app/modules/pose/pose.worker.ts
+++ b/src/app/modules/pose/pose.worker.ts
@@ -13,6 +13,7 @@ async function loadModel(): Promise<void> {
 
   model.setOptions({modelComplexity: 1});
   await model.initialize();
+  console.log('Pose model loaded!');
 
   results = new Observable(subscriber => {
     model.onResults((results) => {
@@ -27,8 +28,8 @@ async function loadModel(): Promise<void> {
 }
 
 async function pose(imageData: ImageData): Promise<any> {
-  if (!model) {
-    throw new Error('Model not loaded');
+  if (!results) {
+    return null;
   }
 
   const image = new OffscreenCanvas(imageData.width, imageData.height);

From e2797e0ccf3ca8488ac96a0f8cf052e4b50d23ac Mon Sep 17 00:00:00 2001
From: Amit <amitmoryossef@gmail.com>
Date: Thu, 9 Sep 2021 18:48:55 +0300
Subject: [PATCH 3/8] feat(pose): use a bitmap rather than data

---
 src/app/modules/pose/pose.service.ts | 9 +++++----
 src/app/modules/pose/pose.worker.ts  | 6 +++---
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/app/modules/pose/pose.service.ts b/src/app/modules/pose/pose.service.ts
index a4212d4a..2a886428 100644
--- a/src/app/modules/pose/pose.service.ts
+++ b/src/app/modules/pose/pose.service.ts
@@ -14,7 +14,7 @@ export class PoseService {
 
   worker: comlink.Remote<{
     loadModel: () => Promise<void>,
-    pose: (imageData: ImageData) => Promise<Pose>,
+    pose: (imageBitmap: ImageBitmap) => Promise<Pose>,
   }>;
 
   async load(): Promise<void> {
@@ -37,10 +37,11 @@ export class PoseService {
     fakeImage.width = width;
     fakeImage.height = height;
     const ctx = fakeImage.getContext('2d');
-    ctx.drawImage(video, 0, 0, width, height);
-    const imageData = ctx.getImageData(0, 0, width, height);
 
-    const result: Pose = await this.worker.pose(imageData);
+    const bitmap = await createImageBitmap(video);
+    ctx.drawImage(bitmap, 0, 0);
+
+    const result: Pose = await this.worker.pose(bitmap);
     if (!result) {
       return null;
     }
diff --git a/src/app/modules/pose/pose.worker.ts b/src/app/modules/pose/pose.worker.ts
index 600daa34..95a0c148 100644
--- a/src/app/modules/pose/pose.worker.ts
+++ b/src/app/modules/pose/pose.worker.ts
@@ -27,14 +27,14 @@ async function loadModel(): Promise<void> {
   });
 }
 
-async function pose(imageData: ImageData): Promise<any> {
+async function pose(imageBitmap: ImageBitmap): Promise<any> {
   if (!results) {
     return null;
   }
 
-  const image = new OffscreenCanvas(imageData.width, imageData.height);
+  const image = new OffscreenCanvas(imageBitmap.width, imageBitmap.height);
   const ctx = image.getContext('2d');
-  ctx.putImageData(imageData, 0, 0);
+  ctx.drawImage(imageBitmap, 0, 0);
 
   const result = results.pipe(first()).toPromise();
   await model.send({image: image as any});

From 1c944b896a193c2d58ee44d7b8adc28acd04d943 Mon Sep 17 00:00:00 2001
From: Amit <amitmoryossef@gmail.com>
Date: Mon, 27 Sep 2021 11:34:06 +0300
Subject: [PATCH 4/8] feat(pose): transfarable image bitmap

---
 src/app/modules/pix2pix/pix2pix.worker.ts |  2 +-
 src/app/modules/pose/pose.service.ts      |  2 +-
 src/app/modules/pose/pose.worker.ts       | 17 ++++++++++++-----
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/app/modules/pix2pix/pix2pix.worker.ts b/src/app/modules/pix2pix/pix2pix.worker.ts
index 889c3a95..429bd943 100644
--- a/src/app/modules/pix2pix/pix2pix.worker.ts
+++ b/src/app/modules/pix2pix/pix2pix.worker.ts
@@ -54,7 +54,7 @@ async function translate(bitmap: ImageBitmap): Promise<Uint8ClampedArray> {
 
   const data = await tf.browser.toPixels(image);
   const cleanData = removeGreenScreen(data);
-  return comlink.transfer(cleanData, [cleanData.buffer])
+  return comlink.transfer(cleanData, [cleanData.buffer]);
 }
 
 
diff --git a/src/app/modules/pose/pose.service.ts b/src/app/modules/pose/pose.service.ts
index 2a886428..ce633793 100644
--- a/src/app/modules/pose/pose.service.ts
+++ b/src/app/modules/pose/pose.service.ts
@@ -41,7 +41,7 @@ export class PoseService {
     const bitmap = await createImageBitmap(video);
     ctx.drawImage(bitmap, 0, 0);
 
-    const result: Pose = await this.worker.pose(bitmap);
+    const result: Pose = await this.worker.pose(comlink.transfer(bitmap, [bitmap]));
     if (!result) {
       return null;
     }
diff --git a/src/app/modules/pose/pose.worker.ts b/src/app/modules/pose/pose.worker.ts
index 95a0c148..31556ab5 100644
--- a/src/app/modules/pose/pose.worker.ts
+++ b/src/app/modules/pose/pose.worker.ts
@@ -9,7 +9,13 @@ let model: Holistic;
 let results: Observable<any>;
 
 async function loadModel(): Promise<void> {
-  model = new Holistic({locateFile: (file) => `/assets/models/holistic/${file}`});
+  model = new Holistic({
+    locateFile: (file) => {
+      const f = new URL(`/assets/models/holistic/${file}`, globalThis.location.origin).toString();
+      console.log('path', f);
+      return f;
+    }
+  });
 
   model.setOptions({modelComplexity: 1});
   await model.initialize();
@@ -32,12 +38,13 @@ async function pose(imageBitmap: ImageBitmap): Promise<any> {
     return null;
   }
 
-  const image = new OffscreenCanvas(imageBitmap.width, imageBitmap.height);
-  const ctx = image.getContext('2d');
-  ctx.drawImage(imageBitmap, 0, 0);
+  // TODO remove
+  // const image = new OffscreenCanvas(imageBitmap.width, imageBitmap.height);
+  // const ctx = image.getContext('2d');
+  // ctx.drawImage(imageBitmap, 0, 0);
 
   const result = results.pipe(first()).toPromise();
-  await model.send({image: image as any});
+  await model.send({image: imageBitmap as any});
   return result;
 }
 

From 4c12a54c356f924b24e183afd0427b9c963a979a Mon Sep 17 00:00:00 2001
From: Amit <amitmoryossef@gmail.com>
Date: Sat, 9 Oct 2021 10:29:17 +0300
Subject: [PATCH 5/8] fix(pose): use transferable image util

---
 src/app/core/helpers/image/transferable.ts | 16 +++++++++++-----
 src/app/modules/pose/pose.service.ts       | 16 +++++-----------
 src/app/modules/pose/pose.worker.ts        |  2 +-
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/app/core/helpers/image/transferable.ts b/src/app/core/helpers/image/transferable.ts
index 9f1be79b..72fc7a82 100644
--- a/src/app/core/helpers/image/transferable.ts
+++ b/src/app/core/helpers/image/transferable.ts
@@ -5,18 +5,24 @@ export async function transferableImage(image: HTMLCanvasElement | HTMLVideoElem
   //   return comlink.transfer(bitmap, [bitmap]);
   // }
 
+  let {width, height} = image;
+  if (image instanceof HTMLVideoElement) {
+    width = image.videoWidth;
+    height = image.videoHeight;
+  }
+
   let ctx: CanvasRenderingContext2D;
   if (image instanceof HTMLCanvasElement) {
     ctx = image.getContext('2d');
   } else {
     const canvas = document.createElement('canvas');
-    canvas.width = image.width;
-    canvas.height = image.height;
-    const ctx = canvas.getContext('2d');
-    ctx.drawImage(image, 0, 0, image.width, image.height);
+    canvas.width = width;
+    canvas.height = height;
+    ctx = canvas.getContext('2d');
+    ctx.drawImage(image, 0, 0, width, height);
   }
 
-  const data = ctx.getImageData(0, 0, image.width, image.height);
+  const data = ctx.getImageData(0, 0, width, height);
   // return comlink.transfer(data, [data.data]);
   return data;
 }
diff --git a/src/app/modules/pose/pose.service.ts b/src/app/modules/pose/pose.service.ts
index ce633793..e9954891 100644
--- a/src/app/modules/pose/pose.service.ts
+++ b/src/app/modules/pose/pose.service.ts
@@ -3,6 +3,7 @@ import {FACEMESH_FACE_OVAL, FACEMESH_LEFT_EYE, FACEMESH_LEFT_EYEBROW, FACEMESH_L
 import * as drawing from '@mediapipe/drawing_utils/drawing_utils.js';
 import {Pose, PoseLandmark} from './pose.state';
 import * as comlink from 'comlink';
+import {transferableImage} from '../../core/helpers/image/transferable';
 
 
 const IGNORED_BODY_LANDMARKS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18, 19, 20, 21, 22];
@@ -14,7 +15,7 @@ export class PoseService {
 
   worker: comlink.Remote<{
     loadModel: () => Promise<void>,
-    pose: (imageBitmap: ImageBitmap) => Promise<Pose>,
+    pose: (imageBitmap: ImageBitmap | ImageData) => Promise<Pose>,
   }>;
 
   async load(): Promise<void> {
@@ -32,21 +33,14 @@ export class PoseService {
       return null;
     }
 
-    // Create a canvas
-    const fakeImage = document.createElement('canvas');
-    fakeImage.width = width;
-    fakeImage.height = height;
-    const ctx = fakeImage.getContext('2d');
+    const image = await transferableImage(video);
 
-    const bitmap = await createImageBitmap(video);
-    ctx.drawImage(bitmap, 0, 0);
-
-    const result: Pose = await this.worker.pose(comlink.transfer(bitmap, [bitmap]));
+    const result: Pose = await this.worker.pose(image);
     if (!result) {
       return null;
     }
 
-    result.image = fakeImage;
+    // result.image = image; // TODO fix
     return result;
   }
 
diff --git a/src/app/modules/pose/pose.worker.ts b/src/app/modules/pose/pose.worker.ts
index 31556ab5..14807361 100644
--- a/src/app/modules/pose/pose.worker.ts
+++ b/src/app/modules/pose/pose.worker.ts
@@ -33,7 +33,7 @@ async function loadModel(): Promise<void> {
   });
 }
 
-async function pose(imageBitmap: ImageBitmap): Promise<any> {
+async function pose(imageBitmap: ImageBitmap | ImageData): Promise<any> {
   if (!results) {
     return null;
   }

From 1b8cb500924365470345ea74c05a5fcae7b9a486 Mon Sep 17 00:00:00 2001
From: Amit <amitmoryossef@gmail.com>
Date: Tue, 7 Dec 2021 10:43:24 +0200
Subject: [PATCH 6/8] attempt getting holistic 0.5 in web worker

---
 src/app/core/helpers/image/transferable.ts     |  14 +++++++-------
 src/app/modules/pix2pix/pix2pix.service.ts     |   1 -
 src/app/modules/translate/translate.service.ts |   5 +++--
 src/assets/tmp/example.pose                    | Bin 0 -> 16755 bytes
 4 files changed, 10 insertions(+), 10 deletions(-)
 create mode 100644 src/assets/tmp/example.pose

diff --git a/src/app/core/helpers/image/transferable.ts b/src/app/core/helpers/image/transferable.ts
index 72fc7a82..4ce77cdd 100644
--- a/src/app/core/helpers/image/transferable.ts
+++ b/src/app/core/helpers/image/transferable.ts
@@ -1,9 +1,10 @@
+import * as comlink from 'comlink';
+
 export async function transferableImage(image: HTMLCanvasElement | HTMLVideoElement): Promise<ImageBitmap | ImageData> {
-  // TODO reconsider this, once I can consistently make it run fast
-  // if (window.createImageBitmap) {
-  //   const bitmap = await createImageBitmap(image);
-  //   return comlink.transfer(bitmap, [bitmap]);
-  // }
+  if (window.createImageBitmap) {
+    const bitmap = await createImageBitmap(image);
+    return comlink.transfer(bitmap, [bitmap]);
+  }
 
   let {width, height} = image;
   if (image instanceof HTMLVideoElement) {
@@ -23,6 +24,5 @@ export async function transferableImage(image: HTMLCanvasElement | HTMLVideoElem
   }
 
   const data = ctx.getImageData(0, 0, width, height);
-  // return comlink.transfer(data, [data.data]);
-  return data;
+  return comlink.transfer(data, [data.data.buffer]);
 }
diff --git a/src/app/modules/pix2pix/pix2pix.service.ts b/src/app/modules/pix2pix/pix2pix.service.ts
index 6015c1ff..af0da006 100644
--- a/src/app/modules/pix2pix/pix2pix.service.ts
+++ b/src/app/modules/pix2pix/pix2pix.service.ts
@@ -17,7 +17,6 @@ export class Pix2PixService {
       return;
     }
 
-    // eslint-disable-next-line
     this.worker = comlink.wrap(new Worker(new URL('./pix2pix.worker', import.meta.url)));
     await this.worker.loadModel();
   }
diff --git a/src/app/modules/translate/translate.service.ts b/src/app/modules/translate/translate.service.ts
index 2942de46..df259f38 100644
--- a/src/app/modules/translate/translate.service.ts
+++ b/src/app/modules/translate/translate.service.ts
@@ -45,8 +45,9 @@ export class TranslationService {
   }
 
   translateSpokenToSigned(text: string, spokenLanguage: string, signedLanguage: string): string {
-    const api = 'https://nlp.biu.ac.il/~ccohenya8/sign/sentence/';
-    return `${api}?slang=${spokenLanguage}&dlang=${signedLanguage}&sentence=${encodeURIComponent(text)}`;
+    // const api = 'https://nlp.biu.ac.il/~ccohenya8/sign/sentence/';
+    // return `${api}?slang=${spokenLanguage}&dlang=${signedLanguage}&sentence=${encodeURIComponent(text)}`;
+    return 'assets/tmp/example.pose'
   }
 
 }
diff --git a/src/assets/tmp/example.pose b/src/assets/tmp/example.pose
new file mode 100644
index 0000000000000000000000000000000000000000..17968448e3f77cdadb06dac3c17e04cf38ca982e
GIT binary patch
literal 16755
zcmeHucT|<f)9|H=igcu0dhZ<+kUPo@s7UkDk=~RpA}SgS5=BHrENBq1A!3O!ifD``
zF~k^S46(%!yC{en>;x3?+h@&t-rqUz_h;Vkug^W)9cSm6XJ&Wr&Q7@^kx2AN?vb2G
zBrj4I%`Pm?OwP?Lon4q+Ft<3_EkjNe8;5!#U6BkM4`rk5Gc!GRq(~YutFR<5BXf?T
zNE(orRyd#aF>|tu=Z+Kw|MyVvKSRNMNKq7$oiQbQw!BEnJ{3gLX$6^??72w=xp|Cn
zFh>S|L<awZlm?UvL;O=3kUB?zQ|F8j1^Z{uj4aF)-j<)5m&ZT_#vn5@Z-hwtUqjOW
z9+C<}G9p>l<XDqu`Y5ob$l3@7E3sKnV}GjbPlNrbGn|NFN3u4Gwb2Y#W^D{>D(ru{
z>~Ab<I_!Vi>`#+5Ezt+|jPPt$YWn|W76V0-_$&tcC%edt*rzN1bmO1y{L_Pfdh$;%
z{yC0+j_04=3j8UTkDcqs&h2C8{;~7;*m-{Jyc9)&0qi~F{^`g6(|dpH-CX|ZUH|Fb
z{^{NS={^4GJ^$&w7~9Zfw{ai4@gF-us?cP2K`Pb>Qn5~uigkiitP`YSogh^}vb(^A
zbpjXG30znwa0y8E5V){T;KDkA3+n_ftP{Af?jskEk6b(i)q=y4Jp|Q+UQmtoo`PyZ
zFQ_K;f@(r9s3!D+YC<olCiKEo3H?9(JpbY6$@xu9_Tv0l&-t;Q^J6{d$9m3>^_(B;
zIX~75T81Zk30ktwTZ!$q5q#GPyHB1qVK)l9Qs#f`xY7Ty<5bwrRAoEVMdZqw8*A>Y
zd9dcmnip&1SR2opH)}qu`LgCBnjrFHO~M+mHj%YStogHstW6M!MFy-HvS!4ZF>5BQ
znX+canmNOnvi+>V+Q%KP#df=}>vh;SKv$&4n!d<FWXYNpYu2pUux87eg~(21&zb{k
zj;uMc=FFO%Xc_;Oj1|pFEyzgD%gmblKMx#j_GQv$ljA=rB0xbDnH&)rsmMBi3_;<)
zAs8;tx~bs`qN&LoH&wt1{|o<!B%urWI2Q78Eac-@NHQOjejJm29Fu+=lYSfv|2P)@
zaV-4fShz4I;)|rf6d1|=#;`x(p!n}Y<-ZP;5&Y=*cyI{IC>$F9wS@mUosUF9`;PGu
ziMrTyn~M}>0!4u$TbXQ8Hp2@eU3^4QMi{Y`5k^EDZ_e?7{E2M-gn<8!@erO6@QRF9
zc!E8^@xq9}UBC+?Iv<x{PWH@M|9=gHN#Zm5|8@-o6C)$~@S*7s1B%ZmL9Hj@;3qYr
zq0(bm)@l#eVxOStvZF|OcO|veUt(msKBY=+Kxc3mW3wNjMvgxWl?71s%oF&HlN|Uz
zPoZKZEpj|;4(FTw7+#j94~_$_-oN~Dq(q0RV*;_dEera0SkVf5DJsV=0HuQ}WV&tw
z))YiSaEvEixR;2hCdK}1GKihksL0S0C#=wd9ywJixe|iKnhG2)t<1uW;bt5@*}E7o
zn(2adrwZK|tVCL&2=$$+)ZelJBMupW$_jO=k==%Gwz)&rN-b(vXu@F!Cn#6aqeCNF
z@XP}@2tQ^>jz{+5`VMPQ8DmcVhj-(F$9mB4z?y15--cf=HiEXV?dit8O*s5q6KXV^
z$v=DzzPg|bZ5v$4f5$R(nQ98X-?$M~<zSJw57>h{`7e_qHCw=$cnvE4$rGoEqQO&9
zi*Ehkh-oV#AfaL`UDLHkwJ(xE_NXr1(|1PS`{B@YRG;g9>(_V)y>CYTCcd~fHXS_I
zT2e`|7vBCN36f6QaGpD_&)~8NG=IE^0Q;Gabju<D*Gx$O{}dxybTth7>cXLBwIMxp
zNk;l+2B>%%Qt-HZ?D#Yq5|Rz*h~FaI_In(ZCF;_mjZ3iPK?1b7>r>N?GTi7I3O7a?
zkZo`|CWhsKY?TqU^e@L)<C!qu%Y>SzEywZ8Q=sRD7TxIxz^ol<aPj6?(z8HpU7H2&
z?s^=*JTi^*6xQ+N?l3sxrc0%^Vd&AA2RB^D(&5X~uxVco*Eh%^3a?%*0C^QlPP6T0
z0_^v;=DMh;&x9HuTME7&g(;Fu=vA>Moz6IXe=48TYm~>L&9+!Rw*$LZVS7dpe^<%w
z3cNWb5&D(%sehmX_px~p-w{)1XeC-EPUG_vJGcs8iKNhf*O+cKtwycKi4ecWl-l>K
zL9zNQXiYIDW$8Msxt+r2P$ObJW|Yo=mfQMN`@R;>EsN#*N}wrGNQdeXI&?O?8rAQy
zIZxE#@9O*_0!ohRQ9{^iT(UM9>MO;R^ma8~u8~4VvLT0;j1LFvR1^N*G{-ECw`{J)
zk`1$9)<`i4FlpsN>nc6I50CmMf=Ef1x-2K-P;)G_by*VGN8pQDGvJh#H(d&nr7u^j
zf_t$KH5T^c&DXv#(C$qgIuEhYVmha3GyfCYTw=ghc>>K!y^P{lS#Y@Coyy)k$Ib7J
zAhyVn?UPrywQCB9rdU(Wkyp66D+<z-%;?P9H#oaA5|&C`>DJtLxa_q$R5aVrQ`cb}
zUNR^NdSF5-r{AOO>j*g5B<A?Kq%a5_<xJM8GGudA70zy9XUkR@nsL~J&8-Q?kIxE$
z4a@WiR>;!#;gjK#h67m|$&<H@DIA<@NsZsiQs@>JxUtWK(}*37p(nzC<CV@mkQAPj
zr?eeCxPJ5J62DPn=-`A~Sor&4iCVZ8iI<*7?U+rHf%LJIV)-4WN?Rqxhjpka{5USm
zSA=iQDbu0jt@yMvU1Gjlm3G9RK$(FGiTJoWr$2czLb73%CWT)90rf6u@pm=eIe~qW
z3W=rBNBU834oY_9sF88WPk7{1uEfwyos=AY#SNYdC2w!bP*&YHxaY=l$;nG&Xnw&-
zEM0WL&s$4{RJQ$s-`|#(oC+R8*Z#PSM*4wFW)TI)oW<NWsif3Ymdta{;lP0de&?M=
zar%c<6@KlfN7KcpSMm6XZa=}^Y>76c6G<iLFeL)CLX9YY=5Y+pwFb+qUaVPu6ceXE
zmh{#g#yVzi)aE<Foe%G@KD!6EiU1BT51@NFCs6CxVab#iDKwv*oxi17Le)V(x^!Qb
z#=P_X*UtL?G+BNNbtox55O)Qn!Elom1vyAjt#%&hS*X&`zVSGyJ_E{QJ?Z(mWSqNr
z#=j=R6;CyaL@(6y7K3M!Dy6Il#o!1v4(FfC!cUhNak%O5VqBW9&c1J|bTN1}KKNDy
zjdxV3=j#pVa77HAYt`wV-8TGoizAflYSZ(c-B`TC9WIpWQC8qyeB<N=hmSDZ|5-DB
z`PLqazcZ)(wY%|Eu`yf>wjt}XZMb5DCEST|pjl@&VL_7#n5H;W*PbfuGc|;_-L6z!
zu?#!<ZQz!cI~88e!L?udLSUUc^-YMt`6sR6)F}-*U+0N`Mx?>RiCR?O>4awQlh~df
zOIv=m!w{DgXbIIL%azV}D=8Yf-s_X!V-Ji@i-#UfbGrA|2esFxfX)j`J|`abGoU)m
zmh;@ym<aVU4xA=EDF&_`cBH_u0jOq@4DGj!X#VXm)SjCRQ(TScbz(B=I7Y(7PYp@!
zk31Y!PKF96Ly|qP2v0IQVYrLgfRRh_+tCrwxL2Rb@0Q`SLlMv$Z9w1Dlw(i+EY7pu
zyBv4!ng^~YOgPPxsz_)?ZMyV*0M2a9gnKW?lJjXqyOtC_&jS9B%ekDVu#W9pqo8M+
zE<1O^aK_P0XlNMA?i16{&2$deH`G50(_9PSP@yHKIo*{4(jC@Zm%^Dd;qXFR>g<cc
zk^?hAImnvbGvZLaJ%`hu-4TnuOFqtR?Y32T;7KHZm*R^GT%{h*WeA?U3RC7J!#R5c
zG7PH3V#jEB{)Hi3f3*rT_lCiFW=Dm+G%6$wtkum(X{ZVZ|4!ui@WgefSysU5v)XDg
zrZfo@ZN+4oQj3AD)49F^jjdTO1kKf<j04peox$wib{+mMHOpu)f2T(pyH{hv1}T@J
zF{}!;AH;B;0({CO6e_(<`Fq#ROy~GYnYDOqYz~xN6LX%v4|3pElOBai*I?4uv0TUE
zpC)7J!8iz>Y{mDUTgeQV(&<h49kOI4RfL8gd?+FLZ)|LHgWgadl0JHf@z<g`P5IqF
zQPL3uwU!e|e&rR+H!6hIiJoLV{UtW(u`{aMk&3px#z(iOLBmsP($IW^@t+34irr>3
zeD!Y}S(OIg+;OM=*!SpDrvr;s9jIuGh<3G#p>>uS!JZGOY83}V`wTez-IoYDBi%?P
zK#n5!>OgIXE!|lvM_6M3DT__H%#(}9L;n;prx{v19j?!CqKaq*Dp647GRQZ}Q}k9%
zu+}r@&wf>;%IuR7hb!KFCxM4D)M5Pqy|?yBI+K*B>G>`EN_t0fSYMSk{P7DmUwtS!
z`K3CQJ^U6UE?R)?56phNJc8{fRH5t5D9S&661R%eC6^wn(TaCR@u>0{PA}kF&iyW_
zzo$y=E8213{RW9UDv-U-am4YrB~qm^)I0Aep8911*msN}!<_GMS*kV&_%lHlaC^4_
zDBToMz3o@nxk&|{ZyrHM@{eFp-3U;dHInme+in6ST}oWeb`^V=UoeP4H@`s(G6Y4_
z4|s0taa8WUF6lU@K-Mztn0KU3qIgM>#LhqAnY~jbg1u?HZb+q-#klfAEbNanB+=|P
z^y)T+_>5k>wEj!nyylf8sNoR0KhYzxx&?SjUSR3eXXrD@7ZjO)Bb{{uPe*-_%<oGf
zV+C!>aW;eKLw>X(U5@Uq@cY-!`v0^y(%HIXzKGE2#}t@vWliQ{DLVGegit$GI&#_z
zj~|PJ#n(OAT{{^Een|V*WO%w$jhf$iVBRAQD0-txCnO<wSzi_Gx2w{HvP{hR(-7JX
zRY|J57$e8&aM)UAHSW4B4{g)b==_fxu<D%{WZ$Wi=xzfpFLq+*tTv6Byc_+tc!0*&
zdgLhGi<gYOAlTTLdfqkTl~eW*?r6bju7LrZSZhOpUfb~5S!;Ou8M|A}+k~bI44{0#
znaU4Wq47ywm}2Wj7iyQ`k(cIRALCB--Z?0v>kDmi9@H2XfoXqR!j<<L>^|#(-`b@!
zKUIsK);gk!TPo;z=upc_TRfzi%;lLcal&T1RH)q~rc1lrF=2ifoLy#4B}ctcx+MmJ
zTCMn;)cJ=)M~^M%Y1y60WfN$Ytce3HBPZ%%>xF(%P#9!P@f3#FPp5+IYa>$AO~P^e
z5@6N~Lpr=Z7gMX^An->+vdmhDp2~&HhtOlb*JAWoAq874F*R%~!|&qaVN0I@UB_}9
z%ue7ugEue7JvU3ie32=qIV+B1XQDP0+XP^RG#AQ_>Ch!J#Peox;MuLm<=J#H4}w8V
z^dtn$zMlaVDtgqhITU}eN@6~T4#j#;!_^7}&~w0?Y=<In>%%N)xM|609#5PDb?G)-
zmq)MXa@`>+3issZK=F7R@=uAw^@=k&z5R$-%<#?SJTFXNg`JXcke<*baaaW&y%@*!
zO4(C^1`D!5y2XH^hnek>kAg#!+5EJu!tFl?K-vO!zh-;Mbipimz1WPe^_+nOj&J_D
z22aJ#=5r{UTZ@m_eXHp^F`d$?#WQn*_&N$S<=@YSbE9>seq=TJen^5|FXq4QuS9i&
z80b2!Px&#cv2IcvIOiMCfLs+??@WOz10xO}PY>jCE9^1Lg>f7&@aejf4$A%poaZ-M
zb70W~b|+vo$~~c6M=htx7-<*{HKUk4*e%6nln%2}eMn`DJXIdkf~-(qX7dK|$Z}_9
z<9*0<$0Hp4b_S>M{`3~Qj+_CVjefLi&oz8L6biEj-Kc)BAFD|jTACf{pmIMNEDVDt
zT^n*v9KcWhN(9S3b87l-5brTRsrOeWx?(7zSRZpvqZ22h@66pGKg^8cH;SkyG9I3f
zGo;uu8JfKbAS2h2w$#bd{ytZ5K4wK1F3Hh`0$=EPV#4t|JOK`=8jy5~JOw@rhl9V^
z)Aa)iv}d^&xc+3x@n0m{fW5UT*YV058!#_6q`Go>`t6w<)M*alL4&`raPlL`+bnsy
zc(4mQz8(Wv1*6!V?-JJR9|cK2D^vfLGkE5l8oRrzGM}RjH%B=D1iZ&H@u#py_czI@
z35vAq>=ziAkih8&HXOjyKTL$s<?7_8e-LM_j|BS>1E_fK6jolbg!(}R5}qA5Rtb8I
zM$(R-&*B`fvy#>=N~GoVE4KD-ku=YEjbh{P@qD~02(k$>puQsr@9Nll5gi(|KzYAB
z4S#wK-zLjK|4%RRn$iUf>(&IzP+4{sokzvjk0k<)^Tpq=_Y*bPwM~u=>0HM}O$s2`
zn}jeHi&$BV^HwE8^zY2qDQH7cu_;_kc#Jj7-fYo%E-BpgIbO-sr@~BINPs?c9T>*U
z{VqJ7wv_odJ0~+|=dTnN7t*G<Y%?gfkx)ve9Q9t8{A*|Zf7+X#VqLN>B5cuPw#3Go
zo$pfAtI2{5HmcOyJRbYDM1xM}NB?G@bNas~L&_dCy7Pqx9{y1q6bDr)cykDDoHiQn
z?NX(;OR`Y)FEMn?S0%%fOR&6M6HW!IlIsFyZ)gOBE>xq3>o#K3S~HLzRwu>j+puw>
z2dtQ(O}5HSc+S}cTG%-#Keh#p^87&d6}uCCyBDjI*qnc0=N+Rt>7dWfO&hY`yA4Zz
zHi!Ig9jLx{6E+?b!_ybeG|GAnuG2Pv;b1pvKeh}jZLFYXp*yv%%E2~Q33OU{u((GA
zDw*2A3VThi%ePyT!1oI+I`!HSqbif2$Ww>TJhny0iOF0Z0pGAD9*UT~3660`ne1s0
zbk&@?GQH8~ml!y&XwCV2>J$Z*Ds~*cMcG`oLJc1rKP?f=(TT;H2;a#kfb~>kGJPJ3
zvE6Z8$HN+l=zA_4>cmC_p1Ig6lL6(=4M|UaA&O7rLQ$q3Ri-Y+*W;#h+t4$*3}^ou
z%kGbcbXKh#Q#Pi6_grJzadJ7<%`XA(CQ}k<3In293{9KA*X@^VXlc`-8>N$Qt8omR
zRM#h2d%}kS1)Qg_j&~xWAXrt8>M#`7_0Qn*VA&mnmqrwGeT8{T5T!woycMVUZAl&!
zuD0R2ys?|Zbw4p2i4Km15aeM)tx9p|m-3PRx?Bvt`Z=5Pj4Woh<Xi;UKG3Bd`W2YY
zY_?$Y9?q`79Ze}9+ipMt&Fhv}xYb}p&W5W{vUw`p`o)BLE>xoCu@q=tWJV8vtHP%C
z7`U+0gwFg~!{QJHd=9_4Q;XfDEQZGHjR4PA58~_i&}ALksLg?k%0kSr8r^zhKz^+*
zwU<<4ZE_4)_v(|f*=kJsO3LoG1~mWI)mUPb3Kxo)KgwX`o*<AlH>EQRtI*_G6vqpE
z9;U@|dI7Gq$$>jF^{K(43YX@w`vKGOj&mRmW=693q%}?1IUSAOrNV%mFC_)Z(`Q*4
zP`J>SjL#0@{F#m%|FpIjt=8l~>rX!H{JM=d+|wZ^(vR8r-?89M1Qb4YV}9@euDGNN
zfhtbaF!yhK<P-vF%WOE_`;#O%qi;bQY~G>tY6y$3I+Obh=F^Zdr`dl=L_TxP;Os;*
za-AYWL6gGZ&UQm`_LQZf11_BB^(%7JUFrdS2G%52m8Vs+#>4dpQxfoBdU?U2`396Q
zPJv9W1hTbuAe~+Xvbkgj&z-F}{s%CD(4S37Z%Cewg&RTrbR&*0UaA28Ca+MwZ5St<
zek^GXm!%D{1Gpvcro`~266t()A1`}rLj2q@EHCgI&iO6~?%Ao+7NhTQ)Qy>Nr%;}r
z>s&(5Q7<Itj>^+pm2c7Qpgv?-DN)9gqj*Bz2BxTwrrz%T_#`qFib_PB&n1OxlJY-f
z$<yf!Hl`Ru{!#^cy7M@Gb>9fKG>#yZ&%eMn&CezB{R3ESeHp#m8YRNBC7M6szPu9>
z{}g63=ATBtCEF#D?FwY5e;mu^ZI{HBKF6-i-%(~&mE`>I{Wz}oBK98(l{8-%Mh)}x
z_{wL!<dnq_Hdmd))9>85e<S2<>dK4R|I<P5sv+$vX~TP4Ea7bUQ(U_G2>vPiSaNv%
z7kHvTkH%)(fpS_ezLbujhcDE?k;T*cnSYbyAcEniDYT2(8=Dk!IBX}OMHzB5`trno
z?X3S#d($^pm)wg9mwTl`>p%9U$1{uB9#uNn;DdM5S={zdPfGhO8P6MK{%bPa*r!Hi
z+McL3UmMQasL|4(5R9=N4TmqQ(mDGq9Jg1W!;4xM<Ju!C96r&w8sB$5lXNaoV{yxk
z7`xvboE<f&X3jSJp~D$&RBDs*@+QQ4&T#e-yUTmD;PjcyHW;yZJ%bCk+rm>@3r^!?
zYzWtnva{@`Z79p`EC)|Hkb8O^zB-@{(S7WDwR;UNou&)drnu3G+sjaQv^i+abEmU+
zb8vL4FF=R~JI5n2wuad%XHBll*4k7SyVD}m!;ZM{ejHG`4wXdMq2>8xE{}j8bczGV
z@5L-0<&N(jPKN7O%xUS*-e_<v8p;%`IUiUP3wp|S9CozK<+7ch=YyV@#Pat}bZvY9
zUhGbUK4u%lVPPnm9m{pRa3%o{uw2eTc8?b1G*!%jdgkkN#4N-)x6|S9dOemGT#UL$
zLP2Mem^Q3hhC}-kApe;GX>={e8ux6@Q>A)2-jB(HcxG<|noVO^Ea;9lfA0@R^C0#s
z9Wr&9gk@jG!*xx4y10hWfO0`EPt4cRt%$|(C+IOBE)*XP#(@8L7Hg^x!s{WWTwh_{
zI+K!t<g7T&tV1*TS_rx*suqL&I6JaUh{A!WB4|&vA>+(ATstp=)1OU=!PzM}oM%>G
zC2Ic^4lBN9XECfq-}XqZ*U8=$_zA{A{0|1?*;j$udYN#Y-8s~@RNyw1aB#k8LVoqD
zFi|HBwltg3xWP4O&(3tmT_)5OwGNkyi})NS#ji)l&6#|?1o&ld2wz8m=H|;HDD=^#
zls(nxemfq@*XdHtOXlC`#e?h<7B6M|?;i|@*v|~8s-Y6SCr^j=ZAR2Og~i!oCPHi}
zyR(O^!Ee5ef-`fB$TzGS>z>3xbb$efUk_zMr$nFb1yx~yMigx6)+OVI0k|h53d&uq
z*}j^N&+cb}c!V#>tH{&pZgr?#>`OtxgE%(F6+-=eX>m<2&SSYHfu^naHg3~Q2Kit8
z2sZtW*DJ%IKiq>lRr*oCP?gzcCu)84H+szofE$B0)P8LMuYVH@wi_+T(&`;b%%Wkq
z!G)CLMI=>G;WVMkWk_RCk;Ny>Sj<3%<Ss?PrCo-k<iYYTi@ZQ|*qP2;W_HER2r|lT
zxSS>3Zcxl}LW0bFUwDDy3${jH3bdvy1TNJ(lDoMgE!t%aS#{P_TBJbhyLF(|#hlZO
zy`v1LJ~8HS@TyKp*Rftaw_AzABXT9?Rl_*{M+M4wSRwI`9YK8tud)8vO_ncLrqDTE
z7+2s5wIym4>VFyiM})9@*gI4i(~Y0!&XxGH+=)tcC;nok1uxB*>_>h>*{KHLFH#_t
z#Gi151wxnhD?D`W2{!DCkdz;Og91;z=pz2!&|NohxB7BFL54%tH}SU}34V0p35M=^
zjY;KB67xmRQRVbwymSAaU)Qy#80!B7zkkdESH6!i^yM=gda3D02OeSQ{kIsiNJ&EC
z9_(B55(@&>_z5~5+WZP{-rUV$L9gj=&3OJsDU$7*rNwyet4ye8@wBtEk7Mp@Gw$Ea
zDr&<_kNcAHb%${{L!T_=odH&K;k{id)bFnj*OmnkWS&52pa^<dJgtn`oA(3eEdDE@
zE6l$slT7;8&ienfH?@E1(#tu7iVmsZ&tmAHEye9I8Ss$Vo2+IZbZ?h}XQ&tT-AqPD
z)0zL83>UAe(Tz?I4EE51B&iyizYD=1eo=vvcg)Ti%*44{#GqKJN}>0cV7rPc7@Db)
zeDiA54<Etlr8XO}e8_}-_cW+3VjIT9J3;IjZHhK(LW>igpeE9%{jM!|K5jgS3ys;m
zbuYg5Z~*%X3r-U&%lu9QTdI_4#L)}wp+Vk}<r3;}|8rv)%66d(GHWpJf(eY;?nYfD
z%W$%c6C`xFGh3B|`4c8WAoFkhaw2f1ts{hcYI0ph<*=B^DVD2LbwYFI3vQUA!{T_h
zxMNQ;mq);#h>e3YN5tg0#2q6GLs)E`<!|a)JgqDSzUi^zd`vFKK)tCQhcEt`#cCxS
zsDFwN{<Sm#q$y6MQ%X3wJC*A-Dkcn<%#DYGPmNeiG69zig}`|U^XHs%vH4gL<eV}j
zMbm}&y?YciT-Bp9cAwyav}hJ97jwPrqBB9G%YZ6=UyhOuMVx2j(&e~uMLM&orquV#
zG7LP%{DY_3)N2)he?Me31s8OvO>+|N%#MRPdwniX=;S=kGdv^&ckGLR6IOaOZeJ))
z{wbEvL;I#69I<c#*H=w960ftn`3-L?zV7!E(xL0D4GFqb4Q7LJh#d_SN8!RbMNopw
z2d#)>xvn%$-`Ek2^q%?OY;LFQti-*&VbIZ`OPAKK#OSNBkme$$9^aMtuptr}j~mby
zW^Y0qS*~l75r^wW1;M4CO(>^f6>6r;f-)9Q6V`f4K^(^`<*h@b;Q~H~9p`Ehixc^J
zot<5a@}e-ljsneEqg+S{Vf7cS)yzII+t#E@t^<`g;g4u|{kc9xvRqM-Q7D%|z&8hm
zLuif>yC1B^*WVJK+k5+~@O(F`8DaPq#cGsY9s+u6ST600HCSXk9Y)3I(?CQOD#auK
zy<_%lc_7ZZ5Cts~YbsBfj_)Ka@1^8Ru4?j>wNo2X-uP11k1U?1<_eule5qgQF{)(6
zaGHZpZ{ufglAtR<Lix<USvxNrijH~Gxvg(-qMI$WGTZPnZ2<EN5rSDPx3qHr_w8nR
zz77j2-|-IjxXgfhTX)h+5Ygc-O~^VV<P>G7pkPRHjoE|vPi1Je>2!E~i^Xrfn155_
z31?Qja@ode>TsSV)8wg1#}YzMGJlB4JU!PRY#j`#e7pihxrM{qVZnD-q=-%WaE{rh
zw}%wy$zCz+xNpvBYSWZJbi$a~P6aAEa!Jyn_yF(3s8UO@w#0DM2Q>5<N%foZCH1Ka
z)Eps0!FfJlDH%-(cV6L_6@f4`U6nc;yYct4sc<K`A3gi}@$F?*nDSzf-TPl)#8G!P
z&+l*m`|xRHAP787T^{4Y@!k+`!RGS)AW9o8;d=2~tWA`oeXd_g4pj`}k<S%r#-t09
zvr|NrHBX5sY`H|d^ge3bkR|QAno#`5Ym8hplD3Mi`MWB998GdF-bh*(yu;$(#!%zv
zE=g9_EiV5rbr$e+XD`~<jG;u`G0^z!GfetZl|BnJ2gT{HP}HJ9o28mOo;E+)h)T1I
zaZ}n%p1+AKK8~FiO&}}!37(qUhJ_6eB#XCyft@VYvn<mRd<(m={;SbscV8J!3<r=a
z^KaA?WkLC63R#cTqT4mr;BF2quOmlq{$Tfo|Al`E&veN<lF;V&R5)c|L%oVpJoz=d
zM>Bh)SSG;(TSDPbm>0{FC!@mcf6u>BV)jO4;)$&jG@;=WHF`cG1gDv6aJV)k3tf|}
zIb41H6TG%*G&~Gfqi>Q|v$OsWiL`~m&o*G-WqlY|sX=FdYrt4J3n+P|&2(wPtJ{1*
zI$obf&D@JRW4uB9g)ud%?!_e^yuk0R1zjB4jqa<=K@7HJe|;O?Y%l`XSVsyP+Jxcz
zjNp8o3l*KN!i9OpFsj3iiWe-yX)!LKDC0pL$FuSL-0@Hx>%r=>rlbE8M>v<R$#wCx
zO$6t^n18d*9#{2Ghg+ZMFn`w;y}z0P4a)j7<(?hB9hnTb$bikWD~gzZlQGSLGH#5+
z&}TvLw9=ZzHa)O?c{1n}*>Tu)Q~@k<a3mdXZ>+XThOT2y?A#}Oa59SPr4tv7JHHNx
zLovo|erDjO%hSMkF+0Dqa`4RgcsOFoe4Ui}SZMoEh8v}e&~g*2^{Nwdy}lgEhR%70
zERMV!PkkQCc?MgTVUxiuFxNEWH0%Acz}|Q)fA75~h0tcNOSNMrqHJd_D6zX!$9%+J
zN~1Z?jCVnpVIIrk^?KyDe;OY6C6doWRd5jYR4?NCDw<A5Erld#F1O+|83{#@p~>na
z+Cp&Bi8QXep-&V}s?UdkuWeZTCmO@U)8KiG74<)g!U;1Mah}SumDtc60Yk2O)DpN7
zr$~jfTTH&sR^S1@6li;8!0NA7qWEew4A>i!R^v+i`U+bYWmDR*e-)-T6vC0e%=lVQ
z)rtq}dnU|}UyJe|SYCw1ZTfDm!z0TwS-jDJ!+Nn{d>sXvqlu}|c}SOn^44JfnkcCG
zQ<nsJ;%8>SJq<DE|J{aoE`xxdzcLl7uCntXt`d9nLm|J%l)v}ZUrBJS#fZcCUCAK&
z$$-NT+p?kLYkk_HQiTeYAza6XPXo}~KA!Ir$2sAspEDQC?I)1Zr}ET#cr-L_pTOdr
zZ}ID=&XAHhfjrkf#vmJ3v-jMW>UQ73Nu#5n`GADgKwih&6GEWx7f%v@`!~958w=Gj
zE-Xeph%Gg&KCQ)$QWV~zyPgzc?^)6!kM}5dBm(?2-085s48<>3hD8SUbj44W#4P@Q
zVX6g(uV^Mgi;6L+Es!Or2wztB;Yt^#%G2(jjG)flj>-q*$>x|lXlym-GE;#B9P5o(
zOk06wl}O=Xi!-}3ji7dxr@8g1EydnbpjMG3cz$ESVRcqVSA5rm!z)I=keqMp!JwyO
z=+B=QNqjep=tT5Lx_LTJa%PSq)kr@e6}Um^)X~&Z_YzP0M6mdq8a3YO#wmp<;Ht!Q
zW%yNDUXW1p7B9VgjQRTMU}`dq0?pP#>C8@uSj}M<tI14)oZzRp<<Q?a)nyX2uNc51
zA;b8W<6n|n79Y@kksQgsQ3OE-(K30;9y%#;{`?^pyU5U9aEBF7USb`qH~2i%4*Hbe
zVs*y|8o5UdN4CAk=bJ~ec$_wztGbQ#m5OwCyBiFhe}XG!j3OI*JGhen3Im&!$xY<K
zpFP;BO23YE;Pyro%j%$*z3KmKCiwknKo^RRW8-l{=!$xRCB<!cd}NQrv+)b;=+UE{
zZ!91vuNzM}jG>5;qoI(+(+cMt$9|>vlG5x{N~qDKN7~jPHl9dB_vC2)E0_Njdvl^s
zmxe0+aoy`wn6=ZI2HeBZS3emNtW{b3!3SHKBVmZ;Z(6z&v13Q-zb3=b88ymU>4~Wk
zI^aK1jn!#{pj@9ehtJq$q0I|ZSdptn8s>}9JyM;+N5-tiXXE~q2+x*wZos`tS}cdA
zK^Jy5U^R<dnqSeTKIPpgM}90GrB8}Qd-3TVPiQVSCcPh;@!C9Vpc@vPW?q*icx$mb
zsL*XJ&SMEJ{SFlAz6lp>7DGP!?(I0a8gJUMxOS2oo&S9)!ah4zujNiFs<Lr5PJpy^
z9+cHR9XmqIp(aR^VwtZta#b80>d+$DTt`&=GYb4;btvty4XXoA<MLRmJD|*+IGB}f
zKpPggVtr{Kt8uU(|GQpzs6Gs?r(08%g9i>6<iS0EI}U5#E8wyXZ1BeN8%Z!~uM-8f
zA<tp<HX8GJvksXC#Z!!F%b|GO)t3zZvyHf%KRii-iatXsp0xn)UXO)qcUhfo;X<4&
zN{82(Vy>6Tv23`QXGqtkEJq#nY!G;!+ggU{i{ik0$duDunwtgD!`l44%A1Oz*hQCG
zTL2TjNQK~W`mE*+S)E=o=h-no2!}_<LM^j5w=AZyJrl?0;mprdQRm7hT;Evl={V<Y
zJUDw=(f&)*SZpvA+E3eXT{?Dt)LrGT2)w0K%<%)SqA;T_o6{TCM`E$_Tv$=AL)VY4
zLVwFZ=wUV4BGF2e_OaS%A2F5gTY;;};=%lh0o{nJz_(YU`23W}RUnEZ;o5)+g(t4U
zJ<hQ#rf*IST5IsEK`_*^I9p0d4L;~k=W`fpwhn7AeOxaAJ|-37och#hRD-*H5}|jc
zF3SN_<4%^NO1Y&=exoWeV?B$HF&PB@jf<n;&bI~>e7q7>>!-ue3nLD@s04!IB~!9i
zufpH@V&TM(MjYOCH53{)8gTggKT}u@n3%jXR^u&7<T^&CGryMkH^M$CbPL16C5u?T
z!H?b=DbUs~Wl&7@qrkd%td`v!tiPN<ook=4c=!}(`O%loiSJ_Kchf-L3)sE(I*vJz
zz;Z>NY<&mtUYa^IN4U_GGlLj$gy7;;JLVI=!%?ouV1B`p((b;;)cY}Tc7ZFYl*rKJ
z@p{Zo*ioN_EUlfd17}^>cZ$J^rxGB(!I%<MSsmIOf6g=hfjqhP*?~%lEtfMRZUT#c
zGJlBSC*Jgf;Q2<>`KtoWcL)VNVzn@@6zPwX4iK(wLnoFflH4a^u<SQydrXm*mg_^k
zx(Vr#BBf~zOR7%wq8jrLMr>}Ah;(IGEwUo%9$789<uQVas)z7(g#@Nlj;2|co?*~m
zaS+RDl!o*kU<->smyUdk0^aJUiJ+l5guPymai}m6#M_5azPTIq*5|^wI2ji2>B1Q)
z$>1pMLycGc*fbmn1G_(98jHdCE!GA3Bv~5A&gVJq1}t78OZEE{$n&?G60JksoTedn
zJe-mrL;>ISl`D(W4r9*i5p*hUG*q%Wv}We_)LV2*<m>LCApcXh@%&x#!z1aOtrZ0J
zzrkx$l&QDT50dKNV=(jQG>?tv_NGa&H<d-$DmxR(q72Ei;5d3`7()fChc2AchW^KT
zC8jK%_Lk*ue*Mvm)#Na|&uHp@{zmeY)u)}yJ&teNM4-d$jdYzRX*5|u<kpFFD?y%m
zt^arI%`KL{$zb(PW5Scb*plU|ox?FFCl#ics?u=lc$_Rh1DvmUQn@G|170QkYxX9#
zRE_NePrSB48}k2Vey&p}y00DyXUx>7a#A)%{9+06p=^KjE<xY-Y8)Onbv3F^ekM8C
zB-E#E#CTg1=(w&yRTH*h!5J@BU#QJ;6;1fA*B7#kSPcE-Ufi|R3(8j+Q(gLAY@cic
z!RIU}{QWN6=x)M(7iCNG-3{o}r4K58j-2PJ(^{Z6*M)?4{as=J8^YaKUH=li5$X&b
zOWj%ia3*H2od7D19(3L-oaL@9K|@ZH?e%dudtx%HF=M}*I^u{5V^ct0qQm5IK<n?5
zK;spwmt**-Tj_A|69dXIVEG%L1W?;z&VFAq4u4^F(&7#)_PZ8Ow0;{7Xa2P1aB~Cu
z&VTJdZSLN<Xn@svvHZ=Bcx1VaRIXQLSqOf=GK}Srjp;~YJT6F&gN9Fy_#8g&h=v`q
zM(p<l3-HaG1U@G!jtkMKFq+i@i22-J`J4Slht*Y<Tq(m9iL=>nR*dQ5CuOXDBounP
zO*zfr@^~=x8O!2_lhG}T`7Ze^KhI>idA)@5gk!|dqlXK`WN#geV?#q(Ew~>0Jxwt7
z*JlAe(IH>sAiUbgYQ1J#klurEoOxUd7fh{4kpEU<A^5W#wXg^7jLQNoMLU-7jX=vs
zg%CJw&Fb1C@!appoL;~$8JEvx=-J9@oxh6X@2d5y!1X_;fFMI}_exYg6a^{lJE*s@
z0)JDM^7#?qmGf9#^>GuPYtfjQ3AKJ^bZGAy_WO)j_FEhiI+##{{T;ci9*=#ON3KI9
zvoOA10{jkUK=TECPLtsm2TS$!$a8oNcA7<T8HD^zb!rx?{S|Zm3%$}=+`@qGrRqsD
z!2Je`b8cUaI~^mq3~jrsaKa#Qd{+Nz>|GWNC07hMO?Pi3=OgSx_wUo;N&xd0+5ES^
zlCro8%WYJKVL0<a7v)W$9cl9PYt%c*s2D%W`Q#mb>SPPzy%Xrf@MDz85p<tG{-V2>
zn-T-1FC^6B^#?w*iDloNaa28d0B0XI2KfS4sxo+sP2&M<TkWYgYlziLg}^NjD^i*;
zjGro}fM|girOlF|37y*P9_CD!WaX%Y<zJJ6EIBOcn9TB6#&r0I9Cb7~f~&44byO%&
zLWw1dqdHLG_X;dVqzx+9EI9tqc7ND#*NEj;6iK=z46a;sqx{KAG~)6YXlZm{=iUfv
z3s!*!vZ3LFinRQ`CKT3NaC&>KUWxmLUQ}eiZHdopmK?k)B8_Py=<UliNumA-3TAOG
zH=ha6d3!XSd-no6P5q(GT%E-Xx>06U3cTF)76rWAQGc+s8pf12kFh&G1s1V7yPB6h
zcyN3oXhqAC|Lli|7o>1)%QJkq>~Bn&2OzwweFdw*v(SQy-(^|cQi1MF*MTeAvb6ua
z0&UUzT@t$f5taqWP+2m7@~%NlTCGSs|6uVw_O64<Bk8fJB2@4FfVE>rQGsT+BuM!_
uUK%xm?tbF~9h>`5e(fk~tQW%xqkh~FK8EVs#zFg@_jr%xw@(<^!v6u{Io$jJ

literal 0
HcmV?d00001


From dc77e527cd29d7d63980e1cf45491d2eadde7dc1 Mon Sep 17 00:00:00 2001
From: Amit Moryossef <amitmoryossef@gmail.com>
Date: Wed, 18 Jan 2023 13:28:31 +0100
Subject: [PATCH 7/8] fix(holistic-worker): correct merge issues

---
 .../components/text-to-speech/text-to-speech.component.ts  | 7 +------
 src/app/modules/detector/detector.service.ts               | 2 ++
 src/app/modules/pose/pose.actions.ts                       | 6 ++++++
 src/app/modules/pose/pose.service.ts                       | 4 ++--
 src/app/modules/pose/pose.state.ts                         | 5 ++---
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/app/components/text-to-speech/text-to-speech.component.ts b/src/app/components/text-to-speech/text-to-speech.component.ts
index b1bf5715..c002ffee 100644
--- a/src/app/components/text-to-speech/text-to-speech.component.ts
+++ b/src/app/components/text-to-speech/text-to-speech.component.ts
@@ -14,12 +14,7 @@ export class TextToSpeechComponent implements OnInit, OnDestroy, OnChanges {
   isSpeaking = false;
 
   // SpeechSynthesisUtterance is not supported on Android native build
-  speech: SpeechSynthesisUtterance = 'SpeechSynthesisUtterance';
-  in;
-  globalThis?;
-  new;
-
-  SpeechSynthesisUtterance(): null;
+  speech: SpeechSynthesisUtterance = 'SpeechSynthesisUtterance' in globalThis ? new SpeechSynthesisUtterance() : null;
 
   private listeners: {[key: string]: EventListener} = {};
 
diff --git a/src/app/modules/detector/detector.service.ts b/src/app/modules/detector/detector.service.ts
index 3890a07c..3e0d69ae 100644
--- a/src/app/modules/detector/detector.service.ts
+++ b/src/app/modules/detector/detector.service.ts
@@ -95,6 +95,8 @@ export class DetectorService {
       EMPTY_LANDMARK,
       EMPTY_LANDMARK,
     ];
+
+    return newFakePose;
   }
 
   isValidLandmark(l: PoseLandmark): boolean {
diff --git a/src/app/modules/pose/pose.actions.ts b/src/app/modules/pose/pose.actions.ts
index 43d2de9a..eb46044d 100644
--- a/src/app/modules/pose/pose.actions.ts
+++ b/src/app/modules/pose/pose.actions.ts
@@ -1,5 +1,11 @@
 import {Pose} from './pose.state';
 
+export class LoadPoseModel {
+  static readonly type = '[Pose] Load Pose Model';
+
+  constructor() {}
+}
+
 export class PoseVideoFrame {
   static readonly type = '[Pose] Pose Video Frame';
 
diff --git a/src/app/modules/pose/pose.service.ts b/src/app/modules/pose/pose.service.ts
index 2b5bedee..df071d85 100644
--- a/src/app/modules/pose/pose.service.ts
+++ b/src/app/modules/pose/pose.service.ts
@@ -43,8 +43,8 @@ export class PoseService {
     });
   }
 
-  async predict(video: HTMLVideoElement): Promise<Pose> {
-    const [width, height] = [video.videoWidth, video.videoHeight];
+  async predict(video: HTMLVideoElement | HTMLImageElement): Promise<Pose> {
+    const width = (video as HTMLVideoElement).videoWidth ?? video.width;
     if (!this.worker || width === 0) {
       return null;
     }
diff --git a/src/app/modules/pose/pose.state.ts b/src/app/modules/pose/pose.state.ts
index 621fc9a9..c04dd879 100644
--- a/src/app/modules/pose/pose.state.ts
+++ b/src/app/modules/pose/pose.state.ts
@@ -1,8 +1,7 @@
 import {Injectable} from '@angular/core';
-import {Action, NgxsOnInit, Select, State, StateContext} from '@ngxs/store';
+import {Action, NgxsOnInit, State, StateContext, Store} from '@ngxs/store';
 import {PoseService} from './pose.service';
 import {LoadPoseModel, PoseVideoFrame, StoreFramePose} from './pose.actions';
-import {Observable} from 'rxjs';
 import {filter, first, tap} from 'rxjs/operators';
 
 export interface PoseLandmark {
@@ -38,7 +37,7 @@ const initialState: PoseStateModel = {
   defaults: initialState,
 })
 export class PoseState implements NgxsOnInit {
-  @Select(state => state.settings.pose) poseSetting$: Observable<boolean>;
+  poseSetting$ = this.store.select<boolean>(state => state.settings.pose);
 
   constructor(private poseService: PoseService, private store: Store) {}
 

From 56bfcd2fff82cf5225b9d20941350561009b8957 Mon Sep 17 00:00:00 2001
From: Amit Moryossef <amitmoryossef@gmail.com>
Date: Wed, 18 Jan 2023 15:26:07 +0100
Subject: [PATCH 8/8] fix(holistic-worker): implement
 https://github.com/google/mediapipe/issues/2506

---
 src/app/modules/detector/detector.service.ts  | 10 +--
 src/app/modules/pose/pose.actions.ts          |  2 -
 src/app/modules/pose/pose.service.ts          | 10 ++-
 src/app/modules/pose/pose.state.ts            |  3 +-
 src/app/modules/pose/pose.worker.ts           | 63 ++++++++++++++-----
 .../settings/settings/settings.component.ts   |  1 +
 .../languages/lazy-map/lazy-map.component.ts  |  2 -
 .../pages/playground/playground.component.ts  |  2 +
 8 files changed, 63 insertions(+), 30 deletions(-)

diff --git a/src/app/modules/detector/detector.service.ts b/src/app/modules/detector/detector.service.ts
index 3e0d69ae..a329615c 100644
--- a/src/app/modules/detector/detector.service.ts
+++ b/src/app/modules/detector/detector.service.ts
@@ -22,10 +22,8 @@ export class DetectorService {
   constructor(private tf: TensorflowService) {}
 
   async loadModel() {
-    return this.tf
-      .load()
-      .then(() => this.tf.loadLayersModel('assets/models/sign-detector/model.json'))
-      .then(model => (this.sequentialModel = model as unknown as LayersModel));
+    await this.tf.load();
+    this.sequentialModel = await this.tf.loadLayersModel('assets/models/sign-detector/model.json');
   }
 
   distance(p1: PoseLandmark, p2: PoseLandmark): number {
@@ -68,7 +66,7 @@ export class DetectorService {
       x: (newPose[POSE_LANDMARKS.LEFT_SHOULDER].x + newPose[POSE_LANDMARKS.RIGHT_SHOULDER].x) / 2,
       y: (newPose[POSE_LANDMARKS.LEFT_SHOULDER].y + newPose[POSE_LANDMARKS.RIGHT_SHOULDER].y) / 2,
     };
-    const newFakePose = [
+    return [
       newPose[POSE_LANDMARKS.NOSE],
       neck,
       newPose[POSE_LANDMARKS.RIGHT_SHOULDER],
@@ -95,8 +93,6 @@ export class DetectorService {
       EMPTY_LANDMARK,
       EMPTY_LANDMARK,
     ];
-
-    return newFakePose;
   }
 
   isValidLandmark(l: PoseLandmark): boolean {
diff --git a/src/app/modules/pose/pose.actions.ts b/src/app/modules/pose/pose.actions.ts
index eb46044d..2bb57229 100644
--- a/src/app/modules/pose/pose.actions.ts
+++ b/src/app/modules/pose/pose.actions.ts
@@ -2,8 +2,6 @@ import {Pose} from './pose.state';
 
 export class LoadPoseModel {
   static readonly type = '[Pose] Load Pose Model';
-
-  constructor() {}
 }
 
 export class PoseVideoFrame {
diff --git a/src/app/modules/pose/pose.service.ts b/src/app/modules/pose/pose.service.ts
index df071d85..f14db8bf 100644
--- a/src/app/modules/pose/pose.service.ts
+++ b/src/app/modules/pose/pose.service.ts
@@ -38,7 +38,7 @@ export class PoseService {
     }
 
     await this.ga.trace('pose', 'load', async () => {
-      this.worker = comlink.wrap(new Worker(new URL('./pose.worker', import.meta.url)));
+      this.worker = comlink.wrap(new Worker(new URL('./pose.worker', import.meta.url), {type: 'module'}));
       await this.worker.loadModel();
     });
   }
@@ -59,7 +59,13 @@ export class PoseService {
         return null;
       }
 
-      // result.image = image; // TODO fix
+      // TODO not sure if this is needed
+      // const newImage = document.createElement('canvas');
+      // newImage.width = image.width;
+      // newImage.height = image.height;
+      // const ctx = newImage.getContext('2d');
+      // ctx.drawImage(image as any, 0, 0);
+      // result.image = newImage;
       return result;
     });
   }
diff --git a/src/app/modules/pose/pose.state.ts b/src/app/modules/pose/pose.state.ts
index c04dd879..446a3bed 100644
--- a/src/app/modules/pose/pose.state.ts
+++ b/src/app/modules/pose/pose.state.ts
@@ -52,9 +52,10 @@ export class PoseState implements NgxsOnInit {
   }
 
   @Action(LoadPoseModel)
-  async load({patchState, dispatch}: StateContext<PoseStateModel>): Promise<void> {
+  async load({patchState}: StateContext<PoseStateModel>): Promise<void> {
     patchState({isLoaded: false});
     await this.poseService.load();
+    // isLoaded is set to true once the first frame is processed.
   }
 
   @Action(PoseVideoFrame)
diff --git a/src/app/modules/pose/pose.worker.ts b/src/app/modules/pose/pose.worker.ts
index 1534dae3..7f462e80 100644
--- a/src/app/modules/pose/pose.worker.ts
+++ b/src/app/modules/pose/pose.worker.ts
@@ -2,32 +2,68 @@
 
 import * as comlink from 'comlink';
 import {Holistic} from '@mediapipe/holistic';
-import {Observable} from 'rxjs';
-import {first} from 'rxjs/operators';
+import {firstValueFrom, Observable} from 'rxjs';
 
+// Fake document to satisfy `"ontouchend" in document`
+globalThis.document = {} as any;
+
+const POSE_CONFIG = {
+  // angular.json copies `@mediapipe/holistic` to `assets/mediapipe/holistic`
+  locateFile: file => new URL(`/assets/models/holistic/${file}`, globalThis.location.origin).toString(),
+};
+
+// Solution taken from https://github.com/google/mediapipe/issues/2506#issuecomment-1386616165
+(self as any).createMediapipeSolutionsWasm = POSE_CONFIG;
+(self as any).createMediapipeSolutionsPackedAssets = POSE_CONFIG;
+
+importScripts(
+  '/assets/models/holistic/holistic_solution_packed_assets_loader.js',
+  '/assets/models/holistic/holistic_solution_simd_wasm_bin.js'
+);
+
+// let model: Holistic;
 let model: Holistic;
 let results: Observable<any>;
 
 async function loadModel(): Promise<void> {
-  model = new Holistic({
-    locateFile: file => {
-      const f = new URL(`/assets/models/holistic/${file}`, globalThis.location.origin).toString();
-      console.log('path', f);
-      return f;
-    },
+  model = new Holistic(POSE_CONFIG);
+  model.setOptions({
+    // TODO use our preferred settings:
+    // modelComplexity: 1,
+    // smoothLandmarks: false
+
+    selfieMode: false,
+    modelComplexity: 2,
+    smoothLandmarks: false,
   });
 
-  model.setOptions({modelComplexity: 1});
+  const solution = (model as any).g;
+  const solutionConfig = solution.g;
+  solutionConfig.files = () => []; // disable default import files behavior
   await model.initialize();
-  console.log('Pose model loaded!');
+  solution.D = solution.h.GL.currentContext.GLctx; // set gl ctx
+
+  // load data files
+  const files = solution.F;
+  files['pose_landmark_heavy.tflite'] = (
+    await fetch(POSE_CONFIG.locateFile('pose_landmark_heavy.tflite'))
+  ).arrayBuffer();
+  files['holistic.binarypb'] = (await fetch(POSE_CONFIG.locateFile('holistic.binarypb'))).arrayBuffer();
+  //
+  // // To be on the safe side, we load the files in the order they are listed in the manifest. TODO: remove this
+  // for (const file of ['pose_landmark_lite.tflite', 'pose_landmark_full.tflite', 'pose_landmark_heavy.tflite']) {
+  //   files[file] = fetch(POSE_CONFIG.locateFile(file)).then(res => res.arrayBuffer());
+  // }
 
   results = new Observable(subscriber => {
     model.onResults(results => {
+      console.log(results); // Currently prints: {image: ImageBitmap, multiFaceGeometry: Array(0)}
       subscriber.next({
         faceLandmarks: results.faceLandmarks,
         poseLandmarks: results.poseLandmarks,
         leftHandLandmarks: results.leftHandLandmarks,
         rightHandLandmarks: results.rightHandLandmarks,
+        image: results.image, // TODO reconsider if sending this is expensive
       });
     });
   });
@@ -38,12 +74,7 @@ async function pose(imageBitmap: ImageBitmap | ImageData): Promise<any> {
     return null;
   }
 
-  // TODO remove
-  // const image = new OffscreenCanvas(imageBitmap.width, imageBitmap.height);
-  // const ctx = image.getContext('2d');
-  // ctx.drawImage(imageBitmap, 0, 0);
-
-  const result = results.pipe(first()).toPromise();
+  const result = firstValueFrom(results);
   await model.send({image: imageBitmap as any});
   return result;
 }
diff --git a/src/app/modules/settings/settings/settings.component.ts b/src/app/modules/settings/settings/settings.component.ts
index 15192a4a..da2f1fd8 100644
--- a/src/app/modules/settings/settings/settings.component.ts
+++ b/src/app/modules/settings/settings/settings.component.ts
@@ -13,6 +13,7 @@ export class SettingsComponent extends BaseSettingsComponent implements OnInit {
   availableSettings: Array<keyof SettingsStateModel> = [
     'detectSign',
     'drawVideo',
+    'pose',
     'drawPose',
     'drawSignWriting',
     'animatePose',
diff --git a/src/app/pages/landing/languages/lazy-map/lazy-map.component.ts b/src/app/pages/landing/languages/lazy-map/lazy-map.component.ts
index 5697a79b..ff7d0ae9 100644
--- a/src/app/pages/landing/languages/lazy-map/lazy-map.component.ts
+++ b/src/app/pages/landing/languages/lazy-map/lazy-map.component.ts
@@ -9,8 +9,6 @@ import {ComponentType} from '@angular/cdk/overlay';
 export class LazyMapComponent implements AfterViewInit {
   @ViewChild('mapHost', {read: ViewContainerRef}) mapHost: ViewContainerRef;
 
-  constructor() {}
-
   async ngAfterViewInit() {
     const chunk = await import('../../../../components/map/map.component');
     const component = Object.values(chunk)[0] as ComponentType<unknown>;
diff --git a/src/app/pages/playground/playground.component.ts b/src/app/pages/playground/playground.component.ts
index 880a17bf..21bccf7d 100644
--- a/src/app/pages/playground/playground.component.ts
+++ b/src/app/pages/playground/playground.component.ts
@@ -4,6 +4,7 @@ import {BaseComponent} from '../../components/base/base.component';
 import {filter, takeUntil, tap} from 'rxjs/operators';
 import {SetVideo, StartCamera} from '../../core/modules/ngxs/store/video/video.actions';
 import {TranslocoService} from '@ngneat/transloco';
+import {SetSetting} from '../../modules/settings/settings.actions';
 
 @Component({
   selector: 'app-playground',
@@ -33,6 +34,7 @@ export class PlaygroundComponent extends BaseComponent implements OnInit {
       )
       .subscribe();
 
+    this.store.dispatch(new SetSetting('pose', true));
     this.store.dispatch(new SetVideo('assets/tmp/example-sentence.mp4'));
   }
 }