Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Optional ScanArea GraphicOverlay and Detection Filtering for OCR #66

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
2 changes: 1 addition & 1 deletion android/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ buildscript {
}

dependencies {
classpath 'com.android.tools.build:gradle:3.6.3'
classpath 'com.android.tools.build:gradle:4.0.1'
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ public class FlutterMobileVisionDelegate
private boolean showText = false;
private int previewWidth = 640;
private int previewHeight = 480;
private int scanAreaWidth = 0;
private int scanAreaHeight = 0;
private int camera = CameraSource.CAMERA_FACING_BACK;
private float fps = 15.0f;

Expand Down Expand Up @@ -188,6 +190,14 @@ private void launchIntent() {
previewHeight = (int) arguments.get("previewHeight");
}

if (arguments.containsKey("scanAreaWidth")) {
scanAreaWidth = (int) arguments.get("scanAreaWidth");
}

if (arguments.containsKey("scanAreaHeight")) {
scanAreaHeight = (int) arguments.get("scanAreaHeight");
}

if (arguments.containsKey("camera")) {
camera = (int) arguments.get("camera");
}
Expand Down Expand Up @@ -226,6 +236,8 @@ private void launchIntent() {
intent.putExtra(AbstractCaptureActivity.SHOW_TEXT, showText);
intent.putExtra(AbstractCaptureActivity.PREVIEW_WIDTH, previewWidth);
intent.putExtra(AbstractCaptureActivity.PREVIEW_HEIGHT, previewHeight);
intent.putExtra(AbstractCaptureActivity.SCAN_AREA_WIDTH, scanAreaWidth);
intent.putExtra(AbstractCaptureActivity.SCAN_AREA_HEIGHT, scanAreaHeight);
intent.putExtra(AbstractCaptureActivity.CAMERA, camera);
intent.putExtra(AbstractCaptureActivity.FPS, fps);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ protected void createCameraSource() throws MobileVisionException {
TextRecognizer textRecognizer = new TextRecognizer.Builder(context)
.build();

OcrTrackerFactory ocrTrackerFactory = new OcrTrackerFactory(graphicOverlay, showText);
OcrTrackerFactory ocrTrackerFactory = new OcrTrackerFactory(graphicOverlay, showText, scanAreaHeight, scanAreaWidth, scanAreaOverlay);

textRecognizer.setProcessor(
new MultiProcessor.Builder<>(ocrTrackerFactory).build());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,22 @@
import com.google.android.gms.vision.Tracker;
import com.google.android.gms.vision.text.TextBlock;

import android.graphics.RectF;

import io.github.edufolly.fluttermobilevision.ui.GraphicOverlay;
import io.github.edufolly.fluttermobilevision.ui.ScanAreaGraphic;


class OcrGraphicTracker extends Tracker<TextBlock> {

private GraphicOverlay<OcrGraphic> overlay;
private OcrGraphic graphic;
private GraphicOverlay<ScanAreaGraphic> scanAreaOverlay;

public OcrGraphicTracker(GraphicOverlay<OcrGraphic> overlay, OcrGraphic graphic) {
public OcrGraphicTracker(GraphicOverlay<OcrGraphic> overlay, OcrGraphic graphic, GraphicOverlay<ScanAreaGraphic> scanAreaOverlay) {
this.overlay = overlay;
this.graphic = graphic;
this.scanAreaOverlay = scanAreaOverlay;
}

@Override
Expand All @@ -23,8 +29,24 @@ public void onNewItem(int id, TextBlock textBlock) {

@Override
public void onUpdate(Detector.Detections<TextBlock> detections, TextBlock textBlock) {
overlay.add(graphic);
graphic.updateItem(textBlock);
// Only one ScanAreaGraphic exists in scanAreaOverlay (if scanArea specified by flutter developer)
ScanAreaGraphic scanAreaGraphic = this.scanAreaOverlay.getBest(0,0);
if (scanAreaGraphic == null) {
overlay.add(graphic);
graphic.updateItem(textBlock);
return;
}
// Translate textBlock to scanAreaOverlay's scale
RectF scaledTextBlockRectF = scanAreaGraphic.translateRect(new RectF(textBlock.getBoundingBox()));
// // Check that detected text fits within scan area in order to add
if (scanAreaGraphic.getBoundingBox().contains(scaledTextBlockRectF)) {
overlay.add(graphic);
graphic.updateItem(textBlock);
}
// If not, remove potential text graphic from overlay (graphic no longer in scan area)
else {
overlay.remove(graphic);
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,28 @@
import com.google.android.gms.vision.text.TextBlock;

import io.github.edufolly.fluttermobilevision.ui.GraphicOverlay;
import io.github.edufolly.fluttermobilevision.ui.ScanAreaGraphic;

public class OcrTrackerFactory implements MultiProcessor.Factory<TextBlock> {
private GraphicOverlay<OcrGraphic> graphicOverlay;
private GraphicOverlay<ScanAreaGraphic> scanAreaOverlay;
private boolean showText;
private int scanAreaHeight;
private int scanAreaWidth;

public OcrTrackerFactory(GraphicOverlay<OcrGraphic> graphicOverlay, boolean showText) {
public OcrTrackerFactory(GraphicOverlay<OcrGraphic> graphicOverlay, boolean showText, int scanAreaHeight, int scanAreaWidth, GraphicOverlay<ScanAreaGraphic> scanAreaOverlay) {
this.graphicOverlay = graphicOverlay;
this.showText = showText;
this.scanAreaHeight = scanAreaHeight;
this.scanAreaWidth = scanAreaWidth;
this.scanAreaOverlay = scanAreaOverlay;
}

@Override
public Tracker<TextBlock> create(TextBlock textBlock) {
OcrGraphic graphic = new OcrGraphic(graphicOverlay, showText);
try {
return new OcrGraphicTracker(graphicOverlay, graphic);
return new OcrGraphicTracker(graphicOverlay, graphic, scanAreaOverlay);
} catch (Exception ex) {
Log.d("OcrTrackerFactory", ex.getMessage(), ex);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.io.IOException;

import io.github.edufolly.fluttermobilevision.util.MobileVisionException;
import io.github.edufolly.fluttermobilevision.R; // for scan area

public class CameraSourcePreview extends ViewGroup {
private static final String TAG = "CameraSourcePreview";
Expand All @@ -39,6 +40,9 @@ public class CameraSourcePreview extends ViewGroup {
private CameraSource cameraSource;

private GraphicOverlay overlay;
private GraphicOverlay scanAreaOverlay;
private int scanAreaHeight;
private int scanAreaWidth;

public CameraSourcePreview(Context context, AttributeSet attrs) {
super(context, attrs);
Expand Down Expand Up @@ -74,6 +78,15 @@ public void start(CameraSource cameraSource, GraphicOverlay overlay)
start(cameraSource);
}

public void start(CameraSource cameraSource, GraphicOverlay overlay, int scanAreaHeight, int scanAreaWidth)
throws IOException, SecurityException, MobileVisionException {

this.overlay = overlay;
this.scanAreaHeight = scanAreaHeight;
this.scanAreaWidth = scanAreaWidth;
start(cameraSource);
}

public void stop() {
if (cameraSource != null) {
cameraSource.stop();
Expand Down Expand Up @@ -103,6 +116,24 @@ private void startIfReady() throws IOException, SecurityException, MobileVisionE
}
overlay.clear();
}
// Set up separate overlay for scan area
this.scanAreaOverlay = findViewById(R.id.scan_area_overlay);
if (this.scanAreaOverlay != null) {
Size size = cameraSource.getPreviewSize();
int min = Math.min(size.getWidth(), size.getHeight());
int max = Math.max(size.getWidth(), size.getHeight());
if (isPortraitMode()) {
// Swap width and height sizes when in portrait, since it will be rotated by
// 90 degrees
scanAreaOverlay.setCameraInfo(min, max, cameraSource.getCameraFacing());
} else {
scanAreaOverlay.setCameraInfo(max, min, cameraSource.getCameraFacing());
}
ScanAreaGraphic scanAreaGraphic = new ScanAreaGraphic(this.scanAreaOverlay, this.scanAreaHeight, this.scanAreaWidth);
if (this.scanAreaHeight != 0 && this.scanAreaWidth != 0) {
scanAreaOverlay.add(scanAreaGraphic);
}
}
startRequested = false;
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Copyright (C) The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.github.edufolly.fluttermobilevision.ui;

import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Paint;
import android.graphics.RectF;
import android.graphics.Point;


import com.google.android.gms.vision.text.Text;
import com.google.android.gms.vision.text.TextBlock;

import java.util.List;

import io.github.edufolly.fluttermobilevision.ui.GraphicOverlay;

/**
* Graphic instance
*/
public class ScanAreaGraphic extends GraphicOverlay.Graphic {

private static final int TEXT_COLOR = Color.WHITE;
private static final Paint rectPaint = new Paint();
private static final Paint textPaint = new Paint();

static {
rectPaint.setColor(TEXT_COLOR);
rectPaint.setStyle(Paint.Style.STROKE);
rectPaint.setStrokeWidth(4.0f);

textPaint.setColor(TEXT_COLOR);
textPaint.setTextSize(50.0f);
}

private int scanAreaHeight;
private int scanAreaWidth;
private RectF rect;

public ScanAreaGraphic(GraphicOverlay overlay, int scanAreaHeight, int scanAreaWidth) {
super(overlay);

this.scanAreaHeight = scanAreaHeight;
this.scanAreaWidth = scanAreaWidth;

postInvalidate();
}

public RectF getBoundingBox() {
return this.rect;
}

/**
* Draws the RectF outlining the scan area
*/
@Override
public void draw(Canvas canvas) {
int canvasW = canvas.getWidth();
int canvasH = canvas.getHeight();
Point centerOfCanvas = new Point(canvasW / 2, canvasH / 2);
int rectH = this.scanAreaHeight;
int rectW = this.scanAreaWidth;
int left = centerOfCanvas.x - (rectW / 2);
int top = centerOfCanvas.y - (rectH / 2);
int right = centerOfCanvas.x + (rectW / 2);
int bottom = centerOfCanvas.y + (rectH / 2);
this.rect = new RectF(left, top, right, bottom);
canvas.drawRect(this.rect, rectPaint);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import io.github.edufolly.fluttermobilevision.ui.CameraSource;
import io.github.edufolly.fluttermobilevision.ui.CameraSourcePreview;
import io.github.edufolly.fluttermobilevision.ui.GraphicOverlay;
import io.github.edufolly.fluttermobilevision.ui.ScanAreaGraphic;

public abstract class AbstractCaptureActivity<T extends GraphicOverlay.Graphic>
extends Activity {
Expand All @@ -31,6 +32,8 @@ public abstract class AbstractCaptureActivity<T extends GraphicOverlay.Graphic>
public static final String SHOW_TEXT = "SHOW_TEXT";
public static final String PREVIEW_WIDTH = "PREVIEW_WIDTH";
public static final String PREVIEW_HEIGHT = "PREVIEW_HEIGHT";
public static final String SCAN_AREA_WIDTH = "SCAN_AREA_WIDTH";
public static final String SCAN_AREA_HEIGHT = "SCAN_AREA_HEIGHT";
public static final String CAMERA = "CAMERA";
public static final String FPS = "FPS";

Expand All @@ -40,6 +43,7 @@ public abstract class AbstractCaptureActivity<T extends GraphicOverlay.Graphic>
protected CameraSource cameraSource;
protected CameraSourcePreview preview;
protected GraphicOverlay<T> graphicOverlay;
protected GraphicOverlay<ScanAreaGraphic> scanAreaOverlay;

protected GestureDetector gestureDetector;

Expand All @@ -50,6 +54,8 @@ public abstract class AbstractCaptureActivity<T extends GraphicOverlay.Graphic>
protected boolean showText;
protected int previewWidth;
protected int previewHeight;
protected int scanAreaWidth;
protected int scanAreaHeight;
protected int camera;
protected float fps;

Expand All @@ -66,6 +72,7 @@ protected void onCreate(Bundle icicle) {

preview = findViewById(R.id.preview);
graphicOverlay = findViewById(R.id.graphic_overlay);
scanAreaOverlay = findViewById(R.id.scan_area_overlay);

autoFocus = getIntent().getBooleanExtra(AUTO_FOCUS, false);
useFlash = getIntent().getBooleanExtra(USE_FLASH, false);
Expand All @@ -74,6 +81,8 @@ protected void onCreate(Bundle icicle) {
showText = getIntent().getBooleanExtra(SHOW_TEXT, false);
previewWidth = getIntent().getIntExtra(PREVIEW_WIDTH, CameraSource.PREVIEW_WIDTH);
previewHeight = getIntent().getIntExtra(PREVIEW_HEIGHT, CameraSource.PREVIEW_HEIGHT);
scanAreaWidth = getIntent().getIntExtra(SCAN_AREA_WIDTH, 0);
scanAreaHeight = getIntent().getIntExtra(SCAN_AREA_HEIGHT, 0);
camera = getIntent().getIntExtra(CAMERA, CameraSource.CAMERA_FACING_BACK);
fps = getIntent().getFloatExtra(FPS, 15.0f);

Expand Down Expand Up @@ -137,7 +146,7 @@ private void startCameraSource() throws SecurityException, MobileVisionException

if (cameraSource != null) {
try {
preview.start(cameraSource, graphicOverlay);
preview.start(cameraSource, graphicOverlay, scanAreaHeight, scanAreaWidth);
} catch (IOException e) {
cameraSource.release();
cameraSource = null;
Expand Down
4 changes: 4 additions & 0 deletions android/src/main/res/layout/capture.xml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@
android:id="@+id/graphic_overlay"
android:layout_width="match_parent"
android:layout_height="match_parent" />
<io.github.edufolly.fluttermobilevision.ui.GraphicOverlay
android:id="@+id/scan_area_overlay"
android:layout_width="match_parent"
android:layout_height="match_parent" />

</io.github.edufolly.fluttermobilevision.ui.CameraSourcePreview>

Expand Down
Loading