Skip to content

myhr-facescan
myhr-facescan
Commit ee7a9a2a by Chokmongkhon Jansanom
Options

fix: match face.

parent be99183c
Showing with 256 additions and 95 deletions
README.md
......@@ -34,7 +34,7 @@ ActiveLivenessVerifier(
|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------|---------------|
| licenseKey | Luxand license key | None |
| imageStreamMilliSecond | Capture image every this value and send image to luxand | 200 ms |
| failAcceptableInSecond | Widget will finish and return false if luxand cannot detect face in picture in 1 / imageStreamInMillisecond / 1000 x failAcceptableInSecond | 3 s |
| failAcceptableInSecond | Widget will finish and return false if luxand cannot detect face in picture in 1 / imageStreamInMillisecond / 1000 x failAcceptableInSecond | 10 s |
### Matching face template
:warning: this widget can match template but not yet send data to user (in development).
......
android/build.gradle
......@@ -69,4 +69,5 @@ android {
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3")
implementation("com.squareup.okhttp3:okhttp:4.10.0")
}
\ No newline at end of file
android/src/main/kotlin/myhr/facescan/myhr_facescan/LuxandSdk.java
......@@ -5,7 +5,6 @@ import android.util.Log;
import com.luxand.FSDK;
import com.luxand.FSDK.HImage;
import java.io.InvalidObjectException;
import java.util.ArrayList;
import java.util.List;
......@@ -16,6 +15,7 @@ import myhr.facescan.myhr_facescan.models.FaceTemplate;
public class LuxandSdk {
static FSDK.HTracker tracker;
static final int CAMERA_ID = 0;
static final float MATCH_CONF = 0.9f;
static ELuxandMode mode = ELuxandMode.Detection;
static LowPassFilter pan = new LowPassFilter();
......@@ -43,8 +43,8 @@ public class LuxandSdk {
"DetectFacialFeatures=true;" +
"HandleArbitraryRotations=false;" +
"DetermineFaceRotationAngle=false;" +
"InternalResizeWidth=100;" +
"FaceDetectionThreshold=5;" +
"InternalResizeWidth=384;" +
"FaceDetectionThreshold=1;" +
"DetectGender=false;" +
"DetectAge=false;" +
"DetectExpression=false;" +
......@@ -231,6 +231,10 @@ public class LuxandSdk {
continue;
}
boolean isMatch = false;
String faceId = "UNKNOWN";
float[] similarityFace = new float[1];
for (int j = 0; j < templates.size(); j++) {
FSDK.FSDK_FaceTemplate t = new FSDK.FSDK_FaceTemplate();
// t.template = availableTemplates.get(j).EyeTemplate;
......@@ -242,21 +246,32 @@ public class LuxandSdk {
// continue;
// }
t.template = templates.get(i).FaceTemplate;
float[] similarityFace = new float[1];
similarityFace = new float[1];
t.template = templates.get(j).FaceTemplate;
libCode = FSDK.MatchFaces(faceTemplate, t, similarityFace);
if (libCode == FSDK.FSDKE_OK) {
result.add(new FaceMatch(
new FacePosition(pos.xc, pos.yc, pos.w),
templates.get(i).Id,
0,
similarityFace[0]));
break;
if (similarityFace[0] >= MATCH_CONF) {
isMatch = true;
faceId = templates.get(j).Id;
break;
}
} else {
Log.e("LUXAND", "Cannot MatchFaces for face: " + libCode);
}
}
if (isMatch) {
result.add(new FaceMatch(
new FacePosition(pos.xc, pos.yc, pos.w),
faceId,
0,
similarityFace[0]));
} else {
result.add(new FaceMatch(
new FacePosition(pos.xc, pos.yc, pos.w),
faceId, 0, similarityFace[0]));
}
}
FSDK.FreeImage(image);
......
android/src/main/kotlin/myhr/facescan/myhr_facescan/MyhrFacescanPlugin.kt
......@@ -20,6 +20,13 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import myhr.facescan.myhr_facescan.models.FaceTemplate
//import okhttp3.MediaType.Companion.toMediaTypeOrNull
//import okhttp3.MultipartBody
//import okhttp3.OkHttpClient
//import okhttp3.Request
//import okhttp3.RequestBody
//import java.net.URL
/** MyhrFacescanPlugin */
class MyhrFacescanPlugin : FlutterPlugin, MethodCallHandler {
/// The MethodChannel that will the communication between Flutter and native Android
......@@ -97,9 +104,15 @@ class MyhrFacescanPlugin : FlutterPlugin, MethodCallHandler {
private fun detectFace(data: Any, result: Result) {
if (detectJob?.isActive == false || detectJob == null) {
detectJob = CoroutineScope(Dispatchers.Main).launch {
val key = data as Map<String, String>
val isFront = key["isFront"] as Boolean
val bytes = withContext(Dispatchers.Default) {
val imageBytes = getImageBytes(data)
val rotateBitmap = rotateImage(byteArrayToBitmap(imageBytes), 270f)
val rotateBitmap = rotateImage(
byteArrayToBitmap(imageBytes), if (isFront) 270f else 90f,
if (isFront) true else false
)
bitmapToByteArray(rotateBitmap, Bitmap.CompressFormat.JPEG, 100)
}
......@@ -115,13 +128,19 @@ class MyhrFacescanPlugin : FlutterPlugin, MethodCallHandler {
private fun detectFaceForMatch(data: Any, result: Result) {
if (matchJob?.isActive == false || matchJob == null) {
matchJob = CoroutineScope(Dispatchers.Main).launch {
val key = data as Map<String, String>
val isFront = key["isFront"] as Boolean
val bytes = withContext(Dispatchers.Default) {
val imageBytes = getImageBytes(data)
val rotateBitmap = rotateImage(byteArrayToBitmap(imageBytes), 270f)
val rotateBitmap = rotateImage(
byteArrayToBitmap(imageBytes),
if (isFront) 270f else 90f,
if (isFront) true else false
)
bitmapToByteArray(rotateBitmap, Bitmap.CompressFormat.JPEG, 100)
}
val key = data as Map<String, String>
val ids = key["ids"] as List<String>
val eyes = key["eyes"] as List<ByteArray>
val faces = key["faces"] as List<ByteArray>
......@@ -148,9 +167,36 @@ class MyhrFacescanPlugin : FlutterPlugin, MethodCallHandler {
}
}
// fun callApi(data: ByteArray) {
// val url = URL("https://mypoint-uat.myhr.co.th/api/File/2") // Replace with your API endpoint
// val client = OkHttpClient()
//
// val builder = MultipartBody.Builder().setType(MultipartBody.FORM)
// val requestBody = RequestBody.create("image/jpeg".toMediaTypeOrNull(), data)
// builder.addFormDataPart("file_field", "filename.jpg", requestBody)
//
// val multipartBody = builder.build()
//
// val request = Request.Builder().url(url).addHeader(
// "Authorization",
// "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJodHRwOi8vc2NoZW1hcy54bWxzb2FwLm9yZy93cy8yMDA1LzA1L2lkZW50aXR5L2NsYWltcy9uYW1laWRlbnRpZmllciI6IjIiLCJodHRwOi8vc2NoZW1hcy54bWxzb2FwLm9yZy93cy8yMDA1LzA1L2lkZW50aXR5L2NsYWltcy9uYW1lIjoi4Lin4Lij4Liy4Lin4Li44LiYIOC5gOC4peC4tOC4qOC4quC4tOC4o-C4tOC4meC4seC4meC4l-C5jCIsImV4cCI6MTcxMTYxMDAwOSwiaXNzIjoiTXlQb2ludCBBZG1pbmlzdHJhdG9yIiwiYXVkIjoiTXlQb2ludCBBZG1pbmlzdHJhdG9yIn0.-vwP-nAcsn9QUkAXVUcQj99YWJed3miU7qIaHmc2UW0"
// ).post(multipartBody).build()
//
// val response = client.newCall(request).execute()
// val responseBody = response.body?.string() ?: "" // P
//
// Log.i("XXXX AA", responseBody)// arse the response body
// }
private fun getFaceTemplate(data: Any): Array<ByteArray> {
val key = data as Map<String, String>
val isFront = key["isFront"] as Boolean
val imageBytes = getImageBytes(data)
val rotateBitmap = rotateImage(byteArrayToBitmap(imageBytes), 270f)
val rotateBitmap = rotateImage(
byteArrayToBitmap(imageBytes), if (isFront) 270f else 90f,
if (isFront) true else false
)
val result = bitmapToByteArray(rotateBitmap, Bitmap.CompressFormat.JPEG, 100)
val templates = LuxandSdk.GetTemplate(result)
......@@ -187,10 +233,13 @@ class MyhrFacescanPlugin : FlutterPlugin, MethodCallHandler {
return BitmapFactory.decodeByteArray(byteArray, 0, byteArray.size)
}
fun rotateImage(bitmap: Bitmap, degrees: Float): Bitmap {
fun rotateImage(bitmap: Bitmap, degrees: Float, flip: Boolean = true): Bitmap {
val matrix = Matrix()
matrix.postRotate(degrees)
matrix.postScale(-1f, 1f, bitmap.width / 2f, bitmap.height / 2f);
if (flip) {
matrix.postScale(-1f, 1f, bitmap.width / 2f, bitmap.height / 2f)
}
val rotatedBitmap = Bitmap.createBitmap(
bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true
......
example/lib/enroll/index.dart
import 'package:flutter/material.dart';
import 'package:get/get.dart';
import 'package:myhr_facescan/enroll/index.dart';
import 'package:myhr_facescan_example/enroll/liveness.dart';
import 'package:myhr_facescan_example/enroll/finish.dart';
import 'package:myhr_facescan_example/env.dart';
class EnrollPage extends StatelessWidget {
......@@ -16,6 +16,7 @@ class EnrollPage extends StatelessWidget {
body: SafeArea(
child: EnrollFace(
licenseKey: Env.luxandLicenseKey,
templateCount: 1,
onFinish: (templates) {
// var con = Get.find<TemplateController>();
......@@ -25,7 +26,9 @@ class EnrollPage extends StatelessWidget {
// Get.snackbar('Success', 'Face & Eye tempalte saved in device.');
// Get.offAll(const HomePage());
Get.off(LivenessPage(templates: templates));
Get.off(FinishPage(templates: templates));
// Get.off(LivenessPage(templates: templates));
},
)),
);
......
lib/active_liveness/index.dart
......@@ -11,7 +11,7 @@ class ActiveLivenessVerifier extends StatelessWidget {
required this.licenseKey,
required this.onFinish,
this.imageStreamMilliSecond = 200,
this.failAcceptableInSecond = 3});
this.failAcceptableInSecond = 10});
final String licenseKey;
final void Function(bool, TemplateModel?) onFinish;
......
lib/enroll/controller.dart
......@@ -48,6 +48,20 @@ class EnrollController extends GetxController {
await _luxand.initialize();
await initCamera();
_settingImageStream();
}
initCamera() async {
camController = await getCameraController();
}
@override
void onClose() async {
super.onClose();
await _clearResource();
}
_settingImageStream() {
camController!.initialize().then((_) {
isCameraReady.value = true;
previewScale = getCameraPreviewScale(camController!.value.aspectRatio);
......@@ -70,16 +84,6 @@ class EnrollController extends GetxController {
});
}
initCamera() async {
camController = await getCameraController();
}
@override
void onClose() async {
super.onClose();
await _clearResource();
}
_clearResource() async {
isCameraReady.value = false;
await camController?.stopImageStream();
......@@ -91,7 +95,8 @@ class EnrollController extends GetxController {
_handleCameraAndroid(CameraImage image) async {
_throttler.run(() async {
if (_gettingCurrentImage) {
var template = await _luxand.getFaceTemplate(image);
var template = await _luxand.getFaceTemplate(
image, camController!.description.lensDirection);
var jpgImage = await convertYUV420toImageColor(image);
if (template.isEmpty) {
......@@ -109,7 +114,8 @@ class EnrollController extends GetxController {
_gettingCurrentImage = false;
} else {
var data = await _luxand.detectFace(image);
var data = await _luxand.detectFace(
image, camController!.description.lensDirection);
if (data == null) {
isFoundFace.value = false;
......@@ -127,4 +133,14 @@ class EnrollController extends GetxController {
_gettingCurrentImage = true;
}
}
switchCamera() async {
await camController!.stopImageStream();
camController = await getCameraController(
direction: camController!.description.lensDirection ==
CameraLensDirection.front
? CameraLensDirection.back
: CameraLensDirection.front);
_settingImageStream();
}
}
lib/enroll/index.dart
......@@ -4,6 +4,7 @@ import 'package:get/get.dart';
import 'package:myhr_facescan/camera_loading.dart';
import 'package:myhr_facescan/enroll/avatar.dart';
import 'package:myhr_facescan/enroll/controller.dart';
import 'package:myhr_facescan/models/face_rect.dart';
import 'package:myhr_facescan/models/template.dart';
import 'package:myhr_facescan/rectangle_paint.dart';
......@@ -45,12 +46,15 @@ class EnrollFace extends StatelessWidget {
con.camController!.value.previewSize!.height,
con.camController!.value.previewSize!.width),
painter: RectanglePainter(
positions: [con.facePosition.value],
faces: [
FaceRect(
pos: con.facePosition.value,
message: topMessage)
],
paperWidth: con
.camController!.value.previewSize!.height,
paperHeight:
con.camController!.value.previewSize!.width,
topMessage: [topMessage]),
paperHeight: con
.camController!.value.previewSize!.width),
)
: Container(),
Positioned.fill(
......@@ -61,6 +65,20 @@ class EnrollFace extends StatelessWidget {
children: [
...con.templates.map((e) => Avatar(image: e.image)),
],
)),
Positioned.fill(
top: 64,
right: 36,
child: Row(
crossAxisAlignment: CrossAxisAlignment.start,
mainAxisAlignment: MainAxisAlignment.end,
children: [
IconButton(
iconSize: 48,
color: Colors.white,
onPressed: con.switchCamera,
icon: const Icon(Icons.switch_camera_rounded))
],
))
],
),
......
lib/luxand_controller.dart
......@@ -37,7 +37,8 @@ class LuxandController extends GetxController {
await platform.invokeMethod<bool>("freeResource");
}
Future<FacePosition?> detectFace(CameraImage image) async {
Future<FacePosition?> detectFace(
CameraImage image, CameraLensDirection direction) async {
List<int> strides = Int32List(image.planes.length * 2);
int index = 0;
final bytes = image.planes.map((plane) {
......@@ -52,7 +53,8 @@ class LuxandController extends GetxController {
'platforms': bytes,
'height': image.height,
'width': image.width,
'strides': strides
'strides': strides,
'isFront': direction == CameraLensDirection.front
});
if (result == null || result[0] <= 0) {
......@@ -62,7 +64,8 @@ class LuxandController extends GetxController {
return FacePosition(x: result[0], y: result[1], width: result[2]);
}
Future<List<Uint8List>> getFaceTemplate(CameraImage image) async {
Future<List<Uint8List>> getFaceTemplate(
CameraImage image, CameraLensDirection direction) async {
List<int> strides = Int32List(image.planes.length * 2);
int index = 0;
final bytes = image.planes.map((plane) {
......@@ -78,7 +81,8 @@ class LuxandController extends GetxController {
'platforms': bytes,
'height': image.height,
'width': image.width,
'strides': strides
'strides': strides,
'isFront': direction == CameraLensDirection.front
});
if (data!['face'] == null) {
......@@ -127,8 +131,8 @@ class LuxandController extends GetxController {
});
}
Future<List<FaceMatch>> detectFaceForMatch(
CameraImage image, List<FaceTemplate> templates) async {
Future<List<FaceMatch>> detectFaceForMatch(CameraImage image,
CameraLensDirection direction, List<FaceTemplate> templates) async {
List<int> strides = Int32List(image.planes.length * 2);
int index = 0;
final bytes = image.planes.map((plane) {
......@@ -152,6 +156,7 @@ class LuxandController extends GetxController {
'ids': ids,
'eyes': eyes,
'faces': faces,
'isFront': direction == CameraLensDirection.front
});
var result = <FaceMatch>[];
......
lib/match_template/controller.dart
......@@ -46,7 +46,20 @@ class MatchTemplateController extends GetxController {
await _luxand.initialize();
// await _luxand.setAvailableTemplate(_templates);
await initCamera();
_settingImageStream();
}
initCamera() async {
camController = await getCameraController();
}
@override
void onClose() async {
super.onClose();
await _clearResource();
}
_settingImageStream() {
camController!.initialize().then((_) {
isCameraReady.value = true;
previewScale = getCameraPreviewScale(camController!.value.aspectRatio);
......@@ -69,16 +82,6 @@ class MatchTemplateController extends GetxController {
});
}
initCamera() async {
camController = await getCameraController();
}
@override
void onClose() async {
super.onClose();
await _clearResource();
}
_clearResource() async {
isCameraReady.value = false;
await camController?.stopImageStream();
......@@ -89,9 +92,20 @@ class MatchTemplateController extends GetxController {
_handleCameraAndroid(CameraImage image) async {
_throttler.run(() async {
var data = await _luxand.detectFaceForMatch(image, _templates);
var data = await _luxand.detectFaceForMatch(
image, camController!.description.lensDirection, _templates);
faceMatchs.value = data;
});
}
switchCamera() async {
await camController!.stopImageStream();
camController = await getCameraController(
direction: camController!.description.lensDirection ==
CameraLensDirection.front
? CameraLensDirection.back
: CameraLensDirection.front);
_settingImageStream();
}
}
lib/match_template/index.dart
......@@ -3,6 +3,7 @@ import 'package:flutter/material.dart';
import 'package:get/get.dart';
import 'package:myhr_facescan/camera_loading.dart';
import 'package:myhr_facescan/match_template/controller.dart';
import 'package:myhr_facescan/models/face_rect.dart';
import 'package:myhr_facescan/models/face_template.dart';
import 'package:myhr_facescan/models/template.dart';
import 'package:myhr_facescan/rectangle_paint.dart';
......@@ -34,18 +35,39 @@ class MatchTemplateScanner extends StatelessWidget {
scale: con.previewScale,
child: CameraPreview(
con.camController!,
child: CustomPaint(
size: Size(con.camController!.value.previewSize!.height,
con.camController!.value.previewSize!.width),
painter: RectanglePainter(
positions: con.faceMatchs.map((d) => d.position).toList(),
paperWidth: con.camController!.value.previewSize!.height,
paperHeight: con.camController!.value.previewSize!.width,
topMessage: con.faceMatchs
.map((d) => '${d.id}, similarity=${d.similarityFace}')
.toList(),
rectColor: Colors.white,
rectStrokeWidth: 2),
child: Stack(
children: [
CustomPaint(
size: Size(con.camController!.value.previewSize!.height,
con.camController!.value.previewSize!.width),
painter: RectanglePainter(
faces: con.faceMatchs
.map((d) => FaceRect(
pos: d.position,
message: d.id,
rectColor:
d.isMatch ? Colors.white : Colors.blue,
rectStrokeWidth: d.isMatch ? 2 : 1))
.toList(),
paperWidth:
con.camController!.value.previewSize!.height,
paperHeight:
con.camController!.value.previewSize!.width),
),
Positioned.fill(
bottom: 18,
left: 36,
child: Row(
crossAxisAlignment: CrossAxisAlignment.end,
children: [
IconButton(
iconSize: 48,
color: Colors.white,
onPressed: con.switchCamera,
icon: const Icon(Icons.switch_camera_rounded))
],
))
],
),
),
)
......
lib/models/face_match.dart
......@@ -6,6 +6,8 @@ class FaceMatch {
final double similarityFace;
final FacePosition position;
get isMatch => id != 'UNKNOWN';
FaceMatch(
{required this.id,
required this.similarityEye,
......
lib/models/face_rect.dart 0 → 100644
import 'dart:ui';
import 'package:myhr_facescan/models/face_position.dart';
class FaceRect {
final FacePosition pos;
final String message;
Color rectColor;
double rectStrokeWidth;
FaceRect(
{required this.pos,
required this.message,
this.rectColor = const Color.fromRGBO(21, 101, 192, 1),
this.rectStrokeWidth = 1});
}
lib/rectangle_paint.dart
import 'dart:ui';
import 'package:flutter/material.dart';
import 'package:myhr_facescan/models/face_position.dart';
import 'package:flutter/widgets.dart';
import 'package:myhr_facescan/models/face_rect.dart';
class RectanglePainter extends CustomPainter {
final List<FacePosition> positions;
final List<FaceRect> faces;
final double paperWidth;
final double paperHeight;
final List<String> topMessage;
Color rectColor;
double rectStrokeWidth;
RectanglePainter(
{super.repaint,
required this.positions,
required this.faces,
required this.paperWidth,
required this.paperHeight,
required this.topMessage,
this.rectColor = const Color.fromRGBO(21, 101, 192, 1),
this.rectStrokeWidth = 1});
required this.paperHeight});
@override
void paint(Canvas canvas, Size size) {
var paint1 = Paint()
..color = rectColor
..style = PaintingStyle.stroke
..strokeWidth = rectStrokeWidth;
const textStyle = TextStyle(
color: Colors.green,
// fontSize: 30,
);
color: Colors.black,
// fontSize: 30,
backgroundColor: Colors.white);
for (var face in faces) {
final xc = lerpDouble(0, size.width, face.pos.x / paperWidth)!;
final yc = lerpDouble(0, size.height, face.pos.y / paperHeight)!;
final width = lerpDouble(0, size.width, face.pos.width / paperWidth)!;
for (var pos in positions) {
final xc = lerpDouble(0, size.width, pos.x / paperWidth)!;
final yc = lerpDouble(0, size.height, pos.y / paperHeight)!;
final width = lerpDouble(0, size.width, pos.width / paperWidth)!;
var paint = Paint()
..color = face.rectColor
..style = PaintingStyle.stroke
..strokeWidth = face.rectStrokeWidth;
canvas.drawRect(
Rect.fromCenter(center: Offset(xc, yc), width: width, height: width),
paint1);
paint);
var textSpan = TextSpan(
text: topMessage[positions.indexOf(pos)],
text: face.message,
style: textStyle,
);
final textPainter = TextPainter(
......@@ -52,7 +46,7 @@ class RectanglePainter extends CustomPainter {
textDirection: TextDirection.ltr,
);
textPainter.layout(
minWidth: 0,
minWidth: width,
maxWidth: size.width,
);
final xCenter = xc - (width / 2);
......@@ -66,7 +60,10 @@ class RectanglePainter extends CustomPainter {
@override
bool? hitTest(Offset position) {
// TODO: implement hitTest
print('positionss ${position.dx},${position.dy}');
for (var face in faces) {}
return super.hitTest(position);
}
}
lib/utils/camera.dart
......@@ -3,10 +3,11 @@ import 'dart:io';
import 'package:camera/camera.dart';
import 'package:get/get.dart';
Future<CameraController> getCameraController() async {
Future<CameraController> getCameraController(
{CameraLensDirection direction = CameraLensDirection.front}) async {
final cameraDescription = (await availableCameras())
.where(
(element) => element.lensDirection == CameraLensDirection.front,
(element) => element.lensDirection == direction,
)
.first;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment