android camera2人脸识别流程分析

节后回来刚好比较空，今天总结一下camera的人脸识别流程，老样子拿原生camera2源码来分析，但是这里有一个非常严峻的问题是系统人脸识别依赖的framework接口，在Camera_api2中原生接口是没有回调函数的，APK的流程在高通和MTK平台是显示不出人脸框的，只有展讯平台对此做了支持，所以要使用此功能需要用API1。这里说明下我所说的google原生API接口可以在http://androidxref.com中查询的。

人脸识别是camera的功能所以只有在打开camera之后才能开启人脸识别，所以源码是在onCameraAvailable中的startPreview()函数开启人脸识别。onCameraAvailable我们前面讲到过是camera open之后的回调函数，下面直接上源码

PhotoModule.java

private CameraProxy mCameraDevice;

@Override
public void onCameraAvailable(CameraProxy cameraProxy) {

mCameraDevice = cameraProxy;

.........

startPreview();

onCameraOpened();
}

private void startPreview() {

.........

CameraAgent.CameraStartPreviewCallback startPreviewCallback =
new CameraAgent.CameraStartPreviewCallback() {
@Override
public void onPreviewStarted() {
mFocusManager.onPreviewStarted();
PhotoModule.this.onPreviewStarted();
SessionStatsCollector.instance().previewActive(true);
if (mSnapshotOnIdle) {
mHandler.post(mDoSnapRunnable);
}
}
};
if (GservicesHelper.useCamera2ApiThroughPortabilityLayer(mActivity)) {
mCameraDevice.startPreview();
//这里调用onPreviewStarted
startPreviewCallback.onPreviewStarted();
} else {
mCameraDevice.startPreviewWithCallback(new Handler(Looper.getMainLooper()),
startPreviewCallback);
}
}

private void onPreviewStarted() {
mAppController.onPreviewStarted();
mAppController.setShutterEnabled(true);
setCameraState(IDLE);
//这里startFaceDetection
startFaceDetection();
settingsFirstRun();
}

//这里startFaceDetection()继承自FocusOverlayManager.Listener
@Override
public void startFaceDetection() {
if (mFaceDetectionStarted || mCameraDevice == null) {
return;
}
if (mCameraCapabilities.getMaxNumOfFacesSupported() > 0) {
mFaceDetectionStarted = true;
//这里mUI是PhotoUI.java
mUI.onStartFaceDetection(mDisplayOrientation, isCameraFrontFacing());

/*这里2个参数分别是Handler和CameraFaceDetectionCallback，这里我们着重讲解camera_api2,而在API2中setFaceDetectionCallback啥事都不干。
所以你将得不到回调，在UI中也无法显示人脸框，所以这里我们用的API1也就是AndroidCameraAgentImpl.java。*/
mCameraDevice.setFaceDetectionCallback(mHandler, mUI);
//这里调用的是CameraAgent.java的startFaceDetection,上一句设置人脸识别的callback函数这句启动人脸识别，这两句的framework流程后面再分析。
mCameraDevice.startFaceDetection();
SessionStatsCollector.instance().faceScanActive(true);
}
}

上面讲到mUI是PhotoUI，下面来看下PhotoUI.java

public class PhotoUI implements PreviewStatusListener,
CameraAgent.CameraFaceDetectionCallback, PreviewStatusListener.PreviewAreaChangedListener {

//这里FaceView就是显示人脸识别的那个框
private final FaceView mFaceView;

public void onStartFaceDetection(int orientation, boolean mirror) {
if (mFaceView != null) {
mFaceView.clear();
mFaceView.setVisibility(View.VISIBLE);
mFaceView.setDisplayOrientation(orientation);
mFaceView.setMirror(mirror);
mFaceView.resume();
}
}

//这里是CameraAgent.CameraFaceDetectionCallback的回调函数，当检测出人脸时会调用
@Override
public void onFaceDetection(Face[] faces, CameraAgent.CameraProxy camera) {
if (mFaceView != null) {
//最后通过这里来显示人脸框
mFaceView.setFaces(faces);
}
}
}

这里附上FaceView的完整代码，FaceView是放在camera的主layout中的

<com.android.camera.ui.FaceView
android:id="@+id/face_view"
android:layout_width="match_parent"
android:layout_height="match_parent" />

public class FaceView extends View
implements FocusIndicator, Rotatable, PreviewStatusListener.PreviewAreaChangedListener {
private static final Log.Tag TAG = new Log.Tag("FaceView");
private final boolean LOGV = false;
// The value for android.hardware.Camera.setDisplayOrientation.
private int mDisplayOrientation;
// The orientation compensation for the face indicator to make it look
// correctly in all device orientations. Ex: if the value is 90, the
// indicator should be rotated 90 degrees counter-clockwise.
private int mOrientation;
private boolean mMirror;
private boolean mPause;
private Matrix mMatrix = new Matrix();
private RectF mRect = new RectF();
// As face detection can be flaky, we add a layer of filtering on top of it
// to avoid rapid changes in state (eg, flickering between has faces and
// not having faces)
private Face[] mFaces;
private Face[] mPendingFaces;
private int mColor;
private Paint mPaint;
private volatile boolean mBlocked;

private static final int MSG_SWITCH_FACES = 1;
private static final int SWITCH_DELAY = 70;
private boolean mStateSwitchPending = false;
private Handler mHandler = new Handler() {
@Override
public void handleMessage(Message msg) {
switch (msg.what) {
case MSG_SWITCH_FACES:
mStateSwitchPending = false;
mFaces = mPendingFaces;
invalidate();
break;
}
}
};
private final RectF mPreviewArea = new RectF();

public FaceView(Context context, AttributeSet attrs) {
super(context, attrs);
Resources res = getResources();
mColor = res.getColor(R.color.face_detect_start);
mPaint = new Paint();
mPaint.setAntiAlias(true);
mPaint.setStyle(Style.STROKE);
mPaint.setStrokeWidth(res.getDimension(R.dimen.face_circle_stroke));
}

public void setFaces(Face[] faces) {
if (LOGV) {
Log.v(TAG, "Num of faces=" + faces.length);
}
if (mPause) return;
if (mFaces != null) {
if ((faces.length > 0 && mFaces.length == 0)
|| (faces.length == 0 && mFaces.length > 0)) {
mPendingFaces = faces;
if (!mStateSwitchPending) {
mStateSwitchPending = true;
mHandler.sendEmptyMessageDelayed(MSG_SWITCH_FACES, SWITCH_DELAY);
}
return;
}
}
if (mStateSwitchPending) {
mStateSwitchPending = false;
mHandler.removeMessages(MSG_SWITCH_FACES);
}
mFaces = faces;
invalidate();
}

public void setDisplayOrientation(int orientation) {
mDisplayOrientation = orientation;
if (LOGV) {
Log.v(TAG, "mDisplayOrientation=" + orientation);
}
}

@Override
public void setOrientation(int orientation, boolean animation) {
mOrientation = orientation;
invalidate();
}

public void setMirror(boolean mirror) {
mMirror = mirror;
if (LOGV) {
Log.v(TAG, "mMirror=" + mirror);
}
}

public boolean faceExists() {
return (mFaces != null && mFaces.length > 0);
}

@Override
public void showStart() {
invalidate();
}

// Ignore the parameter. No autofocus animation for face detection.
@Override
public void showSuccess(boolean timeout) {
invalidate();
}

// Ignore the parameter. No autofocus animation for face detection.
@Override
public void showFail(boolean timeout) {
invalidate();
}

@Override
public void clear() {
// Face indicator is displayed during preview. Do not clear the
// drawable.
mFaces = null;
invalidate();
}

public void pause() {
mPause = true;
}

public void resume() {
mPause = false;
}

public void setBlockDraw(boolean block) {
mBlocked = block;
}

@Override
protected void onDraw(Canvas canvas) {
if (!mBlocked && (mFaces != null) && (mFaces.length > 0)) {
int rw, rh;
rw = (int) mPreviewArea.width();
rh = (int) mPreviewArea.height();
// Prepare the matrix.
if (((rh > rw) && ((mDisplayOrientation == 0) || (mDisplayOrientation == 180)))
|| ((rw > rh) && ((mDisplayOrientation == 90) || (mDisplayOrientation == 270)))) {
int temp = rw;
rw = rh;
rh = temp;
}
CameraUtil.prepareMatrix(mMatrix, mMirror, mDisplayOrientation, rw, rh);
// Focus indicator is directional. Rotate the matrix and the canvas
// so it looks correctly in all orientations.
canvas.save();
mMatrix.postRotate(mOrientation); // postRotate is clockwise
canvas.rotate(-mOrientation); // rotate is counter-clockwise (for canvas)
for (int i = 0; i < mFaces.length; i++) {
// Filter out false positives.
if (mFaces[i].score < 50) continue;

// Transform the coordinates.
mRect.set(mFaces[i].rect);
if (LOGV) {
CameraUtil.dumpRect(mRect, "Original rect");
}
mMatrix.mapRect(mRect);
if (LOGV) {
CameraUtil.dumpRect(mRect, "Transformed rect");
}
mPaint.setColor(mColor);
mRect.offset(mPreviewArea.left, mPreviewArea.top);
canvas.drawRect(mRect, mPaint);
}
canvas.restore();
}
super.onDraw(canvas);
}

@Override
public void onPreviewAreaChanged(RectF previewArea) {
mPreviewArea.set(previewArea);
}
}

UI层面就是这样，可以看到在每次setFaces时会发送MSG_SWITCH_FACES消息来刷新View，具体绘制onDraw中查看。

下面回头看framework中流程，mCameraDevice.setFaceDetectionCallback和mCameraDevice.startFaceDetection

CameraAgent.java

/**
* Starts the face detection.
*/
public void startFaceDetection() {
getDispatchThread().runJob(new Runnable() {
@Override
public void run() {
getCameraHandler().sendEmptyMessage(CameraActions.START_FACE_DETECTION);
}});
}

/**
* Stops the face detection.
*/
public void stopFaceDetection() {
getDispatchThread().runJob(new Runnable() {
@Override
public void run() {
getCameraHandler().sendEmptyMessage(CameraActions.STOP_FACE_DETECTION);
}});
}

AndroidCameraAgentImpl.java

@Override
public void setFaceDetectionCallback(final Handler handler,
final CameraFaceDetectionCallback cb) {
mDispatchThread.runJob(new Runnable() {
@Override
public void run() {
//创建FaceDetectionCallbackForward并发送handler
mCameraHandler.obtainMessage(CameraActions.SET_FACE_DETECTION_LISTENER,
FaceDetectionCallbackForward
.getNewInstance(handler, AndroidCameraProxyImpl.this, cb))
.sendToTarget();
}
});
}

//上面setFaceDetectionCallback内创建的FaceDetectionCallbackForward源码如下
private static class FaceDetectionCallbackForward implements FaceDetectionListener {
private final Handler mHandler;
private final CameraFaceDetectionCallback mCallback;
private final CameraProxy mCamera;

/**
* Returns a new instance of {@link FaceDetectionCallbackForward}.
*
* @param handler The handler in which the callback will be invoked in.
* @param camera  The {@link CameraProxy} which the callback is from.
* @param cb      The callback to be invoked.
* @return        The instance of the {@link FaceDetectionCallbackForward},
*                or null if any parameter is null.
*/
public static FaceDetectionCallbackForward getNewInstance(
Handler handler, CameraProxy camera, CameraFaceDetectionCallback cb) {
if (handler == null || camera == null || cb == null) {
return null;
}
return new FaceDetectionCallbackForward(handler, camera, cb);
}

private FaceDetectionCallbackForward(
Handler h, CameraProxy camera, CameraFaceDetectionCallback cb) {
mHandler = h;
mCamera = camera;
mCallback = cb;
}

@Override
public void onFaceDetection(
final Camera.Face[] faces, Camera camera) {
mHandler.post(new Runnable() {
@Override
public void run() {
//这里调用PhotoUI内的onFaceDetection，来绘制人脸识别框
mCallback.onFaceDetection(faces, mCamera);
}
});
}
}

public void handleMessage(final Message msg) {
.........
case CameraActions.SET_FACE_DETECTION_LISTENER: {
setFaceDetectionListener((FaceDetectionListener) msg.obj);
break;
}

case CameraActions.START_FACE_DETECTION: {
startFaceDetection();
break;
}

.........
}

//这里mCamera是Camera.java对象
private void startFaceDetection() {
mCamera.startFaceDetection();
}

private void setFaceDetectionListener(FaceDetectionListener listener) {
mCamera.setFaceDetectionListener(listener);
}

下面查看Camera.java

public final void setFaceDetectionListener(FaceDetectionListener listener)
{
mFaceListener = listener;
}

//当收到底层上传的CAMERA_MSG_PREVIEW_METADATA消息时会触发mFaceListener
case CAMERA_MSG_PREVIEW_METADATA:
if (mFaceListener != null) {
mFaceListener.onFaceDetection((Face[])msg.obj, mCamera);
}
return;

public final void startFaceDetection() {
if (mFaceDetectionRunning) {
throw new RuntimeException("Face detection is already running");
}
//调用JNI层接口启动此功能
_startFaceDetection(CAMERA_FACE_DETECTION_HW);
mFaceDetectionRunning = true;
}

到这里JAVA层流程梳理完毕。自己写的简易相机也可已用这个流程来做人脸识别。