Real time face detection with Camera on swift 3

Question

How can I do face detection in realtime just as \"Camera\" does? like white round shape around and over the face. I use AVCapturSession . I found that the image I s

Answer 1

Based on this article

Details

• Xcode 10.2.1 (10E1001), Swift 5

Features

This solution allow:

• to check camera access
• to select front or back camera
• if no access to the camera show alert with link to the app settings page
• to highlight face (face rectangle + eyebrows, lips..)

Solution

FaceDetectionService

import UIKit
import AVFoundation
import Vision

class FaceDetectionService: NSObject {

    private weak var previewView: UIView?
    private weak var faceView: FaceView?
    private var cameraIsReadyToUse = false
    private let session = AVCaptureSession()
    private lazy var cameraPosition = AVCaptureDevice.Position.front
    private weak var previewLayer: AVCaptureVideoPreviewLayer?
    private lazy var sequenceHandler = VNSequenceRequestHandler()
    private lazy var dataOutputQueue = DispatchQueue(label: "FaceDetectionService",
                                                     qos: .userInitiated, attributes: [],
                                                     autoreleaseFrequency: .workItem)
    private var preparingCompletionHandler: ((Bool) -> Void)?

    func prepare(previewView: UIView,
                 cameraPosition: AVCaptureDevice.Position,
                 completion: ((Bool) -> Void)?) {
        self.previewView = previewView
        self.preparingCompletionHandler = completion
        self.cameraPosition = cameraPosition
        checkCameraAccess { allowed in
            if allowed { self.setup() }
            completion?(allowed)
            self.preparingCompletionHandler = nil
        }
    }

    private func setup() {
        guard let bounds = previewView?.bounds else { return }
        let faceView = FaceView(frame: bounds)
        previewView?.addSubview(faceView)
        faceView.backgroundColor = .clear
        self.faceView = faceView
        configureCaptureSession()
    }
    func start() { if cameraIsReadyToUse { session.startRunning() } }
    func stop() { session.stopRunning() }
}

extension FaceDetectionService {

    private func askUserForCameraPermission(_ completion:  ((Bool) -> Void)?) {
        AVCaptureDevice.requestAccess(for: AVMediaType.video) { (allowedAccess) -> Void in
            DispatchQueue.main.async { completion?(allowedAccess) }
        }
    }

    private func checkCameraAccess(completion: ((Bool) -> Void)?) {
        askUserForCameraPermission { [weak self] allowed in
            guard let self = self, let completion = completion else { return }
            self.cameraIsReadyToUse = allowed
            if allowed {
                completion(true)
            } else {
                self.showDisabledCameraAlert(completion: completion)
            }
        }
    }

    private func configureCaptureSession() {
        guard let previewView = previewView else { return }
        // Define the capture device we want to use

        guard let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: cameraPosition) else {
            let error = NSError(domain: "", code: 0, userInfo: [NSLocalizedDescriptionKey : "No front camera available"])
            show(error: error)
            return
        }

        // Connect the camera to the capture session input
        do {

            try camera.lockForConfiguration()
            defer { camera.unlockForConfiguration() }

            if camera.isFocusModeSupported(.continuousAutoFocus) {
                camera.focusMode = .continuousAutoFocus
            }

            if camera.isExposureModeSupported(.continuousAutoExposure) {
                camera.exposureMode = .continuousAutoExposure
            }

            let cameraInput = try AVCaptureDeviceInput(device: camera)
            session.addInput(cameraInput)

        } catch {
            show(error: error as NSError)
            return
        }

        // Create the video data output
        let videoOutput = AVCaptureVideoDataOutput()
        videoOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)
        videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]

        // Add the video output to the capture session
        session.addOutput(videoOutput)

        let videoConnection = videoOutput.connection(with: .video)
        videoConnection?.videoOrientation = .portrait

        // Configure the preview layer
        let previewLayer = AVCaptureVideoPreviewLayer(session: session)
        previewLayer.videoGravity = .resizeAspectFill
        previewLayer.frame = previewView.bounds
        previewView.layer.insertSublayer(previewLayer, at: 0)
        self.previewLayer = previewLayer
    }
}

extension FaceDetectionService: AVCaptureVideoDataOutputSampleBufferDelegate {
    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
        let detectFaceRequest = VNDetectFaceLandmarksRequest(completionHandler: detectedFace)
        do {
            try sequenceHandler.perform(
                [detectFaceRequest],
                on: imageBuffer,
                orientation: .leftMirrored)
        } catch { show(error: error as NSError) }
    }
}

extension FaceDetectionService {
    private func detectedFace(request: VNRequest, error: Error?) {
        guard   let previewLayer = previewLayer,
                let results = request.results as? [VNFaceObservation],
                let result = results.first
                else { faceView?.clearAndSetNeedsDisplay(); return }
        faceView?.read(result: result, previewLayer: previewLayer)
    }
}

// Navigation

extension FaceDetectionService {

    private func show(alert: UIAlertController) {
        DispatchQueue.main.async {
            UIApplication.topViewController?.present(alert, animated: true, completion: nil)
        }
    }

    private func showDisabledCameraAlert(completion: ((Bool) -> Void)?) {
        let alertVC = UIAlertController(title: "Enable Camera Access",
                                        message: "Please provide access to your camera",
                                        preferredStyle: .alert)
        alertVC.addAction(UIAlertAction(title: "Go to Settings", style: .default, handler: { action in
            guard   let previewView = self.previewView,
                let settingsUrl = URL(string: UIApplication.openSettingsURLString),
                UIApplication.shared.canOpenURL(settingsUrl) else { return }
            UIApplication.shared.open(settingsUrl) { [weak self] _ in
                guard let self = self else { return }
                self.prepare(previewView: previewView,
                             cameraPosition: self.cameraPosition,
                             completion: self.preparingCompletionHandler)
            }
        }))
        alertVC.addAction(UIAlertAction(title: "Cancel", style: .cancel, handler: { _ in completion?(false) }))
        show(alert: alertVC)
    }

    private func show(error: NSError) {
        let alertVC = UIAlertController(title: "Error", message: error.localizedDescription, preferredStyle: .alert)
        alertVC.addAction(UIAlertAction(title: "Ok", style: .cancel, handler: nil ))
        show(alert: alertVC)
    }
}

FaceView

import UIKit
import Vision
import AVFoundation

struct FaceElement {
    let points: [CGPoint]
    let needToClosePath: Bool

    func draw(in context: CGContext) {
        if points.isEmpty { return }
        context.addLines(between: points)
        if needToClosePath { context.closePath() }
        context.strokePath()
    }
}

class FaceView: UIView {
    private var faceElements = [FaceElement]()
    private var boundingBox = CGRect.zero

    func clearAndSetNeedsDisplay() {
        faceElements = []
        boundingBox = .zero
        DispatchQueue.main.async { [weak self] in self?.setNeedsDisplay() }
    }

    private func drawElement(context: CGContext, points: [CGPoint], needToClosePath: Bool) {
        if !points.isEmpty {
            context.addLines(between: points)
            if needToClosePath { context.closePath() }
            context.strokePath()
        }
    }

    override func draw(_ rect: CGRect) {
        super.draw(rect)
        guard let context = UIGraphicsGetCurrentContext() else { return }
        context.saveGState()
        defer { context.restoreGState()}

        context.addRect(boundingBox)
        UIColor.red.setStroke()
        context.strokePath()

        UIColor.white.setStroke()
        faceElements.forEach { $0.draw(in: context) }
    }

    func read(result: VNFaceObservation, previewLayer: AVCaptureVideoPreviewLayer) {
        defer { DispatchQueue.main.async { [weak self] in self?.setNeedsDisplay() } }

        let rect = result.boundingBox
        let origin = previewLayer.layerPointConverted(fromCaptureDevicePoint: rect.origin)
        let size = previewLayer.layerPointConverted(fromCaptureDevicePoint: rect.size.cgPoint).cgSize
        boundingBox = CGRect(origin: origin, size: size)

        func addFaceElement(from landmark: VNFaceLandmarkRegion2D?, needToClosePath: Bool) {
            guard let normalizedPoints = landmark?.normalizedPoints else { return }
            let points = normalizedPoints.compactMap { point -> CGPoint in
                let absolute = point.absolutePoint(in: result.boundingBox)
                let converted = previewLayer.layerPointConverted(fromCaptureDevicePoint: absolute)
                return converted
            }
            faceElements.append(FaceElement(points: points, needToClosePath: needToClosePath))
        }

        guard let landmarks = result.landmarks else { return }
        faceElements = []
        addFaceElement(from: landmarks.leftEye, needToClosePath: true)
        addFaceElement(from: landmarks.rightEye, needToClosePath: true)
        addFaceElement(from: landmarks.leftEyebrow, needToClosePath: false)
        addFaceElement(from: landmarks.rightEyebrow, needToClosePath: false)
        addFaceElement(from: landmarks.nose, needToClosePath: false)
        addFaceElement(from: landmarks.outerLips, needToClosePath: true)
        addFaceElement(from: landmarks.innerLips, needToClosePath: true)
        addFaceElement(from: landmarks.faceContour, needToClosePath: false)
    }
}

Helpers

import CoreGraphics

func + (left: CGPoint, right: CGPoint) -> CGPoint {
    return CGPoint(x: left.x + right.x, y: left.y + right.y)
}

extension CGSize {
    var cgPoint: CGPoint { return CGPoint(x: width, y: height) }
}

extension CGPoint {
    var cgSize: CGSize { return CGSize(width: x, height: y) }

    func absolutePoint(in rect: CGRect) -> CGPoint {
        return CGPoint(x: x * rect.size.width, y: y * rect.size.height) + rect.origin
    }
}

import UIKit

extension UIApplication {
    private class func topViewController(controller: UIViewController? = UIApplication.shared.keyWindow?.rootViewController) -> UIViewController? {
        if let navigationController = controller as? UINavigationController {
            return topViewController(controller: navigationController.visibleViewController)
        }
        if let tabController = controller as? UITabBarController {
            if let selected = tabController.selectedViewController {
                return topViewController(controller: selected)
            }
        }
        if let presented = controller?.presentedViewController {
            return topViewController(controller: presented)
        }
        return controller
    }

    class var topViewController: UIViewController? { return topViewController() }
}

Usage

private lazy var faceDetectionService = FaceDetectionService()

//....

faceDetectionService.prepare(previewView: previewView, cameraPosition: .front) { [weak self] _ in
    self?.faceDetectionService.start()
}

Full sample

import UIKit

class ViewController: UIViewController {

    private lazy var faceDetectionService = FaceDetectionService()
    private weak var previewView: UIView!

    override func viewDidLoad() {
        super.viewDidLoad()
        // Do any additional setup after loading the view.
        let previewView = UIView(frame: .zero)
        view.addSubview(previewView)
        previewView.translatesAutoresizingMaskIntoConstraints = false
        previewView.topAnchor.constraint(equalTo: view.topAnchor).isActive = true
        previewView.leftAnchor.constraint(equalTo: view.leftAnchor).isActive = true
        previewView.rightAnchor.constraint(equalTo: view.rightAnchor).isActive = true
        previewView.bottomAnchor.constraint(equalTo: view.bottomAnchor).isActive = true
        previewView.layoutIfNeeded()
        self.previewView = previewView

        faceDetectionService.prepare(previewView: previewView, cameraPosition: .front) { [weak self] _ in
            self?.faceDetectionService.start()
        }
    }

    // Ensure that the interface stays locked in Portrait.
    override var supportedInterfaceOrientations: UIInterfaceOrientationMask { return .portrait }

    // Ensure that the interface stays locked in Portrait.
    override var preferredInterfaceOrientationForPresentation: UIInterfaceOrientation { return .portrait }
}

More info

• How to make UIImage using AVCaptureVideoPreviewLayer
• https://developer.apple.com/documentation/vision/tracking_the_user_s_face_in_real_time