Implementing the app that detects people without a mask with Vision (Apple official framework)

You can easily create an app that analyzes the camera frame of your iPad or iPhone and detects people who are not wearing a mask.
Please put it at the entrance of the shop.

Find human face

Vision

Vision’s ability to detect faces is VNDetectFaceRectanglesRequest.
If you pass a camera frame to this request, it will return whether there is a human face in the image and its position.

Set as follows.

@discardableResult
func setupVision() -> NSError? {
// Setup Vision parts
let error: NSError! = nil
let detectFaceRequest:VNDetectFaceRectanglesRequest = {
let request = VNDetectFaceRectanglesRequest(completionHandler: { (request, error) in
DispatchQueue.main.async(execute: {
// perform all the UI updates on the main queue
if let results = request.results {
self.processVisionRequestResults(results)
}
})
})
request.revision = VNDetectFaceRectanglesRequestRevision2
return request
}()
self.requests = [faceCropRequest]

return error
}

Pass the camera image to Vision for analysis

import UIKit
import Foundation
import Vision
import AVFoundation

extension ViewController: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
if isRequest {
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return
}
currentBuffer = pixelBuffer

let exifOrientation = exifOrientationFromDeviceOrientation()
let imageRequestHandler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: exifOrientation, options: [:])
do {
try imageRequestHandler.perform(self.requests)
} catch {
print(error)
}
isRequest = false
}
}

func setupAVCapture() {
var deviceInput: AVCaptureDeviceInput!

// Select a video device, make an input
let videoDevice = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera], mediaType: .video, position: .front).devices.first
do {
deviceInput = try AVCaptureDeviceInput(device: videoDevice!)
} catch {
print("Could not create video device input: \(error)")
return
}

session.beginConfiguration()
session.sessionPreset = .vga640x480 // Model image size is smaller.

// Add a video input
guard session.canAddInput(deviceInput) else {
print("Could not add video device input to the session")
session.commitConfiguration()
return
}
session.addInput(deviceInput)
if session.canAddOutput(videoDataOutput) {
session.addOutput(videoDataOutput)
// Add a video data output
videoDataOutput.alwaysDiscardsLateVideoFrames = true
videoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)]
videoDataOutput.setSampleBufferDelegate(self, queue: videoDataOutputQueue)
} else {
print("Could not add video data output to the session")
session.commitConfiguration()
return
}
let captureConnection = videoDataOutput.connection(with: .video)
captureConnection?.videoOrientation = .portrait
// Always process the frames
captureConnection?.isEnabled = true
do {
try videoDevice!.lockForConfiguration()
let dimensions = CMVideoFormatDescriptionGetDimensions((videoDevice?.activeFormat.formatDescription)!)
bufferSize.width = CGFloat(dimensions.height)
bufferSize.height = CGFloat(dimensions.width)
videoDevice!.unlockForConfiguration()
} catch {
print(error)
}
session.commitConfiguration()

setupVision()

// start the capture
startCaptureSession()
}

func startCaptureSession() {
session.startRunning()
}

public func exifOrientationFromDeviceOrientation() -> CGImagePropertyOrientation {
let curDeviceOrientation = UIDevice.current.orientation
let exifOrientation: CGImagePropertyOrientation

switch curDeviceOrientation {
case UIDeviceOrientation.portraitUpsideDown: // Device oriented vertically, home button on the top
exifOrientation = .left
case UIDeviceOrientation.landscapeLeft: // Device oriented horizontally, home button on the right
exifOrientation = .upMirrored
case UIDeviceOrientation.landscapeRight: // Device oriented horizontally, home button on the left
exifOrientation = .down
case UIDeviceOrientation.portrait: // Device oriented vertically, home button on the bottom
exifOrientation = .up
default:
exifOrientation = .up
}
return exifOrientation
}

@discardableResult
func setupVision() -> NSError? {
// Setup Vision parts
let error: NSError! = nil

let faceCropRequest:VNDetectFaceRectanglesRequest = {
let request = VNDetectFaceRectanglesRequest(completionHandler: { (request, error) in
DispatchQueue.main.async(execute: {
// perform all the UI updates on the main queue
if let results = request.results {
self.drawVisionRequestResults(results)
}
})
})
request.revision = VNDetectFaceRectanglesRequestRevision2
return request
}()

self.requests = [faceCropRequest]

return error
}

func processVisionRequestResults(_ results: [Any]) {
if currentBuffer != nil {
currentBuffer = nil
guard let observation = results.first as? VNFaceObservation else {
masking()
return
}
noMasking()
}
}
}

AVCaptureVideoDataOutputSampleBufferDelegate processes each camera frame, it passes the image of each frame to Vision’s face detection request.

Request processing is executed in the following part.

func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
if isRequest {
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return
}
currentBuffer = pixelBuffer

let exifOrientation = exifOrientationFromDeviceOrientation()
let imageRequestHandler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: exifOrientation, options: [:])
do {
try imageRequestHandler.perform(self.requests)
} catch {
print(error)
}
isRequest = false
}
}

Process the result (whether there is an unmasked person)

let request = VNDetectFaceRectanglesRequest(completionHandler: { (request, error) in
DispatchQueue.main.async(execute: {
// perform all the UI updates on the main queue
if let results = request.results {
self.processVisionRequestResults(results)
}
})
})
request.revision = VNDetectFaceRectanglesRequestRevision2
return request
}()

If the image contains a face, the request result will have an observation that the image contained a face. This is VNFaceObservation.
If the result has this, then call the no-mask function. Because there is the face.

func processVisionRequestResults(_ results: [Any]) {
if currentBuffer != nil {
currentBuffer = nil
guard let observation = results.first as? VNFaceObservation else {
masking()
return
}
noMasking()
}
}

If someone don’t wear a mask…

else

With AVFoundation’s AVSpeechSynthesizer, you’ll hear a voice saying “Please wear a mask.”

var talker = AVSpeechSynthesizer()
var talking = false

func noMasking() {
if !talking {
let utterance = AVSpeechUtterance(string: NSLocalizedString("Please wear a mask.", comment: "") )
utterance.voice = AVSpeechSynthesisVoice(language: NSLocalizedString("en-US", comment: ""))
talker.speak(utterance)
talking = true
}

By the way, VNDetectFaceObservation also gives us the coordinates of the rectangle surrounding the face.
If you overlap the “not wearing mask” square warning on the camera image, the app will have a more monitoring taste.

Click here for the actual app

Follow me on Twitter. Please.
https://twitter.com/JackdeS11

Please send your work request to this email.
rockyshikoku@gmail.com

Also, clap your hand!

Chao 🐣!

Freelance iOS developer. You can ask me for a job from any country. rockyshikoku@gmail.com