MobileScanner.swift
12.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
//
// SwiftMobileScanner.swift
// mobile_scanner
//
// Created by Julian Steenbakker on 15/02/2022.
//
import Foundation
import AVFoundation
import MLKitVision
import MLKitBarcodeScanning
typealias MobileScannerCallback = ((Array<Barcode>?, Error?, UIImage) -> ())
typealias TorchModeChangeCallback = ((Int?) -> ())
typealias ZoomScaleChangeCallback = ((Double?) -> ())
public class MobileScanner: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate, FlutterTexture {
/// Capture session of the camera
var captureSession: AVCaptureSession!
/// The selected camera
var device: AVCaptureDevice!
/// Barcode scanner for results
var scanner = BarcodeScanner.barcodeScanner()
/// Return image buffer with the Barcode event
var returnImage: Bool = false
/// Default position of camera
var videoPosition: AVCaptureDevice.Position = AVCaptureDevice.Position.back
/// When results are found, this callback will be called
let mobileScannerCallback: MobileScannerCallback
/// When torch mode is changes, this callback will be called
let torchModeChangeCallback: TorchModeChangeCallback
/// When zoom scale is changes, this callback will be called
let zoomScaleChangeCallback: ZoomScaleChangeCallback
/// If provided, the Flutter registry will be used to send the output of the CaptureOutput to a Flutter texture.
private let registry: FlutterTextureRegistry?
/// Image to be sent to the texture
var latestBuffer: CVImageBuffer!
/// Texture id of the camera preview for Flutter
private var textureId: Int64!
var detectionSpeed: DetectionSpeed = DetectionSpeed.noDuplicates
init(registry: FlutterTextureRegistry?, mobileScannerCallback: @escaping MobileScannerCallback, torchModeChangeCallback: @escaping TorchModeChangeCallback, zoomScaleChangeCallback: @escaping ZoomScaleChangeCallback) {
self.registry = registry
self.mobileScannerCallback = mobileScannerCallback
self.torchModeChangeCallback = torchModeChangeCallback
self.zoomScaleChangeCallback = zoomScaleChangeCallback
super.init()
}
/// Check if we already have camera permission.
func checkPermission() -> Int {
let status = AVCaptureDevice.authorizationStatus(for: .video)
switch status {
case .notDetermined:
return 0
case .authorized:
return 1
default:
return 2
}
}
/// Request permissions for video
func requestPermission(_ result: @escaping FlutterResult) {
AVCaptureDevice.requestAccess(for: .video, completionHandler: { result($0) })
}
/// Gets called when a new image is added to the buffer
public func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
print("Failed to get image buffer from sample buffer.")
return
}
latestBuffer = imageBuffer
registry?.textureFrameAvailable(textureId)
if ((detectionSpeed == DetectionSpeed.normal || detectionSpeed == DetectionSpeed.noDuplicates) && i > 10 || detectionSpeed == DetectionSpeed.unrestricted) {
i = 0
let ciImage = latestBuffer.image
let image = VisionImage(image: ciImage)
image.orientation = imageOrientation(
deviceOrientation: UIDevice.current.orientation,
defaultOrientation: .portrait,
position: videoPosition
)
scanner.process(image) { [self] barcodes, error in
if (detectionSpeed == DetectionSpeed.noDuplicates) {
let newScannedBarcodes = barcodes?.map { barcode in
return barcode.rawValue
}
if (error == nil && barcodesString != nil && newScannedBarcodes != nil && barcodesString!.elementsEqual(newScannedBarcodes!)) {
return
} else {
barcodesString = newScannedBarcodes
}
}
mobileScannerCallback(barcodes, error, ciImage)
}
} else {
i+=1
}
}
/// Start scanning for barcodes
func start(barcodeScannerOptions: BarcodeScannerOptions?, returnImage: Bool, cameraPosition: AVCaptureDevice.Position, torch: AVCaptureDevice.TorchMode, detectionSpeed: DetectionSpeed) throws -> MobileScannerStartParameters {
self.detectionSpeed = detectionSpeed
if (device != nil) {
throw MobileScannerError.alreadyStarted
}
scanner = barcodeScannerOptions != nil ? BarcodeScanner.barcodeScanner(options: barcodeScannerOptions!) : BarcodeScanner.barcodeScanner()
captureSession = AVCaptureSession()
textureId = registry?.register(self)
// Open the camera device
if #available(iOS 13.0, *) {
device = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInTripleCamera, .builtInDualWideCamera, .builtInDualCamera, .builtInWideAngleCamera], mediaType: .video, position: cameraPosition).devices.first
} else {
device = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInDualCamera, .builtInWideAngleCamera], mediaType: .video, position: cameraPosition).devices.first
}
if (device == nil) {
throw MobileScannerError.noCamera
}
device.addObserver(self, forKeyPath: #keyPath(AVCaptureDevice.torchMode), options: .new, context: nil)
device.addObserver(self, forKeyPath: #keyPath(AVCaptureDevice.videoZoomFactor), options: .new, context: nil)
do {
try device.lockForConfiguration()
if device.isFocusModeSupported(.continuousAutoFocus) {
device.focusMode = .continuousAutoFocus
}
if #available(iOS 15.4, *) , device.isFocusModeSupported(.autoFocus){
device.automaticallyAdjustsFaceDrivenAutoFocusEnabled = false
}
device.unlockForConfiguration()
} catch {}
captureSession.beginConfiguration()
// Add device input
do {
let input = try AVCaptureDeviceInput(device: device)
captureSession.addInput(input)
} catch {
throw MobileScannerError.cameraError(error)
}
captureSession.sessionPreset = AVCaptureSession.Preset.photo;
// Add video output.
let videoOutput = AVCaptureVideoDataOutput()
videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
videoOutput.alwaysDiscardsLateVideoFrames = true
videoPosition = cameraPosition
// calls captureOutput()
videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue.main)
captureSession.addOutput(videoOutput)
for connection in videoOutput.connections {
connection.videoOrientation = .portrait
if cameraPosition == .front && connection.isVideoMirroringSupported {
connection.isVideoMirrored = true
}
}
captureSession.commitConfiguration()
captureSession.startRunning()
// Enable the torch if parameter is set and torch is available
// torch should be set after 'startRunning' is called
do {
try toggleTorch(torch)
} catch {
print("Failed to set initial torch state.")
}
let dimensions = CMVideoFormatDescriptionGetDimensions(device.activeFormat.formatDescription)
return MobileScannerStartParameters(width: Double(dimensions.height), height: Double(dimensions.width), hasTorch: device.hasTorch, textureId: textureId)
}
/// Stop scanning for barcodes
func stop() throws {
if (device == nil) {
throw MobileScannerError.alreadyStopped
}
captureSession.stopRunning()
for input in captureSession.inputs {
captureSession.removeInput(input)
}
for output in captureSession.outputs {
captureSession.removeOutput(output)
}
device.removeObserver(self, forKeyPath: #keyPath(AVCaptureDevice.torchMode))
device.removeObserver(self, forKeyPath: #keyPath(AVCaptureDevice.videoZoomFactor))
registry?.unregisterTexture(textureId)
textureId = nil
captureSession = nil
device = nil
}
/// Toggle the flashlight between on and off
func toggleTorch(_ torch: AVCaptureDevice.TorchMode) throws {
if (device == nil) {
throw MobileScannerError.torchWhenStopped
}
if (device.hasTorch && device.isTorchAvailable) {
do {
try device.lockForConfiguration()
device.torchMode = torch
device.unlockForConfiguration()
} catch {
throw MobileScannerError.torchError(error)
}
}
}
// Observer for torch state
public override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) {
switch keyPath {
case "torchMode":
// off = 0; on = 1; auto = 2;
let state = change?[.newKey] as? Int
torchModeChangeCallback(state)
case "videoZoomFactor":
let zoomFactor = change?[.newKey] as? CGFloat ?? 1
let zoomScale = (zoomFactor - 1) / 4
zoomScaleChangeCallback(Double(zoomScale))
default:
break
}
}
/// Set the zoom factor of the camera
func setScale(_ scale: CGFloat) throws {
if (device == nil) {
throw MobileScannerError.torchWhenStopped
}
do {
try device.lockForConfiguration()
var maxZoomFactor = device.activeFormat.videoMaxZoomFactor
var actualScale = (scale * 4) + 1
// Set maximum zoomrate of 5x
actualScale = min(5.0, actualScale)
// Limit to max rate of camera
actualScale = min(maxZoomFactor, actualScale)
// Limit to 1.0 scale
device.ramp(toVideoZoomFactor: actualScale, withRate: 5)
device.unlockForConfiguration()
} catch {
throw MobileScannerError.zoomError(error)
}
}
/// Analyze a single image
func analyzeImage(image: UIImage, position: AVCaptureDevice.Position, callback: @escaping BarcodeScanningCallback) {
let image = VisionImage(image: image)
image.orientation = imageOrientation(
deviceOrientation: UIDevice.current.orientation,
defaultOrientation: .portrait,
position: position
)
scanner.process(image, completion: callback)
}
var i = 0
var barcodesString: Array<String?>?
// /// Convert image buffer to jpeg
// private func ciImageToJpeg(ciImage: CIImage) -> Data {
//
// // let ciImage = CIImage(cvPixelBuffer: latestBuffer)
// let context:CIContext = CIContext.init(options: nil)
// let cgImage:CGImage = context.createCGImage(ciImage, from: ciImage.extent)!
// let uiImage:UIImage = UIImage(cgImage: cgImage, scale: 1, orientation: UIImage.Orientation.up)
//
// return uiImage.jpegData(compressionQuality: 0.8)!;
// }
/// Rotates images accordingly
func imageOrientation(
deviceOrientation: UIDeviceOrientation,
defaultOrientation: UIDeviceOrientation,
position: AVCaptureDevice.Position
) -> UIImage.Orientation {
switch deviceOrientation {
case .portrait:
return position == .front ? .leftMirrored : .right
case .landscapeLeft:
return position == .front ? .downMirrored : .up
case .portraitUpsideDown:
return position == .front ? .rightMirrored : .left
case .landscapeRight:
return position == .front ? .upMirrored : .down
case .faceDown, .faceUp, .unknown:
return .up
@unknown default:
return imageOrientation(deviceOrientation: defaultOrientation, defaultOrientation: .portrait, position: .back)
}
}
/// Sends output of OutputBuffer to a Flutter texture
public func copyPixelBuffer() -> Unmanaged<CVPixelBuffer>? {
if latestBuffer == nil {
return nil
}
return Unmanaged<CVPixelBuffer>.passRetained(latestBuffer)
}
struct MobileScannerStartParameters {
var width: Double = 0.0
var height: Double = 0.0
var hasTorch = false
var textureId: Int64 = 0
}
}