updated scaling implementation

casvanluijtelaar
Commit b1d26967d3b2ee8ee3c6665b292da0e2cc51fefc b1d26967 1 parent 55a5d7f8
Showing 6 changed files with 196 additions and 56 deletions
android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt
example/lib/barcode_scanner_window.dart
ios/Classes/SwiftMobileScannerPlugin.swift
lib/src/mobile_scanner.dart
lib/src/mobile_scanner_controller.dart
macos/Classes/MobileScannerPlugin.swift
--- a/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt
View file @b1d2696
+++ b/android/src/main/kotlin/dev/steenbakker/mobile_scanner/MobileScanner.kt
View file @b1d2696
@@ -58,6 +58,7 @@ class MobileScanner(private val activity: Activity, private val textureRegistry:
 //            "analyze" -> switchAnalyzeMode(call, result)
             "stop" -> stop(result)
             "analyzeImage" -> analyzeImage(call, result)
+            "updateScanWindow" -> updateScanWindow(call)
             else -> result.notImplemented()
         }
     }
@@ -108,9 +109,10 @@ class MobileScanner(private val activity: Activity, private val textureRegistry:
                 scanner.process(inputImage)
                         .addOnSuccessListener { barcodes ->
                             for (barcode in barcodes) {
+
                                 if(scanWindow != null) {
-                                    val boundingBox = barcode.getBoundingBox()
-                                    if(boundingBox == null || !scanWindow!!.contains(boundingBox!!)) continue
+                                    val match = isbarCodeInScanWindow(scanWindow!!, barcode, inputImage)
+                                    if(!match) continue
                                 }
                                 val event = mapOf("name" to "barcode", "data" to barcode.data)
@@ -126,6 +128,34 @@ class MobileScanner(private val activity: Activity, private val textureRegistry:
     private var scanner = BarcodeScanning.getClient()
+
+    private fun updateScanWindow(call: MethodCall) {
+        val scanWindowData: List<Int>? = call.argument("rect")
+        if(scanWindowData == null) return
+            
+        scanWindow = Rect(scanWindowData!![0], scanWindowData!![1], scanWindowData!![2], scanWindowData!![3])
+    }
+
+    // scales the scanWindow to the provided inputImage and checks if that scaled
+    // scanWindow contains the barcode
+    private fun isbarCodeInScanWindow(scanWindow: Rect, barcode: Barcode, inputImage: InputImage): Boolean {
+        val barcodeBoundingBox = barcode.getBoundingBox()
+        if(barcodeBoundingBox == null) return false
+        
+        val imageWidth = inputImage.getWidth();
+        val imageHeight = inputImage.getHeight();
+
+        val left = scanWindow.left * imageWidth
+        val top = scanWindow.top * imageHeight
+        val right = scanWindow.right * imageWidth
+        val bottom = scanWindow.bottom * imageHeight
+
+        val scaledScanWindow = Rect(left, top, right, bottom)
+        return scaledScanWindow.contains(barcodeBoundingBox)
+    }
+
+
+
     @ExperimentalGetImage
     private fun start(call: MethodCall, result: MethodChannel.Result) {
         if (camera?.cameraInfo != null && preview != null && textureEntry != null) {
@@ -142,9 +172,6 @@ class MobileScanner(private val activity: Activity, private val textureRegistry:
             val torch: Boolean = call.argument<Boolean>("torch") ?: false
             val formats: List<Int>? = call.argument<List<Int>>("formats")
-            val scanWindowData: List<Int>? = call.argument("scanWindow")
-            if(scanWindowData != null) scanWindow = Rect(scanWindowData!![0], scanWindowData!![1], scanWindowData!![2], scanWindowData!![3])
-
             if (formats != null) {
                 val formatsList: MutableList<Int> = mutableListOf()
                 for (index in formats) {
@@ -253,8 +280,8 @@ class MobileScanner(private val activity: Activity, private val textureRegistry:
             .addOnSuccessListener { barcodes ->
                 for (barcode in barcodes) {
                     if(scanWindow != null) {
-                        val boundingBox = barcode.getBoundingBox()
-                        if(boundingBox == null || !scanWindow!!.contains(boundingBox!!)) continue
+                        val match = isbarCodeInScanWindow(scanWindow!!, barcode, inputImage)
+                        if(!match) continue
                     }
                     barcodeFound = true
--- a/example/lib/barcode_scanner_window.dart
View file @b1d2696
+++ b/example/lib/barcode_scanner_window.dart
View file @b1d2696
@@ -36,7 +36,6 @@ class _BarcodeScannerWithScanWindowState
     controller = MobileScannerController(
       torchEnabled: true,
-      scanWindow: scanWindow,
     );
   }
@@ -52,6 +51,7 @@ class _BarcodeScannerWithScanWindowState
             children: [
               MobileScanner(
                 fit: BoxFit.contain,
+                scanWindow: scanWindow,
                 controller: controller,
                 onDetect: (barcode, args) {
                   setState(() {
--- a/ios/Classes/SwiftMobileScannerPlugin.swift
View file @b1d2696
+++ b/ios/Classes/SwiftMobileScannerPlugin.swift
View file @b1d2696
@@ -64,6 +64,8 @@ public class SwiftMobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHan
             stop(result)
         case "analyzeImage":
             analyzeImage(call, result)
+        case "updateScanWindow":
+            updateScanWindow(call)
         default:
             result(FlutterMethodNotImplemented)
         }
@@ -114,8 +116,8 @@ public class SwiftMobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHan
                     for barcode in barcodes! {
                         if scanWindow != nil {
-                            let boundingBox = barcode.frame
-                            if !scanWindow!.contains(boundingBox) {
+                            let match = isbarCodeInScanWindow(scanWindow!, barcode, buffer!.image)
+                            if (!match) {
                                 continue
                             }
                         }
@@ -167,6 +169,38 @@ public class SwiftMobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHan
         AVCaptureDevice.requestAccess(for: .video, completionHandler: { result($0) })
     }
+    func updateScanWindow(_ call: FlutterMethodCall) {
+        let argReader = MapArgumentReader(call.arguments as? [String: Any])
+        let scanWindowData: Array? = argReader.floatArray(key: "rect")
+
+        if (scanWindowData == nil) {
+            return 
+        }
+
+        let minX = scanWindowData![0] 
+        let minY = scanWindowData![1]
+
+        let width = scanWindowData![2]  - minX
+        let height = scanWindowData![3] - minY
+
+        scanWindow = CGRect(x: minX, y: minY, width: width, height: height)
+    }
+
+   func isbarCodeInScanWindow(_ scanWindow: CGRect, _ barcode: Barcode, _ inputImage: UIImage) -> Bool {
+        let barcodeBoundingBox = barcode.frame
+
+        let imageWidth = inputImage.size.width;
+        let imageHeight = inputImage.size.height;
+
+        let minX = scanWindow.minX * imageWidth
+        let minY = scanWindow.minY * imageHeight
+        let width = scanWindow.width * imageWidth
+        let height = scanWindow.height * imageHeight
+
+        let scaledScanWindow = CGRect(x: minX, y: minY, width: width, height: height)
+        return scaledScanWindow.contains(barcodeBoundingBox)
+   }
+
     func start(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) {
         if (device != nil) {
             result(FlutterError(code: "MobileScanner",
@@ -184,21 +218,7 @@ public class SwiftMobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHan
         let torch: Bool = argReader.bool(key: "torch") ?? false
         let facing: Int = argReader.int(key: "facing") ?? 1
         let formats: Array = argReader.intArray(key: "formats") ?? []
-        let scanWindowData: Array? = argReader.floatArray(key: "scanWindow")
-
-        if(scanWindowData != nil) {
-
-            let minX = scanWindowData![0] 
-            let minY = scanWindowData![1]
-     
-            let width = scanWindowData![2]  - minX
-            let height = scanWindowData![3] - minY
-
-            scanWindow = CGRect(x: minX, y: minY, width: width, height: height)
-        }
-        
-        
-        
+               
         let formatList: NSMutableArray = []
         for index in formats {
             formatList.add(BarcodeFormat(rawValue: index))
@@ -319,8 +339,8 @@ public class SwiftMobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHan
                 for barcode in barcodes! {
                     if scanWindow != nil {
-                        let boundingBox = barcode.frame
-                        if !scanWindow!.contains(boundingBox) {
+                        let match = isbarCodeInScanWindow(scanWindow!, barcode, uiImage!)
+                        if (!match) {
                             continue
                         }
                     }
--- a/lib/src/mobile_scanner.dart
View file @b1d2696
+++ b/lib/src/mobile_scanner.dart
View file @b1d2696
@@ -27,6 +27,13 @@ class MobileScanner extends StatefulWidget {
   /// Set to false if you don't want duplicate scans.
   final bool allowDuplicates;
+  /// if set barcodes will only be scanned if they fall within this [Rect]
+  /// useful for having a cut-out overlay for example. these [Rect]
+  /// coordinates are relative to the widget size, so by how much your
+  /// rectangle overlays the actual image can depend on things like the
+  /// [BoxFit]
+  final Rect? scanWindow;
+
   /// Create a [MobileScanner] with a [controller], the [controller] must has been initialized.
   const MobileScanner({
     Key? key,
@@ -34,6 +41,7 @@ class MobileScanner extends StatefulWidget {
     this.controller,
     this.fit = BoxFit.cover,
     this.allowDuplicates = false,
+    this.scanWindow,
   }) : super(key: key);
   @override
@@ -67,6 +75,58 @@ class _MobileScannerState extends State<MobileScanner>
   String? lastScanned;
+  /// the [scanWindow] rect will be relative and scaled to the [widgetSize] not the texture. so it is possible,
+  /// depending on the [fit], for the [scanWindow] to partially or not at all overlap the [textureSize]
+  ///
+  /// since when using a [BoxFit] the content will always be centered on its parent. we can convert the rect
+  /// to be relative to the texture.
+  ///
+  /// since the textures size and the actuall image (on the texture size) might not be the same, we also need to
+  /// calculate the scanWindow in terms of percentages of the texture, not pixels.
+  Rect calculateScanWindowRelativeToTextureInPercentage(
+    BoxFit fit,
+    Rect scanWindow,
+    Size textureSize,
+    Size widgetSize,
+  ) {
+    /// map the texture size to get its new size after fitted to screen
+    final fittedSizes = applyBoxFit(fit, textureSize, widgetSize);
+    final fittedTextureSize = fittedSizes.destination;
+
+    final minX = widgetSize.width / 2 - fittedTextureSize.width / 2;
+    final minY = widgetSize.height / 2 - fittedTextureSize.height / 2;
+    final width = fittedTextureSize.width;
+    final height = fittedTextureSize.height;
+
+    final textureWindow = Rect.fromLTWH(minX, minY, width, height);
+
+    /// create a new scan window and with only the area of the rect intersecting the texture
+    final scanWindowInTexture = scanWindow.intersect(textureWindow);
+
+    /// update the scanWindow left and top to be relative to the texture not the widget
+    final newLeft = scanWindowInTexture.left - minX;
+    final newTop = scanWindowInTexture.top - minY;
+    final newWidth = scanWindowInTexture.width;
+    final newHeight = scanWindowInTexture.height;
+
+    /// new scanWindow that is adapted to the boxfit and relative to the texture
+    final windowInTexture = Rect.fromLTWH(newLeft, newTop, newWidth, newHeight);
+
+    /// get the scanWindow as a percentage of the texture
+    final percentageLeft = windowInTexture.left / fittedTextureSize.width;
+    final percentageRight = windowInTexture.right / fittedTextureSize.width;
+    final percentageTop = windowInTexture.top / fittedTextureSize.height;
+    final percentagebottom = windowInTexture.bottom / fittedTextureSize.height;
+
+    /// this rectangle can be send to native code and used to cut out a rectangle of the scan image
+    return Rect.fromLTRB(
+      percentageLeft,
+      percentageRight,
+      percentageTop,
+      percentagebottom,
+    );
+  }
+
   @override
   Widget build(BuildContext context) {
     return LayoutBuilder(
@@ -78,6 +138,17 @@ class _MobileScannerState extends State<MobileScanner>
             if (value == null) {
               return Container(color: Colors.black);
             } else {
+              if (widget.scanWindow != null) {
+                final window = calculateScanWindowRelativeToTextureInPercentage(
+                  widget.fit,
+                  widget.scanWindow!,
+                  value.size,
+                  Size(constraints.maxWidth, constraints.maxHeight),
+                );
+                print(window);
+                controller.updateScanWindow(window);
+              }
+
               controller.barcodes.listen((barcode) {
                 if (!widget.allowDuplicates) {
                   if (lastScanned != barcode.rawValue) {
--- a/lib/src/mobile_scanner_controller.dart
View file @b1d2696
+++ b/lib/src/mobile_scanner_controller.dart
View file @b1d2696
@@ -49,9 +49,6 @@ class MobileScannerController {
   /// WARNING: On iOS, only 1 format is supported.
   final List<BarcodeFormat>? formats;
-  /// can be used to limit the scan area to a portion of the screen
-  final Rect? scanWindow;
-
   CameraFacing facing;
   bool hasTorch = false;
   late StreamController<Barcode> barcodesController;
@@ -63,7 +60,6 @@ class MobileScannerController {
     this.ratio,
     this.torchEnabled,
     this.formats,
-    this.scanWindow,
   }) {
     // In case a new instance is created before calling dispose()
     if (_controllerHashcode != null) {
@@ -160,14 +156,14 @@ class MobileScannerController {
     // Set the starting arguments for the camera
     final Map arguments = {};
     arguments['facing'] = facing.index;
-    if (scanWindow != null) {
+ /*    if (scanWindow != null) {
       arguments['scanWindow'] = [
         scanWindow!.left,
         scanWindow!.top,
         scanWindow!.right,
         scanWindow!.bottom,
       ];
-    }
+    } */
     if (ratio != null) arguments['ratio'] = ratio;
     if (torchEnabled != null) arguments['torch'] = torchEnabled;
@@ -295,4 +291,10 @@ class MobileScannerController {
         'MobileScannerController methods should not be used after calling dispose.';
     assert(hashCode == _controllerHashcode, message);
   }
+
+  /// updates the native scanwindow
+  Future<void> updateScanWindow(Rect window) async {
+    final data = [window.left, window.right, window.top, window.bottom];
+    await methodChannel.invokeMethod('updateScanWindow', {'rect': data});
+  }
 }
--- a/macos/Classes/MobileScannerPlugin.swift
View file @b1d2696
+++ b/macos/Classes/MobileScannerPlugin.swift
View file @b1d2696
@@ -61,6 +61,8 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHandler,
 //            switchAnalyzeMode(call, result)
         case "stop":
             stop(result)
+        case "updateScanWindow":
+            updateScanWindow(call)
         default:
             result(FlutterMethodNotImplemented)
         }
@@ -113,18 +115,17 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHandler,
               try imageRequestHandler.perform([VNDetectBarcodesRequest { (request, error) in
                   if error == nil {
                       if let results = request.results as? [VNBarcodeObservation] {
-                                for barcode in results {
-                                    if scanWindow != nil {
-                                        let boundingBox = barcode.frame
-                                        if !scanWindow!.contains(boundingBox) {
-                                            continue
-                                        }
+                            for barcode in results {
+                                if scanWindow != nil {
+                                    let match = isbarCodeInScanWindow(scanWindow!, barcode, buffer!.image)
+                                    if (!match) {
+                                        continue
                                     }
+                                }
-
-                                      let barcodeType = String(barcode.symbology.rawValue).replacingOccurrences(of: "VNBarcodeSymbology", with: "")
-                                      let event: [String: Any?] = ["name": "barcodeMac", "data" : ["payload": barcode.payloadStringValue, "symbology": barcodeType]]
-                                      self.sink?(event)
+                                let barcodeType = String(barcode.symbology.rawValue).replacingOccurrences(of: "VNBarcodeSymbology", with: "")
+                                let event: [String: Any?] = ["name": "barcodeMac", "data" : ["payload": barcode.payloadStringValue, "symbology": barcodeType]]
+                                self.sink?(event)
   //                                    if barcodeType == "QR" {
   //                                        let image = CIImage(image: source)
@@ -170,6 +171,38 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHandler,
         }
     }
+    func updateScanWindow(_ call: FlutterMethodCall) {
+        let argReader = MapArgumentReader(call.arguments as? [String: Any])
+        let scanWindowData: Array? = argReader.floatArray(key: "rect")
+
+        if (scanWindowData == nil) {
+            return 
+        }
+
+        let minX = scanWindowData![0] 
+        let minY = scanWindowData![1]
+
+        let width = scanWindowData![2]  - minX
+        let height = scanWindowData![3] - minY
+
+        scanWindow = CGRect(x: minX, y: minY, width: width, height: height)
+    }
+
+   func isbarCodeInScanWindow(_ scanWindow: CGRect, _ barcode: Barcode, _ inputImage: UIImage) -> Bool {
+        let barcodeBoundingBox = barcode.frame
+
+        let imageWidth = inputImage.size.width;
+        let imageHeight = inputImage.size.height;
+
+        let minX = scanWindow.minX * imageWidth
+        let minY = scanWindow.minY * imageHeight
+        let width = scanWindow.width * imageWidth
+        let height = scanWindow.height * imageHeight
+
+        let scaledScanWindow = CGRect(x: minX, y: minY, width: width, height: height)
+        return scaledScanWindow.contains(barcodeBoundingBox)
+    }
+
     func start(_ call: FlutterMethodCall, _ result: @escaping FlutterResult) {
         if (device != nil) {
             result(FlutterError(code: "MobileScanner",
@@ -187,19 +220,6 @@ public class MobileScannerPlugin: NSObject, FlutterPlugin, FlutterStreamHandler,
         let torch: Bool = argReader.bool(key: "torch") ?? false
         let facing: Int = argReader.int(key: "facing") ?? 1
-        let scanWindowData: Array? = argReader.floatArray(key: "scanWindow")
-
-        if(scanWindowData != nil) {
-
-            let minX = scanWindowData![0] 
-            let minY = scanWindowData![1]
-     
-            let width = scanWindowData![2]  - minX
-            let height = scanWindowData![3] - minY
-
-            scanWindow = CGRect(x: minX, y: minY, width: width, height: height)
-        }
-
         // Set the camera to use
         position = facing == 0 ? AVCaptureDevice.Position.front : .back