fix: wip color and format conversion

thomas-coldwell · thomas-coldwell · commit 9c51f12c5c90 · 2024-09-28T15:51:26.000+01:00
diff --git a/cpp/FOCV_Object.cpp b/cpp/FOCV_Object.cpp
@@ -29,7 +29,7 @@ constexpr uint64_t hashString(const char* str, size_t length) {
   return hash;
 }
 
-jsi::Object FOCV_Object::create(jsi::Runtime& runtime, const jsi::Value* arguments) {
+jsi::Object FOCV_Object::create(jsi::Runtime& runtime, const jsi::Value* arguments, size_t argCount) {
     std::string id = "";
     std::string objectType = arguments[0].asString(runtime).utf8(runtime);
 
@@ -38,29 +38,27 @@ jsi::Object FOCV_Object::create(jsi::Runtime& runtime, const jsi::Value* argumen
             int rows = arguments[1].asNumber();
             int cols = arguments[2].asNumber();
             int type = arguments[3].asNumber();
-            
-            if(arguments[4].isObject()) {
-                auto rawArray = arguments[4].asObject(runtime);
-                auto array = rawArray.asArray(runtime);
-                
-                auto rawLength = rawArray.getProperty(runtime, "length");
-                auto length = rawLength.asNumber();
-                
-                std::vector<float> vec;
-                
-                for(auto i = 0; i < length; i++) {
-                    vec.push_back(array.getValueAtIndex(runtime, i).asNumber());
-                }
-
-                cv::Mat mat{vec, true};
-                mat = mat.reshape(1, rows);
-                mat.convertTo(mat, type);
-                
-                id = FOCV_Storage::save(mat);
-            } else {
-                cv::Mat object(rows, cols, type);
-                id = FOCV_Storage::save(object);
+          
+            cv::Mat mat(rows, cols, type);
+            memset(mat.data, 0, rows * cols * mat.elemSize1());
+          
+            if (argCount == 5 && arguments[4].isObject()) {
+              std::vector<float> vec;
+              auto rawArray = arguments[4].asObject(runtime);
+              auto array = rawArray.asArray(runtime);
+              
+              auto rawLength = rawArray.getProperty(runtime, "length");
+              auto length = rawLength.asNumber();
+              
+              for(auto i = 0; i < length; i++) {
+                vec.push_back(array.getValueAtIndex(runtime, i).asNumber());
+              }
+              memcpy(mat.data, vec.data(), vec.size() * mat.elemSize1());
+              mat = mat.reshape(1, rows);
+              mat.convertTo(mat, type);
             }
+            
+            id = FOCV_Storage::save(mat);
         } break;
         case hashString("mat_vector", 10): {
             std::vector<cv::Mat> object;
diff --git a/cpp/FOCV_Object.hpp b/cpp/FOCV_Object.hpp
@@ -30,7 +30,7 @@ using namespace facebook;
 
 class FOCV_Object {
 public:
-    static jsi::Object create(jsi::Runtime& runtime, const jsi::Value* arguments);
+    static jsi::Object create(jsi::Runtime& runtime, const jsi::Value* arguments, size_t argCount);
     static jsi::Object convertToJSI(jsi::Runtime& runtime, const jsi::Value* arguments);
     static jsi::Object copyObjectFromVector(jsi::Runtime& runtime, const jsi::Value* arguments);
 };
diff --git a/cpp/react-native-fast-opencv.cpp b/cpp/react-native-fast-opencv.cpp
@@ -47,36 +47,51 @@ jsi::Value OpenCVPlugin::get(jsi::Runtime& runtime, const jsi::PropNameID& propN
         double rows = arguments[0].asNumber();
         double cols = arguments[1].asNumber();
         int channels = arguments[2].asNumber();
-        jsi::Object input = arguments[3].asObject(runtime);
+        std::string dataType = arguments[3].asString(runtime).utf8(runtime);
+        jsi::Object input = arguments[4].asObject(runtime);
 
         int type = -1;
         if (channels == 1) {
-          type = CV_8U;
+          type = dataType == "float32" ? CV_32F : CV_8U;
         }
         if (channels == 3) {
-          type = CV_8UC3;
+          type = dataType == "float32" ? CV_32FC3 : CV_8UC3;
         }
         if (channels == 4) {
-          type = CV_8UC4;
+          type = dataType == "float32" ? CV_32FC4 : CV_8UC4;
         }
               
         if (channels == -1) {
           throw std::runtime_error("Fast OpenCV Error: Invalid channel count passed to frameBufferToMat!");
         }
+              
+        TypedArrayKind typedArrayType = dataType == "float32" ? TypedArrayKind::Float32Array : TypedArrayKind::Uint8Array;
+        
+        if (dataType == "float32") {
+          TypedArrayBase inputBuffer = getTypedArray(runtime, std::move(input));
+          auto vec = inputBuffer.toVector(runtime);
+
+          cv::Mat mat(rows, cols, type);
+          memcpy(mat.data, vec.data(), (int)rows * (int)cols * channels * 4);
+          auto id = FOCV_Storage::save(mat);
+          
+          return FOCV_JsiObject::wrap(runtime, "mat", id);
+        }
+        else {
+          TypedArrayBase inputBuffer = getTypedArray(runtime, std::move(input));
+          auto vec = inputBuffer.toVector(runtime);
 
-        TypedArrayBase inputBuffer = getTypedArray(runtime, std::move(input));
-        auto vec = inputBuffer.toVector(runtime);
-
-        cv::Mat mat(rows, cols, type);
-        memcpy(mat.data, vec.data(), (int)rows * (int)cols * channels);
-        auto id = FOCV_Storage::save(mat);
-
-        return FOCV_JsiObject::wrap(runtime, "mat", id);
+          cv::Mat mat(rows, cols, type);
+          memcpy(mat.data, vec.data(), (int)rows * (int)cols * channels);
+          auto id = FOCV_Storage::save(mat);
+          
+          return FOCV_JsiObject::wrap(runtime, "mat", id);
+        }
     });
   }
   else if (propName == "base64ToMat") {
       return jsi::Function::createFromHostFunction(
-          runtime, jsi::PropNameID::forAscii(runtime, "frameBufferToMat"), 1,
+          runtime, jsi::PropNameID::forAscii(runtime, "base64ToMat"), 1,
           [=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
               size_t count) -> jsi::Object {
 
@@ -123,8 +138,8 @@ jsi::Value OpenCVPlugin::get(jsi::Runtime& runtime, const jsi::PropNameID& propN
           runtime, jsi::PropNameID::forAscii(runtime, "createObject"), 1,
           [=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
               size_t count) -> jsi::Object {
-
-          return FOCV_Object::create(runtime, arguments);
+                
+          return FOCV_Object::create(runtime, arguments, count);
       });
     }
   else if (propName == "toJSValue") {
diff --git a/example/src/examples/CameraAffineTransform.tsx b/example/src/examples/CameraAffineTransform.tsx
@@ -9,7 +9,12 @@ import {
 } from '@shopify/react-native-skia';
 import { useEffect } from 'react';
 import { SafeAreaView, StyleSheet, Text, View } from 'react-native';
-import { DataTypes, ObjectType, OpenCV } from 'react-native-fast-opencv';
+import {
+  ColorConversionCodes,
+  DataTypes,
+  ObjectType,
+  OpenCV,
+} from 'react-native-fast-opencv';
 import { useSharedValue } from 'react-native-reanimated';
 import {
   Camera,
@@ -22,10 +27,10 @@ import { useResizePlugin, type Options } from 'vision-camera-resize-plugin';
 
 type PixelFormat = Options<'uint8'>['pixelFormat'];
 
-const WIDTH = 300;
-const HEIGHT = 300;
-const TARGET_TYPE = 'uint8' as const;
-const TARGET_FORMAT: PixelFormat = 'rgba';
+const WIDTH = 50;
+const HEIGHT = 50;
+const TARGET_TYPE = 'float32' as const;
+const TARGET_FORMAT: PixelFormat = 'rgb';
 
 let lastWarn: PixelFormat | undefined;
 lastWarn = undefined;
@@ -114,6 +119,7 @@ export function CameraAffineTransform() {
     (frame) => {
       'worklet';
 
+      // Step 1. Resize the frame to the target size
       const resized = resize(frame, {
         scale: {
           width: WIDTH,
@@ -130,41 +136,77 @@ export function CameraAffineTransform() {
         rotation: '90deg',
       });
 
-      const frameMat = OpenCV.frameBufferToMat(HEIGHT, WIDTH, 4, resized);
-      const outputMat = OpenCV.createObject(
-        ObjectType.Mat,
+      // Step 2. Create an OpenCV Mat from the resized frame
+      const frameMat = OpenCV.frameBufferToMat(
         HEIGHT,
         WIDTH,
-        DataTypes.CV_8UC4
+        3,
+        'float32',
+        resized
       );
 
+      // Step 3. Apply affine transform to the frame
+      const transformedMat = OpenCV.createObject(
+        ObjectType.Mat,
+        HEIGHT,
+        WIDTH,
+        DataTypes.CV_32FC3,
+        undefined
+      );
       const scaleX = frame.height / frame.width;
       const scaledFrameWidth = WIDTH * scaleX;
       const translateX = (scaledFrameWidth - WIDTH) / 2;
-
       // 2x3 affine matrix has the following form (for scale and translation only):
       // | sx 0  tx |
       // | 0  sy ty |
       const matrix = [
         [scaleX, 0.0, -translateX],
         [0.0, 1.0, 0.0],
       ];
-
       const transformMat = OpenCV.createObject(
         ObjectType.Mat,
         2,
         3,
         DataTypes.CV_32F,
         matrix.flat()
       );
+      OpenCV.invoke('warpAffine', frameMat, transformedMat, transformMat);
 
-      OpenCV.invoke('warpAffine', frameMat, outputMat, transformMat);
-
-      const output = OpenCV.matToBuffer(outputMat, 'uint8');
-      const data = Skia.Data.fromBytes(output.buffer);
+      // Step 4. Convert the transformed frame to RGBA
+      const uint8TransformedMat = OpenCV.createObject(
+        ObjectType.Mat,
+        HEIGHT,
+        WIDTH,
+        DataTypes.CV_8UC3,
+        undefined
+      );
+      OpenCV.invoke(
+        'convertTo',
+        transformedMat,
+        uint8TransformedMat,
+        DataTypes.CV_8UC3,
+        255.0,
+        0
+      );
+      const rgbaOutputMat = OpenCV.createObject(
+        ObjectType.Mat,
+        HEIGHT,
+        WIDTH,
+        DataTypes.CV_8UC4,
+        undefined
+      );
+      OpenCV.invoke(
+        'cvtColor',
+        uint8TransformedMat,
+        rgbaOutputMat,
+        ColorConversionCodes.COLOR_RGB2RGBA
+      );
 
-      updatePreviewImageFromData(data, TARGET_FORMAT).then(() => {
-        data.dispose();
+      // Step 5. Convert the transformed frame to Skia Image
+      const output = OpenCV.matToBuffer(rgbaOutputMat, 'uint8');
+      const outputData = Skia.Data.fromBytes(output.buffer);
+      updatePreviewImageFromData(outputData, 'rgba').then(() => {
+        outputData.dispose();
       });
 
       OpenCV.clearBuffers(); // REMEMBER TO CLEAN
@@ -224,5 +266,7 @@ const styles = StyleSheet.create({
     height: HEIGHT,
     borderColor: 'red',
     borderWidth: 2,
+    transformOrigin: 'bottom',
+    transform: [{ scale: 4 }],
   },
 });
diff --git a/example/src/examples/CameraPassthrough.tsx b/example/src/examples/CameraPassthrough.tsx
@@ -130,7 +130,13 @@ export function CameraPassthrough() {
         rotation: '90deg',
       });
 
-      const frameMat = OpenCV.frameBufferToMat(HEIGHT, WIDTH, 4, resized);
+      const frameMat = OpenCV.frameBufferToMat(
+        HEIGHT,
+        WIDTH,
+        4,
+        'uint8',
+        resized
+      );
       const output = OpenCV.matToBuffer(frameMat, 'uint8');
       const data = Skia.Data.fromBytes(output.buffer);
 
diff --git a/example/src/examples/CameraRealtimeDetection.tsx b/example/src/examples/CameraRealtimeDetection.tsx
@@ -47,7 +47,7 @@ export function CameraRealtimeDetection() {
       dataType: 'uint8',
     });
 
-    const src = OpenCV.frameBufferToMat(height, width, 3, resized);
+    const src = OpenCV.frameBufferToMat(height, width, 3, 'uint8', resized);
     const dst = OpenCV.createObject(ObjectType.Mat, 0, 0, DataTypes.CV_8U);
 
     const lowerBound = OpenCV.createObject(ObjectType.Scalar, 30, 60, 60);
diff --git a/src/utils/UtilsFunctions.ts b/src/utils/UtilsFunctions.ts
@@ -12,11 +12,12 @@ export type UtilsFunctions = {
    * @param channels - the number of channels in the Mat
    * @param input - the byte array to convert
    */
-  frameBufferToMat(
+  frameBufferToMat<T extends 'uint8' | 'float32'>(
     rows: number,
     cols: number,
     channels: number,
-    input: Uint8Array
+    type: T,
+    input: T extends 'uint8' ? Uint8Array : Float32Array
   ): Mat;
   /**
    * Converts a base64 string to a Mat.
