feat(mobile): hand tracking model with drawing

client/.fvmrc

@@ -1,3 +1,4 @@
 {
-  "flutter": "3.22.1"
+  "flutter": "3.22.1",
+  "flavors": {}
 }

client/android/app/build.gradle

@@ -10,7 +10,7 @@ if (localPropertiesFile.exists()) {
 
 def flutterRoot = localProperties.getProperty('flutter.sdk')
 if (flutterRoot == null) {
-    throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file.")
+    throw GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file.")
 }
 
 def flutterVersionCode = localProperties.getProperty('flutter.versionCode')
@@ -52,7 +52,7 @@ android {
 
     // ----- END flavorDimensions (autogenerated by flutter_flavorizr) -----
 
-   compileSdkVersion 33
+   compileSdkVersion 34
 
     compileOptions {
         sourceCompatibility JavaVersion.VERSION_1_8
@@ -69,7 +69,7 @@ android {
     defaultConfig {
         // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html).
         applicationId "com.xmartlabs.simon_ai"
-        minSdkVersion 21
+        minSdkVersion 26
         targetSdkVersion 33
         versionCode flutterVersionCode.toInteger()
         versionName flutterVersionName

client/android/app/src/main/AndroidManifest.xml

@@ -1,5 +1,10 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.xmartlabs.simon_ai">
     <application android:label="@string/app_name" android:icon="@mipmap/ic_launcher">
+
+        <uses-native-library android:name="libOpenCL.so" android:required="false" />
+        <uses-native-library android:name="libOpenCL-car.so" android:required="false" />
+        <uses-native-library android:name="libOpenCL-pixel.so" android:required="false" />
+
         <activity android:name=".MainActivity" android:exported="true" android:launchMode="singleTop" android:theme="@style/LaunchTheme" android:configChanges="orientation|keyboardHidden|keyboard|screenSize|smallestScreenSize|locale|layoutDirection|fontScale|screenLayout|density|uiMode" android:hardwareAccelerated="true" android:windowSoftInputMode="adjustResize">
             <!-- Specifies an Android theme to apply to this Activity as soon as
                  the Android process has started. This theme is visible to the user

client/android/build.gradle

@@ -1,5 +1,5 @@
 buildscript {
-    ext.kotlin_version = '1.6.10'
+    ext.kotlin_version = '2.0.0'
     repositories {
         google()
         mavenCentral()

client/ios/Podfile.lock

@@ -21,6 +21,28 @@ PODS:
   - sqflite (0.0.3):
     - Flutter
     - FlutterMacOS
+  - TensorFlowLiteC (2.12.0):
+    - TensorFlowLiteC/Core (= 2.12.0)
+  - TensorFlowLiteC/Core (2.12.0)
+  - TensorFlowLiteC/CoreML (2.12.0):
+    - TensorFlowLiteC/Core
+  - TensorFlowLiteC/Metal (2.12.0):
+    - TensorFlowLiteC/Core
+  - TensorFlowLiteSwift (2.12.0):
+    - TensorFlowLiteSwift/Core (= 2.12.0)
+  - TensorFlowLiteSwift/Core (2.12.0):
+    - TensorFlowLiteC (= 2.12.0)
+  - TensorFlowLiteSwift/CoreML (2.12.0):
+    - TensorFlowLiteC/CoreML (= 2.12.0)
+    - TensorFlowLiteSwift/Core (= 2.12.0)
+  - TensorFlowLiteSwift/Metal (2.12.0):
+    - TensorFlowLiteC/Metal (= 2.12.0)
+    - TensorFlowLiteSwift/Core (= 2.12.0)
+  - tflite_flutter (0.0.1):
+    - Flutter
+    - TensorFlowLiteSwift (= 2.12.0)
+    - TensorFlowLiteSwift/CoreML (= 2.12.0)
+    - TensorFlowLiteSwift/Metal (= 2.12.0)
 
 DEPENDENCIES:
   - app_settings (from `.symlinks/plugins/app_settings/ios`)
@@ -33,6 +55,12 @@ DEPENDENCIES:
   - permission_handler_apple (from `.symlinks/plugins/permission_handler_apple/ios`)
   - shared_preferences_foundation (from `.symlinks/plugins/shared_preferences_foundation/darwin`)
   - sqflite (from `.symlinks/plugins/sqflite/darwin`)
+  - tflite_flutter (from `.symlinks/plugins/tflite_flutter/ios`)
+
+SPEC REPOS:
+  trunk:
+    - TensorFlowLiteC
+    - TensorFlowLiteSwift
 
 EXTERNAL SOURCES:
   app_settings:
@@ -55,6 +83,8 @@ EXTERNAL SOURCES:
     :path: ".symlinks/plugins/shared_preferences_foundation/darwin"
   sqflite:
     :path: ".symlinks/plugins/sqflite/darwin"
+  tflite_flutter:
+    :path: ".symlinks/plugins/tflite_flutter/ios"
 
 SPEC CHECKSUMS:
   app_settings: 017320c6a680cdc94c799949d95b84cb69389ebc
@@ -67,6 +97,9 @@ SPEC CHECKSUMS:
   permission_handler_apple: 9878588469a2b0d0fc1e048d9f43605f92e6cec2
   shared_preferences_foundation: fcdcbc04712aee1108ac7fda236f363274528f78
   sqflite: 673a0e54cc04b7d6dba8d24fb8095b31c3a99eec
+  TensorFlowLiteC: 20785a69299185a379ba9852b6625f00afd7984a
+  TensorFlowLiteSwift: 3a4928286e9e35bdd3e17970f48e53c80d25e793
+  tflite_flutter: 9433d086a3060431bbc9f3c7c20d017db0e72d08
 
 PODFILE CHECKSUM: 2890ea34b9e9ffb3bc0e2dccd6c1c1d3f8a39969
 

client/ios/Runner/Info.plist

@@ -56,4 +56,4 @@
     <key>UIViewControllerBasedStatusBarAppearance</key>
     <false/>
   </dict>
-</plist>
+</plist>

client/lib/core/di/di_provider.dart

@@ -1,6 +1,7 @@
 import 'package:get_it/get_it.dart';
 import 'package:simon_ai/core/di/app_providers_module.dart';
 import 'package:simon_ai/core/di/di_repository_module.dart';
+import 'package:simon_ai/core/di/di_utils_mobile_module.dart';
 import 'package:simon_ai/core/di/di_utils_module.dart';
 
 abstract class DiProvider {
@@ -10,6 +11,8 @@ abstract class DiProvider {
     // Setup app providers have to be done first
     await AppProvidersModule().setupModule(_instance);
     UtilsDiModule().setupModule(_instance);
+    // TODO add conditional import when having web support
+    PlatformUtilsDiModule().setupModule(_instance);
     RepositoryDiModule().setupModule(_instance);
     await _instance.allReady();
   }

client/lib/core/di/di_utils_mobile_module.dart

@@ -0,0 +1,25 @@
+import 'package:get_it/get_it.dart';
+import 'package:simon_ai/core/manager/keypoints/keypoints_manager.dart';
+import 'package:simon_ai/core/manager/keypoints/keypoints_manager_mobile.dart';
+
+class PlatformUtilsDiModule {
+  PlatformUtilsDiModule._privateConstructor();
+
+  static final PlatformUtilsDiModule _instance =
+      PlatformUtilsDiModule._privateConstructor();
+
+  factory PlatformUtilsDiModule() => _instance;
+
+  void setupModule(GetIt locator) {
+    locator._setupUtilsModule();
+  }
+}
+
+extension _GetItUseCaseDiModuleExtensions on GetIt {
+  void _setupUtilsModule() {
+    registerLazySingleton<KeyPointsMobileManager>(KeyPointsMobileManager.new);
+    registerLazySingleton<KeyPointsManager>(
+      () => get<KeyPointsMobileManager>(),
+    );
+  }
+}

client/lib/core/manager/keypoints/hand_tracking_classifier.dart

@@ -0,0 +1,162 @@
+import 'dart:io';
+import 'dart:math';
+
+import 'package:image/image.dart' as img;
+import 'package:simon_ai/core/common/logger.dart';
+import 'package:simon_ai/core/manager/keypoints/image_utils.dart';
+import 'package:simon_ai/core/manager/keypoints/keypoints_manager_mobile.dart';
+import 'package:simon_ai/gen/assets.gen.dart';
+import 'package:tflite_flutter/tflite_flutter.dart';
+
+class HandTrackingClassifier {
+  final bool _logInit = true;
+  final bool _logResultTime = false;
+
+  final int modelInputSize = 224;
+  final String modelName = Assets.models.handLandmarksDetector;
+
+  late Interpreter _interpreter;
+  Interpreter get interpreter => _interpreter;
+
+  Map<int, Object> outputs = {};
+  late Tensor outputTensor;
+
+  final stopwatch = Stopwatch();
+
+  HandTrackingClassifier({Interpreter? interpreter}) {
+    loadModel(interpreter: interpreter);
+  }
+
+  Future<Interpreter> _createModelInterpreter() {
+    final options = InterpreterOptions()..threads = 4;
+    if (Platform.isAndroid) {
+      options.addDelegate(
+        GpuDelegateV2(
+          options: GpuDelegateOptionsV2(
+            isPrecisionLossAllowed: false,
+            inferencePriority1: 2,
+          ),
+        ),
+      );
+    }
+    return Interpreter.fromAsset(
+      modelName,
+      options: options,
+    );
+  }
+
+  Future<void> loadModel({Interpreter? interpreter}) async {
+    try {
+      _interpreter = interpreter ?? await _createModelInterpreter();
+
+      if (_logInit && interpreter == null) {
+        final inputTensors = _interpreter.getInputTensors();
+        final outputTensors = _interpreter.getOutputTensors();
+        for (final tensor in outputTensors) {
+          Logger.d('Output Tensor: $tensor');
+        }
+        for (final tensor in inputTensors) {
+          Logger.d('Input Tensor: $tensor');
+        }
+        Logger.d('Interpreter loaded successfully');
+      }
+      outputTensor = _interpreter.getOutputTensors().first;
+    } catch (error) {
+      Logger.e('Error while creating interpreter: $error', error);
+    }
+  }
+
+  Future<HandLandmarksResultData> performOperations(img.Image image) async {
+    stopwatch.start();
+
+    final inputImage = ImageUtils.getProcessedImage(image, modelInputSize);
+    stopwatch.stop();
+    final processImageTime = stopwatch.elapsedMilliseconds;
+
+    stopwatch.start();
+    _runModel(inputImage);
+    final result = parseLandmarkData(image);
+
+    stopwatch.stop();
+    final processModelTime = stopwatch.elapsedMilliseconds;
+
+    if (_logResultTime) {
+      Logger.d('Process image time $processImageTime, '
+          'processModelTime: $processModelTime');
+    }
+
+    stopwatch.reset();
+    return result;
+  }
+
+  void _runModel(img.Image inputImage) {
+    final imageMatrix = List.generate(
+      inputImage.height,
+      (y) => List.generate(
+        inputImage.width,
+        (x) {
+          final pixel = inputImage.getPixel(x, y);
+          // Normalize pixel values to [0, 1]
+          return [pixel.r / 255.0, pixel.g / 255.0, pixel.b / 255.0];
+        },
+      ),
+    );
+    final inputs = [imageMatrix];
+    outputs = <int, Object>{
+      // Output 0: Presence of a hand in the image. A float scalar value.
+      0: [List<double>.filled(outputTensor.shape[1], 0.0)],
+      // Output 1: 21 3D screen landmarks normalized by image size.
+      // Represented as a 1x63 tensor.Only valid when the presence score
+      // (Output 0) is above a certain threshold.
+      1: [List<double>.filled(outputTensor.shape.first, 0.0)],
+      // Output 2: Handedness of the predicted hand. A float scalar value.
+      // Only valid when the presence score (Output 0) is above a certain
+      // threshold.
+      2: [List<double>.filled(outputTensor.shape.first, 0.0)],
+      // Output 3: 21 3D world landmarks based on the GHUM hand model.
+      // Represented as a 1x63 tensor.
+      // Only valid when the presence score (Output 0) is above a
+      // certain threshold.
+      3: [List<double>.filled(outputTensor.shape[1], 0.0)],
+    };
+    interpreter.runForMultipleInputs([inputs], outputs);
+  }
+
+  HandLandmarksResultData parseLandmarkData(img.Image image) {
+    final data = (outputs.values.first as List<List<double>>).first;
+    final confidence =
+        (outputs.values.elementAt(1) as List<List<double>>).first.first;
+    final result = <double>[];
+    double x;
+    double y;
+    double z;
+
+    final padSize = max(image.height, image.width);
+    final padY = max(0, (image.width - image.height) / 2);
+    final padX = max(0, (image.height - image.width) / 2);
+
+    const landmarksOutputDimensions = 63;
+
+    for (var i = 0; i < landmarksOutputDimensions; i += 3) {
+      final double padXRatio = padX / padSize;
+      final double padYRatio = padY / padSize;
+
+      final double normalizedPadX = padXRatio * modelInputSize;
+      final double normalizedPadY = padYRatio * modelInputSize;
+
+      final double adjustedModelInputSizeX =
+          modelInputSize - (2 * normalizedPadX);
+      final double adjustedModelInputSizeY =
+          modelInputSize - (2 * normalizedPadY);
+
+      final double normalizedDataX = data[0 + i] - normalizedPadX;
+      final double normalizedDataY = data[1 + i] - normalizedPadY;
+
+      x = (normalizedDataX / adjustedModelInputSizeX) * image.width;
+      y = (normalizedDataY / adjustedModelInputSizeY) * image.height;
+      z = data[2 + i];
+      result.addAll([y, x, z]);
+    }
+    return (confidence: confidence, keyPoints: result);
+  }
+}

client/lib/core/manager/keypoints/hand_tracking_isolate.dart

@@ -0,0 +1,67 @@
+import 'dart:io';
+import 'dart:isolate';
+
+import 'package:camera/camera.dart';
+import 'package:image/image.dart' as img;
+import 'package:simon_ai/core/common/logger.dart';
+import 'package:simon_ai/core/manager/keypoints/image_utils.dart';
+import 'package:tflite_flutter/tflite_flutter.dart';
+
+import 'package:simon_ai/core/manager/keypoints/hand_tracking_classifier.dart';
+
+class HandTrackingIsolateUtils {
+  static const _logTimes = false;
+
+  final ReceivePort _receivePort = ReceivePort();
+  late SendPort _sendPort;
+
+  SendPort get sendPort => _sendPort;
+
+  Future<void> start() async {
+    await Isolate.spawn<SendPort>(
+      entryPoint,
+      _receivePort.sendPort,
+      debugName: 'MoveNetIsolate',
+    );
+
+    _sendPort = await _receivePort.first;
+  }
+
+  static void entryPoint(SendPort sendPort) {
+    final port = ReceivePort();
+    sendPort.send(port.sendPort);
+
+    port.listen((data) {
+      if (data is IsolateData) {
+        final classifier = HandTrackingClassifier(
+          interpreter: Interpreter.fromAddress(data.interpreterAddress),
+        );
+        final stopwatch = Stopwatch()..start();
+        var image = ImageUtils.convertCameraImage(data.cameraImage)!;
+        if (Platform.isAndroid) {
+          image = img.copyRotate(image, angle: 270);
+          image = img.flipHorizontal(image);
+        }
+        stopwatch.stop();
+        final elapsedToProcessImage = stopwatch.elapsedMilliseconds;
+        stopwatch.start();
+
+        classifier.performOperations(image).then((result) {
+          data.responsePort.send(result);
+
+          if (_logTimes) {
+            Logger.d('Process image $elapsedToProcessImage ms, process model '
+                '${stopwatch.elapsedMilliseconds}ms');
+          }
+        });
+      }
+    });
+  }
+}
+
+/// Bundles data to pass between Isolate
+typedef IsolateData = ({
+  CameraImage cameraImage,
+  int interpreterAddress,
+  SendPort responsePort,
+});