video changes

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

app/build.gradle.kts

@@ -39,6 +39,10 @@ android {
     }
     buildFeatures {
         compose = true
+        mlModelBinding = true
+    }
+    aaptOptions {
+        noCompress += "tflite"
     }
 }
 
@@ -61,6 +65,12 @@ dependencies {
     implementation("com.google.mlkit:object-detection:17.0.2")
     implementation("com.google.android.gms:play-services-mlkit-subject-segmentation:16.0.0-beta1")
 
+    // Tensorflow Lite
+    implementation(libs.tensorflow.lite.support)
+    implementation(libs.tensorflow.lite)
+    implementation(libs.tensorflow.lite.gpu)
+    implementation(libs.tensorflow.lite.task.vision)
+
     //Koin
     implementation(libs.koin.android)
     implementation(libs.koin.androidx.compose)

app/src/main/AndroidManifest.xml

@@ -25,6 +25,11 @@
         android:roundIcon="@mipmap/ic_launcher_round"
         android:supportsRtl="true"
         android:theme="@style/LivingAI.Starting.Theme">
+
+        <meta-data
+            android:name="com.google.mlkit.vision.DEPENDENCIES"
+            android:value="obj" />
+            
         <activity
             android:name=".MainActivity"
             android:exported="true"

app/src/main/java/com/example/livingai/data/ml/AIModelImpl.kt

@@ -74,17 +74,4 @@ class AIModelImpl : AIModel {
         }
         return mask
     }
-
-    fun jaccardIndex(maskA: BooleanArray, maskB: BooleanArray): Double {
-        val len = max(maskA.size, maskB.size)
-        var inter = 0
-        var union = 0
-        for (i in 0 until len) {
-            val a = if (i < maskA.size) maskA[i] else false
-            val b = if (i < maskB.size) maskB[i] else false
-            if (a || b) union++
-            if (a && b) inter++
-        }
-        return if (union == 0) 0.0 else inter.toDouble() / union.toDouble()
-    }
 }

app/src/main/java/com/example/livingai/di/AppModule.kt

@@ -1,6 +1,7 @@
 package com.example.livingai.di
 
 import android.content.Context
+import android.util.Log
 import androidx.datastore.core.DataStore
 import androidx.datastore.preferences.core.Preferences
 import androidx.datastore.preferences.preferencesDataStore
@@ -16,6 +17,11 @@ import com.example.livingai.data.repository.business.AnimalRatingRepositoryImpl
 import com.example.livingai.data.repository.media.CameraRepositoryImpl
 import com.example.livingai.data.repository.media.VideoRepositoryImpl
 import com.example.livingai.domain.ml.AIModel
+import com.example.livingai.domain.ml.AnalyzerThresholds
+import com.example.livingai.domain.ml.FeedbackAnalyzer
+import com.example.livingai.domain.ml.FeedbackAnalyzerImpl
+import com.example.livingai.domain.ml.ObjectDetector
+import com.example.livingai.domain.ml.ObjectDetectorImpl
 import com.example.livingai.domain.repository.AppDataRepository
 import com.example.livingai.domain.repository.CameraRepository
 import com.example.livingai.domain.repository.VideoRepository
@@ -49,6 +55,7 @@ import com.example.livingai.utils.CoroutineDispatchers
 import com.example.livingai.utils.DefaultCoroutineDispatchers
 import com.example.livingai.utils.ScreenDimensions
 import com.example.livingai.utils.SilhouetteManager
+import com.example.livingai.utils.TiltSensorManager
 import org.koin.android.ext.koin.androidContext
 import org.koin.core.module.dsl.viewModel
 import org.koin.dsl.module
@@ -105,6 +112,17 @@ val appModule = module {
 
     // ML Model
     single<AIModel> { AIModelImpl() }
+    single<ObjectDetector> { 
+        ObjectDetectorImpl(
+            context = androidContext(),
+            onResults = { _, _ -> }, // Callback will be set by ViewModel
+            onError = { error -> Log.e("ObjectDetector", "Error: $error") }
+        )
+    }
+    
+    single<FeedbackAnalyzer> { FeedbackAnalyzerImpl(AnalyzerThresholds()) }
+
+    single { TiltSensorManager(androidContext()) }
 
     // Repositories
     single<AnimalProfileRepository> { AnimalProfileRepositoryImpl(get()) }
@@ -146,5 +164,5 @@ val appModule = module {
     viewModel { SettingsViewModel(get()) }
     viewModel { RatingViewModel(get(), get(), get(), get()) }
     viewModel { CameraViewModel(get(), get(), get(), get()) }
-    viewModel { VideoViewModel() }
+    viewModel { VideoViewModel(get(), get(), get()) }
 }

app/src/main/java/com/example/livingai/domain/ml/FeedbackAnalyzer.kt

@@ -0,0 +1,150 @@
+package com.example.livingai.domain.ml
+
+import android.graphics.RectF
+import kotlin.math.abs
+import kotlin.math.max
+import kotlin.math.sin
+
+// ------------------------------------------------------------
+//  CONFIG CLASS
+// ------------------------------------------------------------
+data class AnalyzerThresholds(
+    // Size thresholds
+    val minCoverage: Float = 0.15f,
+    val maxCoverage: Float = 0.70f,
+
+    // Vertical centering thresholds
+    val topMarginRatio: Float = 0.05f,
+    val bottomMarginRatio: Float = 0.05f,
+    val centerToleranceRatio: Float = 0.10f,
+
+    // Height estimation thresholds
+    val targetHeightMeters: Float = 0.70f,
+    val heightToleranceMeters: Float = 0.05f,   // ±5 cm allowed
+
+    // Real height of subject (for estimating camera distance)
+    val subjectRealHeightMeters: Float = 1.70f   // default human height or silhouette height
+)
+
+
+// ------------------------------------------------------------
+//  STATES
+// ------------------------------------------------------------
+sealed class FeedbackState(val message: String) {
+    object Idle : FeedbackState("")
+    object Searching : FeedbackState("Searching for subject...")
+    object TooFar : FeedbackState("Move closer")
+    object TooClose : FeedbackState("Move back")
+    object NotCentered : FeedbackState("Center the subject")
+    object TooLow : FeedbackState("Raise phone")
+    object TooHigh : FeedbackState("Lower phone")
+    object Optimal : FeedbackState("Hold still")
+}
+
+
+// ------------------------------------------------------------
+//  ANALYZER
+// ------------------------------------------------------------
+interface FeedbackAnalyzer {
+    fun analyze(
+        detection: ObjectDetector.DetectionResult?,
+        frameWidth: Int,
+        frameHeight: Int,
+        tiltDegrees: Float,        // phone pitch for height estimation
+        focalLengthPx: Float       // camera intrinsics for height estimation
+    ): FeedbackState
+}
+
+
+// ------------------------------------------------------------
+//  IMPLEMENTATION
+// ------------------------------------------------------------
+class FeedbackAnalyzerImpl(
+    private val thresholds: AnalyzerThresholds
+) : FeedbackAnalyzer {
+
+    override fun analyze(
+        detection: ObjectDetector.DetectionResult?,
+        frameWidth: Int,
+        frameHeight: Int,
+        tiltDegrees: Float,
+        focalLengthPx: Float
+    ): FeedbackState {
+
+        if (detection == null) return FeedbackState.Searching
+        if (frameWidth <= 0 || frameHeight <= 0) return FeedbackState.Idle
+
+        val box = detection.boundingBox
+
+        val pc = Precomputed(box, frameWidth, frameHeight, thresholds)
+
+        val cameraHeight = estimateCameraHeight(pc, tiltDegrees, focalLengthPx)
+
+        return when {
+            isTooFar(pc) -> FeedbackState.TooFar
+            isTooClose(pc) -> FeedbackState.TooClose
+            isNotCentered(pc) -> FeedbackState.NotCentered
+            isHeightTooLow(cameraHeight) -> FeedbackState.TooLow
+            isHeightTooHigh(cameraHeight) -> FeedbackState.TooHigh
+            else -> FeedbackState.Optimal
+        }
+    }
+
+    private fun isTooFar(pc: Precomputed) =
+        pc.verticalCoverage < thresholds.minCoverage
+
+    private fun isTooClose(pc: Precomputed) =
+        pc.verticalCoverage > thresholds.maxCoverage
+
+    private fun isNotCentered(pc: Precomputed) =
+        !pc.topWithinMargin || !pc.bottomWithinMargin || !pc.centeredVertically
+
+    private fun isHeightTooLow(camHeight: Float) =
+        camHeight < (thresholds.targetHeightMeters - thresholds.heightToleranceMeters)
+
+    private fun isHeightTooHigh(camHeight: Float) =
+        camHeight > (thresholds.targetHeightMeters + thresholds.heightToleranceMeters)
+
+    private fun estimateCameraHeight(
+        pc: Precomputed,
+        tiltDegrees: Float,
+        focalLengthPx: Float
+    ): Float {
+
+        val tiltRad = Math.toRadians(tiltDegrees.toDouble())
+
+        val realHeight = thresholds.subjectRealHeightMeters
+        val pixelHeight = pc.heightPx
+
+        if (pixelHeight <= 0f || focalLengthPx <= 0f) return -1f
+
+        val distance = (realHeight * focalLengthPx) / pixelHeight
+
+        val height = distance * sin(tiltRad)
+
+        return height.toFloat() // in meters
+    }
+
+    private data class Precomputed(
+        val box: RectF,
+        val frameWidth: Int,
+        val frameHeight: Int,
+        val t: AnalyzerThresholds
+    ) {
+        val top = box.top
+        val bottom = box.bottom
+        val heightPx = max(0f, bottom - top)
+        val verticalCoverage: Float = heightPx / frameHeight
+
+        private val topMarginPx = frameHeight * t.topMarginRatio
+        private val bottomMarginPx = frameHeight * t.bottomMarginRatio
+        private val centerTolerancePx = frameHeight * t.centerToleranceRatio
+
+        val topWithinMargin = top >= topMarginPx
+        val bottomWithinMargin = bottom <= (frameHeight - bottomMarginPx)
+
+        val centerY = box.centerY()
+        val frameCenterY = frameHeight / 2f
+        val centeredVertically = abs(centerY - frameCenterY) <= centerTolerancePx
+    }
+}

app/src/main/java/com/example/livingai/domain/ml/ObjectDetector.kt

@@ -0,0 +1,12 @@
+package com.example.livingai.domain.ml
+
+import android.graphics.Bitmap
+import android.graphics.RectF
+
+interface ObjectDetector {
+    var onResults: (List<DetectionResult>, Long) -> Unit
+    var onError: (String) -> Unit
+    fun detect(bitmap: Bitmap, imageRotation: Int)
+
+    data class DetectionResult(val boundingBox: RectF, val text: String, val score: Float)
+}

app/src/main/java/com/example/livingai/domain/ml/ObjectDetectorImpl.kt

@@ -0,0 +1,119 @@
+package com.example.livingai.domain.ml
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.graphics.RectF
+import org.tensorflow.lite.DataType
+import org.tensorflow.lite.Interpreter
+import org.tensorflow.lite.support.image.ImageProcessor
+import org.tensorflow.lite.support.image.TensorImage
+import org.tensorflow.lite.support.image.ops.ResizeOp
+import org.tensorflow.lite.support.image.ops.Rot90Op
+import java.io.FileInputStream
+import java.nio.channels.FileChannel
+
+class ObjectDetectorImpl(
+    private val context: Context,
+    override var onResults: (List<ObjectDetector.DetectionResult>, Long) -> Unit,
+    override var onError: (String) -> Unit
+) : ObjectDetector {
+
+    private var interpreter: Interpreter? = null
+    private val modelName = "efficientdet-lite0.tflite"
+    private val inputSize = 320 // EfficientDet-Lite0 expects 320x320
+
+    init {
+        setupInterpreter()
+    }
+
+    private fun setupInterpreter() {
+        try {
+            val assetFileDescriptor = context.assets.openFd(modelName)
+            val fileInputStream = FileInputStream(assetFileDescriptor.fileDescriptor)
+            val fileChannel = fileInputStream.channel
+            val startOffset = assetFileDescriptor.startOffset
+            val declaredLength = assetFileDescriptor.declaredLength
+            val modelBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength)
+            
+            val options = Interpreter.Options()
+            options.setNumThreads(4)
+            interpreter = Interpreter(modelBuffer, options)
+        } catch (e: Exception) {
+            onError(e.message ?: "Error loading model")
+        }
+    }
+
+    override fun detect(bitmap: Bitmap, imageRotation: Int) {
+        val tflite = interpreter ?: return
+
+        val startTime = System.currentTimeMillis()
+
+        // 1. Preprocess Image
+        // Rotate -> Resize -> Convert to TensorImage (UINT8)
+        val imageProcessor = ImageProcessor.Builder()
+            .add(Rot90Op(-imageRotation / 90))
+            .add(ResizeOp(inputSize, inputSize, ResizeOp.ResizeMethod.BILINEAR))
+            .build()
+
+        var tensorImage = TensorImage(DataType.UINT8)
+        tensorImage.load(bitmap)
+        tensorImage = imageProcessor.process(tensorImage)
+
+        // 2. Prepare Output Buffers
+        val outputBoxes = Array(1) { Array(25) { FloatArray(4) } }
+        val outputClasses = Array(1) { FloatArray(25) }
+        val outputScores = Array(1) { FloatArray(25) }
+        val outputCount = FloatArray(1)
+
+        val outputs = mapOf(
+            0 to outputBoxes,
+            1 to outputClasses,
+            2 to outputScores,
+            3 to outputCount
+        )
+
+        // 3. Run Inference
+        try {
+            tflite.runForMultipleInputsOutputs(arrayOf(tensorImage.buffer), outputs)
+        } catch (e: Exception) {
+            onError(e.message ?: "Inference failed")
+            return
+        }
+
+        val inferenceTime = System.currentTimeMillis() - startTime
+
+        // 4. Parse Results
+        val results = mutableListOf<ObjectDetector.DetectionResult>()
+        
+        // Calculate dimensions of the rotated image (the coordinate space of the detections)
+        val rotatedWidth = if (imageRotation % 180 == 0) bitmap.width else bitmap.height
+        val rotatedHeight = if (imageRotation % 180 == 0) bitmap.height else bitmap.width
+
+        for (i in 0 until 25) {
+            val score = outputScores[0][i]
+            if (score < 0.4f) continue 
+
+            val classId = outputClasses[0][i].toInt()
+            val label = if (classId == 20) "Cow" else "Object $classId"
+            
+            // Get box: [ymin, xmin, ymax, xmax] (normalized 0..1)
+            val box = outputBoxes[0][i]
+            val ymin = box[0]
+            val xmin = box[1]
+            val ymax = box[2]
+            val xmax = box[3]
+
+            // Scale to rotated image dimensions
+            val left = xmin * rotatedWidth
+            val top = ymin * rotatedHeight
+            val right = xmax * rotatedWidth
+            val bottom = ymax * rotatedHeight
+
+            val boundingBox = RectF(left, top, right, bottom)
+            
+            results.add(ObjectDetector.DetectionResult(boundingBox, label, score))
+        }
+
+        onResults(results, inferenceTime)
+    }
+}

app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt

@@ -4,7 +4,6 @@ import android.Manifest
 import android.annotation.SuppressLint
 import android.content.ContentValues
 import android.content.pm.ActivityInfo
-import android.net.Uri
 import android.provider.MediaStore
 import androidx.camera.core.CameraSelector
 import androidx.camera.video.MediaStoreOutputOptions
@@ -13,6 +12,7 @@ import androidx.camera.video.VideoRecordEvent
 import androidx.camera.view.CameraController
 import androidx.camera.view.LifecycleCameraController
 import androidx.camera.view.video.AudioConfig
+import androidx.compose.foundation.Canvas
 import androidx.compose.foundation.background
 import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.Column
@@ -28,7 +28,9 @@ import androidx.compose.material3.FabPosition
 import androidx.compose.material3.FloatingActionButton
 import androidx.compose.material3.FloatingActionButtonDefaults
 import androidx.compose.material3.Icon
+import androidx.compose.material3.MaterialTheme
 import androidx.compose.material3.Scaffold
+import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.LaunchedEffect
 import androidx.compose.runtime.collectAsState
@@ -38,18 +40,24 @@ import androidx.compose.runtime.remember
 import androidx.compose.runtime.setValue
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
+import androidx.compose.ui.geometry.Offset
+import androidx.compose.ui.geometry.Size
 import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.graphics.drawscope.Stroke
 import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.text.style.TextAlign
 import androidx.compose.ui.unit.dp
 import androidx.core.content.ContextCompat
 import androidx.navigation.NavController
+import com.example.livingai.domain.ml.FeedbackState
 import com.example.livingai.pages.components.CameraPreview
 import com.example.livingai.pages.components.PermissionWrapper
 import com.example.livingai.pages.navigation.Route
 import com.example.livingai.ui.theme.RecordingRed
+import com.example.livingai.utils.Constants
 import com.example.livingai.utils.SetScreenOrientation
 import org.koin.androidx.compose.koinViewModel
-import java.io.File
+import kotlin.math.max
 
 @SuppressLint("MissingPermission")
 @Composable
@@ -64,6 +72,10 @@ fun VideoRecordScreen(
     val state by viewModel.state.collectAsState()
     var recording by remember { mutableStateOf<Recording?>(null) }
     
+    var frameWidth by remember { mutableStateOf(0) }
+    var frameHeight by remember { mutableStateOf(0) }
+    var rotation by remember { mutableStateOf(0) }
+
     // We need RECORD_AUDIO permission for video with audio
     PermissionWrapper(
         permissions = listOf(
@@ -73,7 +85,7 @@ fun VideoRecordScreen(
     ) { 
         val controller = remember {
             LifecycleCameraController(context).apply {
-                setEnabledUseCases(CameraController.VIDEO_CAPTURE)
+                setEnabledUseCases(CameraController.VIDEO_CAPTURE or CameraController.IMAGE_ANALYSIS)
                 cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
             }
         }
@@ -147,7 +159,12 @@ fun VideoRecordScreen(
                 CameraPreview(
                     modifier = Modifier.fillMaxSize(),
                     controller = controller,
-                    onFrame = { _, _ -> }
+                    onFrame = { bitmap, imageRotation ->
+                        frameWidth = bitmap.width
+                        frameHeight = bitmap.height
+                        rotation = imageRotation
+                        viewModel.onEvent(VideoEvent.FrameReceived(bitmap, imageRotation))
+                    }
                 )
                 
                 // Overlay
@@ -162,7 +179,7 @@ fun VideoRecordScreen(
                     )
                     Spacer(
                         modifier = Modifier
-                            .height(300.dp) // Constant height for the transparent area
+                            .height(Constants.VIDEO_SILHOETTE_FRAME_HEIGHT) // Constant height for the transparent area
                             .fillMaxWidth()
                     )
                     Box(
@@ -172,6 +189,82 @@ fun VideoRecordScreen(
                             .background(Color.Black.copy(alpha = 0.5f))
                     )
                 }
+
+                // Feedback Text Overlay
+                if (state.feedback !is FeedbackState.Idle && state.feedback !is FeedbackState.Optimal) {
+                    Box(
+                        modifier = Modifier
+                            .fillMaxSize()
+                            .padding(top = 80.dp),
+                        contentAlignment = Alignment.TopCenter
+                    ) {
+                        Text(
+                            text = state.feedback.message,
+                            color = Color.White,
+                            style = MaterialTheme.typography.headlineMedium,
+                            textAlign = TextAlign.Center,
+                            modifier = Modifier
+                                .background(Color.Black.copy(alpha = 0.6f))
+                                .padding(horizontal = 16.dp, vertical = 8.dp)
+                        )
+                    }
+                }
+
+                // Bounding Box Overlay
+                Canvas(modifier = Modifier.fillMaxSize()) {
+                    val canvasWidth = size.width
+                    val canvasHeight = size.height
+                    
+                    if (frameWidth > 0 && frameHeight > 0) {
+                        // Determine source dimensions based on rotation
+                        val isRotated = rotation == 90 || rotation == 270
+                        val sourceWidth = if (isRotated) frameHeight.toFloat() else frameWidth.toFloat()
+                        val sourceHeight = if (isRotated) frameWidth.toFloat() else frameHeight.toFloat()
+
+                        // Calculate scale preserving aspect ratio (CenterCrop equivalent)
+                        val scaleX = canvasWidth / sourceWidth
+                        val scaleY = canvasHeight / sourceHeight
+                        val scale = max(scaleX, scaleY)
+
+                        // Calculate the dimensions of the scaled image
+                        val scaledWidth = sourceWidth * scale
+                        val scaledHeight = sourceHeight * scale
+
+                        // Calculate the offset to center the image
+                        val dx = (canvasWidth - scaledWidth) / 2
+                        val dy = (canvasHeight - scaledHeight) / 2
+
+                        // Draw framing box (debug/guide)
+                        val targetHeight = Constants.VIDEO_SILHOETTE_FRAME_HEIGHT.toPx()
+                        val targetTop = (canvasHeight - targetHeight) / 2
+                        
+                        drawRect(
+                            color = Color.Blue,
+                            topLeft = Offset(0f, targetTop),
+                            size = Size(canvasWidth, targetHeight),
+                            style = Stroke(width = 2.dp.toPx())
+                        )
+
+                        state.cowDetection?.let { result ->
+                            val box = result.boundingBox
+                            
+                            val boxColor = if (state.feedback is FeedbackState.Optimal) Color.Green else Color.Yellow
+                            
+                            drawRect(
+                                color = boxColor,
+                                topLeft = Offset(
+                                    x = box.left * scale + dx,
+                                    y = box.top * scale + dy
+                                ),
+                                size = Size(
+                                    width = box.width() * scale,
+                                    height = box.height() * scale
+                                ),
+                                style = Stroke(width = 3.dp.toPx())
+                            )
+                        }
+                    }
+                }
             }
         }
     }

app/src/main/java/com/example/livingai/pages/camera/VideoViewModel.kt

@@ -1,15 +1,68 @@
 package com.example.livingai.pages.camera
 
+import android.graphics.Bitmap
 import android.net.Uri
+import android.util.Log
 import androidx.lifecycle.ViewModel
+import androidx.lifecycle.viewModelScope
+import com.example.livingai.domain.ml.FeedbackAnalyzer
+import com.example.livingai.domain.ml.FeedbackState
+import com.example.livingai.domain.ml.ObjectDetector
+import com.example.livingai.utils.TiltSensorManager
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.asStateFlow
+import kotlinx.coroutines.launch
 
-class VideoViewModel : ViewModel() {
+class VideoViewModel(
+    private val objectDetector: ObjectDetector,
+    private val feedbackAnalyzer: FeedbackAnalyzer,
+    private val tiltSensorManager: TiltSensorManager
+) : ViewModel() {
 
     private val _state = MutableStateFlow(VideoState())
     val state = _state.asStateFlow()
 
+    init {
+        tiltSensorManager.start()
+
+        viewModelScope.launch {
+            tiltSensorManager.tilt.collect { (pitch, _, _) ->
+                _state.value = _state.value.copy(tilt = pitch)
+            }
+        }
+
+        objectDetector.onResults = { results, _ ->
+            if (results.isNotEmpty()) {
+                Log.d(
+                    "VideoViewModel",
+                    "Detected objects: ${results.joinToString { "${it.text} (${it.score})" }}"
+                )
+            }
+
+            val cow = results.firstOrNull { it.text.contains("cow", ignoreCase = true) }
+            val feedback = feedbackAnalyzer.analyze(
+                detection = cow,
+                frameWidth = state.value.frameWidth,
+                frameHeight = state.value.frameHeight,
+                tiltDegrees = state.value.tilt,
+                focalLengthPx = state.value.focalLength
+            )
+            _state.value = _state.value.copy(
+                cowDetection = cow,
+                feedback = feedback
+            )
+        }
+
+        objectDetector.onError = { error ->
+            Log.e("VideoViewModel", "Detection error: $error")
+        }
+    }
+
+    override fun onCleared() {
+        super.onCleared()
+        tiltSensorManager.stop()
+    }
+
     fun onEvent(event: VideoEvent) {
         when (event) {
             is VideoEvent.VideoRecorded -> {
@@ -24,13 +77,28 @@ class VideoViewModel : ViewModel() {
             is VideoEvent.ClearRecordedVideo -> {
                 _state.value = _state.value.copy(recordedVideoUri = null)
             }
+            is VideoEvent.FrameReceived -> {
+                // Update frame dimensions and focal length, but let TiltSensorManager handle tilt
+                _state.value = _state.value.copy(
+                    frameWidth = event.bitmap.width,
+                    frameHeight = event.bitmap.height,
+                    focalLength = event.focalLength
+                )
+                objectDetector.detect(event.bitmap, event.rotation)
+            }
         }
     }
 }
 
 data class VideoState(
     val isRecording: Boolean = false,
-    val recordedVideoUri: Uri? = null
+    val recordedVideoUri: Uri? = null,
+    val cowDetection: ObjectDetector.DetectionResult? = null,
+    val feedback: FeedbackState = FeedbackState.Idle,
+    val frameWidth: Int = 0,
+    val frameHeight: Int = 0,
+    val tilt: Float = 0f,
+    val focalLength: Float = 0f
 )
 
 sealed class VideoEvent {
@@ -38,4 +106,10 @@ sealed class VideoEvent {
     object StartRecording : VideoEvent()
     object StopRecording : VideoEvent()
     object ClearRecordedVideo : VideoEvent()
+    data class FrameReceived(
+        val bitmap: Bitmap, 
+        val rotation: Int,
+        val tilt: Float = 0f,
+        val focalLength: Float = 0f
+    ) : VideoEvent()
 }

app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt

@@ -18,7 +18,7 @@ import java.util.concurrent.Executors
 fun CameraPreview(
     modifier: Modifier = Modifier,
     controller: LifecycleCameraController? = null,
-    onFrame: (Bitmap, Int) -> Unit
+    onFrame: ((Bitmap, Int) -> Unit)? = null
 ) {
     val context = LocalContext.current
     val lifecycleOwner = LocalLifecycleOwner.current
@@ -26,12 +26,14 @@ fun CameraPreview(
 
     val cameraController = controller ?: remember { LifecycleCameraController(context) }
 
-    LaunchedEffect(cameraController) {
+    if (onFrame != null) {
-        cameraController.setImageAnalysisAnalyzer(cameraExecutor) { imageProxy ->
+        LaunchedEffect(cameraController) {
-            val bitmap = imageProxy.toBitmap()
+            cameraController.setImageAnalysisAnalyzer(cameraExecutor) { imageProxy ->
-            val rotationDegrees = imageProxy.imageInfo.rotationDegrees
+                val bitmap = imageProxy.toBitmap()
-            onFrame(bitmap, rotationDegrees)
+                val rotationDegrees = imageProxy.imageInfo.rotationDegrees
-            imageProxy.close()
+                onFrame(bitmap, rotationDegrees)
+                imageProxy.close()
+            }
         }
     }
     

app/src/main/java/com/example/livingai/utils/Constants.kt

@@ -1,5 +1,7 @@
 package com.example.livingai.utils
 
+import androidx.compose.ui.unit.dp
+
 object Constants {
     const val USER_SETTINGS = "USER_SETTINGS"
     const val APP_ENTRY = "APP_ENTRY"
@@ -15,5 +17,6 @@ object Constants {
         "angleview"
     )
 
+    val VIDEO_SILHOETTE_FRAME_HEIGHT = 300.dp
     const val JACCARD_THRESHOLD = 85
 }

app/src/main/java/com/example/livingai/utils/TiltSensorManager.kt

@@ -0,0 +1,58 @@
+package com.example.livingai.utils
+
+import android.content.Context
+import android.hardware.Sensor
+import android.hardware.SensorEvent
+import android.hardware.SensorEventListener
+import android.hardware.SensorManager
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
+
+class TiltSensorManager(
+    context: Context
+) : SensorEventListener {
+
+    private val sensorManager = context.getSystemService(Context.SENSOR_SERVICE) as SensorManager
+    private val rotationVectorSensor = sensorManager.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR)
+
+    private val _tilt = MutableStateFlow(Triple(0f, 0f, 0f)) // pitch, roll, azimuth
+    val tilt: StateFlow<Triple<Float, Float, Float>> = _tilt.asStateFlow()
+
+    fun start() {
+        rotationVectorSensor?.let {
+            sensorManager.registerListener(
+                this,
+                it,
+                SensorManager.SENSOR_DELAY_GAME
+            )
+        }
+    }
+
+    fun stop() {
+        sensorManager.unregisterListener(this)
+    }
+
+    override fun onSensorChanged(event: SensorEvent?) {
+        event ?: return
+        if (event.sensor.type != Sensor.TYPE_ROTATION_VECTOR) return
+
+        val rotationMatrix = FloatArray(9)
+        val orientationAngles = FloatArray(3)
+
+        // Convert rotation vector to rotation matrix
+        SensorManager.getRotationMatrixFromVector(rotationMatrix, event.values)
+
+        // Convert rotation matrix to orientation angles
+        SensorManager.getOrientation(rotationMatrix, orientationAngles)
+
+        // orientationAngles: [azimuth, pitch, roll] in radians
+        val azimuth = Math.toDegrees(orientationAngles[0].toDouble()).toFloat()
+        val pitch = Math.toDegrees(orientationAngles[1].toDouble()).toFloat()
+        val roll = Math.toDegrees(orientationAngles[2].toDouble()).toFloat()
+
+        _tilt.value = Triple(pitch, roll, azimuth)
+    }
+
+    override fun onAccuracyChanged(sensor: Sensor?, accuracy: Int) {}
+}

gradle/libs.versions.toml

@@ -17,6 +17,9 @@ ui = "1.9.5"
 kotlinxSerialization = "1.6.3"
 koin = "4.1.1"
 window = "1.5.1"
+tensorflowLiteSupport = "0.4.4"
+tensorflowLite = "2.14.0"
+tensorflowLiteTaskVision = "0.4.4"
 
 [libraries]
 androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" }
@@ -44,6 +47,10 @@ kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-
 koin-android = { group = "io.insert-koin", name = "koin-android", version.ref = "koin" }
 koin-androidx-compose = { group = "io.insert-koin", name = "koin-androidx-compose", version.ref = "koin" }
 androidx-window = { group = "androidx.window", name = "window", version.ref = "window" }
+tensorflow-lite-support = { group = "org.tensorflow", name = "tensorflow-lite-support", version.ref = "tensorflowLiteSupport" }
+tensorflow-lite = { group = "org.tensorflow", name = "tensorflow-lite", version.ref = "tensorflowLite" }
+tensorflow-lite-gpu = { group = "org.tensorflow", name = "tensorflow-lite-gpu", version.ref = "tensorflowLite" }
+tensorflow-lite-task-vision = { group = "org.tensorflow", name = "tensorflow-lite-task-vision", version.ref = "tensorflowLiteTaskVision" }
 
 [plugins]
 android-application = { id = "com.android.application", version.ref = "agp" }