diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.kotlin/sessions/kotlin-compiler-16408385281587638559.salive b/.kotlin/sessions/kotlin-compiler-2318937954061801521.salive similarity index 100% rename from .kotlin/sessions/kotlin-compiler-16408385281587638559.salive rename to .kotlin/sessions/kotlin-compiler-2318937954061801521.salive diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 6374db0..4654746 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -39,6 +39,10 @@ android { } buildFeatures { compose = true + mlModelBinding = true + } + aaptOptions { + noCompress += "tflite" } } @@ -61,6 +65,12 @@ dependencies { implementation("com.google.mlkit:object-detection:17.0.2") implementation("com.google.android.gms:play-services-mlkit-subject-segmentation:16.0.0-beta1") + // Tensorflow Lite + implementation(libs.tensorflow.lite.support) + implementation(libs.tensorflow.lite) + implementation(libs.tensorflow.lite.gpu) + implementation(libs.tensorflow.lite.task.vision) + //Koin implementation(libs.koin.android) implementation(libs.koin.androidx.compose) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 800685e..f9c81c7 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -25,6 +25,11 @@ android:roundIcon="@mipmap/ic_launcher_round" android:supportsRtl="true" android:theme="@style/LivingAI.Starting.Theme"> + + + { AIModelImpl() } + single { + ObjectDetectorImpl( + context = androidContext(), + onResults = { _, _ -> }, // Callback will be set by ViewModel + onError = { error -> Log.e("ObjectDetector", "Error: $error") } + ) + } + + single { FeedbackAnalyzerImpl(AnalyzerThresholds()) } + + single { TiltSensorManager(androidContext()) } // Repositories single { AnimalProfileRepositoryImpl(get()) } @@ -146,5 +164,5 @@ val appModule = module { viewModel { SettingsViewModel(get()) } viewModel { RatingViewModel(get(), get(), get(), get()) } viewModel { CameraViewModel(get(), get(), get(), get()) } - viewModel { VideoViewModel() } + viewModel { VideoViewModel(get(), get(), get()) } } diff --git a/app/src/main/java/com/example/livingai/domain/ml/FeedbackAnalyzer.kt b/app/src/main/java/com/example/livingai/domain/ml/FeedbackAnalyzer.kt new file mode 100644 index 0000000..be7b184 --- /dev/null +++ b/app/src/main/java/com/example/livingai/domain/ml/FeedbackAnalyzer.kt @@ -0,0 +1,150 @@ +package com.example.livingai.domain.ml + +import android.graphics.RectF +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.sin + +// ------------------------------------------------------------ +// CONFIG CLASS +// ------------------------------------------------------------ +data class AnalyzerThresholds( + // Size thresholds + val minCoverage: Float = 0.15f, + val maxCoverage: Float = 0.70f, + + // Vertical centering thresholds + val topMarginRatio: Float = 0.05f, + val bottomMarginRatio: Float = 0.05f, + val centerToleranceRatio: Float = 0.10f, + + // Height estimation thresholds + val targetHeightMeters: Float = 0.70f, + val heightToleranceMeters: Float = 0.05f, // ±5 cm allowed + + // Real height of subject (for estimating camera distance) + val subjectRealHeightMeters: Float = 1.70f // default human height or silhouette height +) + + +// ------------------------------------------------------------ +// STATES +// ------------------------------------------------------------ +sealed class FeedbackState(val message: String) { + object Idle : FeedbackState("") + object Searching : FeedbackState("Searching for subject...") + object TooFar : FeedbackState("Move closer") + object TooClose : FeedbackState("Move back") + object NotCentered : FeedbackState("Center the subject") + object TooLow : FeedbackState("Raise phone") + object TooHigh : FeedbackState("Lower phone") + object Optimal : FeedbackState("Hold still") +} + + +// ------------------------------------------------------------ +// ANALYZER +// ------------------------------------------------------------ +interface FeedbackAnalyzer { + fun analyze( + detection: ObjectDetector.DetectionResult?, + frameWidth: Int, + frameHeight: Int, + tiltDegrees: Float, // phone pitch for height estimation + focalLengthPx: Float // camera intrinsics for height estimation + ): FeedbackState +} + + +// ------------------------------------------------------------ +// IMPLEMENTATION +// ------------------------------------------------------------ +class FeedbackAnalyzerImpl( + private val thresholds: AnalyzerThresholds +) : FeedbackAnalyzer { + + override fun analyze( + detection: ObjectDetector.DetectionResult?, + frameWidth: Int, + frameHeight: Int, + tiltDegrees: Float, + focalLengthPx: Float + ): FeedbackState { + + if (detection == null) return FeedbackState.Searching + if (frameWidth <= 0 || frameHeight <= 0) return FeedbackState.Idle + + val box = detection.boundingBox + + val pc = Precomputed(box, frameWidth, frameHeight, thresholds) + + val cameraHeight = estimateCameraHeight(pc, tiltDegrees, focalLengthPx) + + return when { + isTooFar(pc) -> FeedbackState.TooFar + isTooClose(pc) -> FeedbackState.TooClose + isNotCentered(pc) -> FeedbackState.NotCentered + isHeightTooLow(cameraHeight) -> FeedbackState.TooLow + isHeightTooHigh(cameraHeight) -> FeedbackState.TooHigh + else -> FeedbackState.Optimal + } + } + + private fun isTooFar(pc: Precomputed) = + pc.verticalCoverage < thresholds.minCoverage + + private fun isTooClose(pc: Precomputed) = + pc.verticalCoverage > thresholds.maxCoverage + + private fun isNotCentered(pc: Precomputed) = + !pc.topWithinMargin || !pc.bottomWithinMargin || !pc.centeredVertically + + private fun isHeightTooLow(camHeight: Float) = + camHeight < (thresholds.targetHeightMeters - thresholds.heightToleranceMeters) + + private fun isHeightTooHigh(camHeight: Float) = + camHeight > (thresholds.targetHeightMeters + thresholds.heightToleranceMeters) + + private fun estimateCameraHeight( + pc: Precomputed, + tiltDegrees: Float, + focalLengthPx: Float + ): Float { + + val tiltRad = Math.toRadians(tiltDegrees.toDouble()) + + val realHeight = thresholds.subjectRealHeightMeters + val pixelHeight = pc.heightPx + + if (pixelHeight <= 0f || focalLengthPx <= 0f) return -1f + + val distance = (realHeight * focalLengthPx) / pixelHeight + + val height = distance * sin(tiltRad) + + return height.toFloat() // in meters + } + + private data class Precomputed( + val box: RectF, + val frameWidth: Int, + val frameHeight: Int, + val t: AnalyzerThresholds + ) { + val top = box.top + val bottom = box.bottom + val heightPx = max(0f, bottom - top) + val verticalCoverage: Float = heightPx / frameHeight + + private val topMarginPx = frameHeight * t.topMarginRatio + private val bottomMarginPx = frameHeight * t.bottomMarginRatio + private val centerTolerancePx = frameHeight * t.centerToleranceRatio + + val topWithinMargin = top >= topMarginPx + val bottomWithinMargin = bottom <= (frameHeight - bottomMarginPx) + + val centerY = box.centerY() + val frameCenterY = frameHeight / 2f + val centeredVertically = abs(centerY - frameCenterY) <= centerTolerancePx + } +} diff --git a/app/src/main/java/com/example/livingai/domain/ml/ObjectDetector.kt b/app/src/main/java/com/example/livingai/domain/ml/ObjectDetector.kt new file mode 100644 index 0000000..c226384 --- /dev/null +++ b/app/src/main/java/com/example/livingai/domain/ml/ObjectDetector.kt @@ -0,0 +1,12 @@ +package com.example.livingai.domain.ml + +import android.graphics.Bitmap +import android.graphics.RectF + +interface ObjectDetector { + var onResults: (List, Long) -> Unit + var onError: (String) -> Unit + fun detect(bitmap: Bitmap, imageRotation: Int) + + data class DetectionResult(val boundingBox: RectF, val text: String, val score: Float) +} diff --git a/app/src/main/java/com/example/livingai/domain/ml/ObjectDetectorImpl.kt b/app/src/main/java/com/example/livingai/domain/ml/ObjectDetectorImpl.kt new file mode 100644 index 0000000..8e404bc --- /dev/null +++ b/app/src/main/java/com/example/livingai/domain/ml/ObjectDetectorImpl.kt @@ -0,0 +1,119 @@ +package com.example.livingai.domain.ml + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.RectF +import org.tensorflow.lite.DataType +import org.tensorflow.lite.Interpreter +import org.tensorflow.lite.support.image.ImageProcessor +import org.tensorflow.lite.support.image.TensorImage +import org.tensorflow.lite.support.image.ops.ResizeOp +import org.tensorflow.lite.support.image.ops.Rot90Op +import java.io.FileInputStream +import java.nio.channels.FileChannel + +class ObjectDetectorImpl( + private val context: Context, + override var onResults: (List, Long) -> Unit, + override var onError: (String) -> Unit +) : ObjectDetector { + + private var interpreter: Interpreter? = null + private val modelName = "efficientdet-lite0.tflite" + private val inputSize = 320 // EfficientDet-Lite0 expects 320x320 + + init { + setupInterpreter() + } + + private fun setupInterpreter() { + try { + val assetFileDescriptor = context.assets.openFd(modelName) + val fileInputStream = FileInputStream(assetFileDescriptor.fileDescriptor) + val fileChannel = fileInputStream.channel + val startOffset = assetFileDescriptor.startOffset + val declaredLength = assetFileDescriptor.declaredLength + val modelBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength) + + val options = Interpreter.Options() + options.setNumThreads(4) + interpreter = Interpreter(modelBuffer, options) + } catch (e: Exception) { + onError(e.message ?: "Error loading model") + } + } + + override fun detect(bitmap: Bitmap, imageRotation: Int) { + val tflite = interpreter ?: return + + val startTime = System.currentTimeMillis() + + // 1. Preprocess Image + // Rotate -> Resize -> Convert to TensorImage (UINT8) + val imageProcessor = ImageProcessor.Builder() + .add(Rot90Op(-imageRotation / 90)) + .add(ResizeOp(inputSize, inputSize, ResizeOp.ResizeMethod.BILINEAR)) + .build() + + var tensorImage = TensorImage(DataType.UINT8) + tensorImage.load(bitmap) + tensorImage = imageProcessor.process(tensorImage) + + // 2. Prepare Output Buffers + val outputBoxes = Array(1) { Array(25) { FloatArray(4) } } + val outputClasses = Array(1) { FloatArray(25) } + val outputScores = Array(1) { FloatArray(25) } + val outputCount = FloatArray(1) + + val outputs = mapOf( + 0 to outputBoxes, + 1 to outputClasses, + 2 to outputScores, + 3 to outputCount + ) + + // 3. Run Inference + try { + tflite.runForMultipleInputsOutputs(arrayOf(tensorImage.buffer), outputs) + } catch (e: Exception) { + onError(e.message ?: "Inference failed") + return + } + + val inferenceTime = System.currentTimeMillis() - startTime + + // 4. Parse Results + val results = mutableListOf() + + // Calculate dimensions of the rotated image (the coordinate space of the detections) + val rotatedWidth = if (imageRotation % 180 == 0) bitmap.width else bitmap.height + val rotatedHeight = if (imageRotation % 180 == 0) bitmap.height else bitmap.width + + for (i in 0 until 25) { + val score = outputScores[0][i] + if (score < 0.4f) continue + + val classId = outputClasses[0][i].toInt() + val label = if (classId == 20) "Cow" else "Object $classId" + + // Get box: [ymin, xmin, ymax, xmax] (normalized 0..1) + val box = outputBoxes[0][i] + val ymin = box[0] + val xmin = box[1] + val ymax = box[2] + val xmax = box[3] + + // Scale to rotated image dimensions + val left = xmin * rotatedWidth + val top = ymin * rotatedHeight + val right = xmax * rotatedWidth + val bottom = ymax * rotatedHeight + + val boundingBox = RectF(left, top, right, bottom) + + results.add(ObjectDetector.DetectionResult(boundingBox, label, score)) + } + + onResults(results, inferenceTime) + } +} diff --git a/app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt b/app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt index ee7b271..10b35aa 100644 --- a/app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt +++ b/app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt @@ -4,7 +4,6 @@ import android.Manifest import android.annotation.SuppressLint import android.content.ContentValues import android.content.pm.ActivityInfo -import android.net.Uri import android.provider.MediaStore import androidx.camera.core.CameraSelector import androidx.camera.video.MediaStoreOutputOptions @@ -13,6 +12,7 @@ import androidx.camera.video.VideoRecordEvent import androidx.camera.view.CameraController import androidx.camera.view.LifecycleCameraController import androidx.camera.view.video.AudioConfig +import androidx.compose.foundation.Canvas import androidx.compose.foundation.background import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -28,7 +28,9 @@ import androidx.compose.material3.FabPosition import androidx.compose.material3.FloatingActionButton import androidx.compose.material3.FloatingActionButtonDefaults import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.collectAsState @@ -38,18 +40,24 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.Stroke import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import androidx.core.content.ContextCompat import androidx.navigation.NavController +import com.example.livingai.domain.ml.FeedbackState import com.example.livingai.pages.components.CameraPreview import com.example.livingai.pages.components.PermissionWrapper import com.example.livingai.pages.navigation.Route import com.example.livingai.ui.theme.RecordingRed +import com.example.livingai.utils.Constants import com.example.livingai.utils.SetScreenOrientation import org.koin.androidx.compose.koinViewModel -import java.io.File +import kotlin.math.max @SuppressLint("MissingPermission") @Composable @@ -63,6 +71,10 @@ fun VideoRecordScreen( val context = LocalContext.current val state by viewModel.state.collectAsState() var recording by remember { mutableStateOf(null) } + + var frameWidth by remember { mutableStateOf(0) } + var frameHeight by remember { mutableStateOf(0) } + var rotation by remember { mutableStateOf(0) } // We need RECORD_AUDIO permission for video with audio PermissionWrapper( @@ -73,7 +85,7 @@ fun VideoRecordScreen( ) { val controller = remember { LifecycleCameraController(context).apply { - setEnabledUseCases(CameraController.VIDEO_CAPTURE) + setEnabledUseCases(CameraController.VIDEO_CAPTURE or CameraController.IMAGE_ANALYSIS) cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA } } @@ -147,7 +159,12 @@ fun VideoRecordScreen( CameraPreview( modifier = Modifier.fillMaxSize(), controller = controller, - onFrame = { _, _ -> } + onFrame = { bitmap, imageRotation -> + frameWidth = bitmap.width + frameHeight = bitmap.height + rotation = imageRotation + viewModel.onEvent(VideoEvent.FrameReceived(bitmap, imageRotation)) + } ) // Overlay @@ -162,7 +179,7 @@ fun VideoRecordScreen( ) Spacer( modifier = Modifier - .height(300.dp) // Constant height for the transparent area + .height(Constants.VIDEO_SILHOETTE_FRAME_HEIGHT) // Constant height for the transparent area .fillMaxWidth() ) Box( @@ -172,6 +189,82 @@ fun VideoRecordScreen( .background(Color.Black.copy(alpha = 0.5f)) ) } + + // Feedback Text Overlay + if (state.feedback !is FeedbackState.Idle && state.feedback !is FeedbackState.Optimal) { + Box( + modifier = Modifier + .fillMaxSize() + .padding(top = 80.dp), + contentAlignment = Alignment.TopCenter + ) { + Text( + text = state.feedback.message, + color = Color.White, + style = MaterialTheme.typography.headlineMedium, + textAlign = TextAlign.Center, + modifier = Modifier + .background(Color.Black.copy(alpha = 0.6f)) + .padding(horizontal = 16.dp, vertical = 8.dp) + ) + } + } + + // Bounding Box Overlay + Canvas(modifier = Modifier.fillMaxSize()) { + val canvasWidth = size.width + val canvasHeight = size.height + + if (frameWidth > 0 && frameHeight > 0) { + // Determine source dimensions based on rotation + val isRotated = rotation == 90 || rotation == 270 + val sourceWidth = if (isRotated) frameHeight.toFloat() else frameWidth.toFloat() + val sourceHeight = if (isRotated) frameWidth.toFloat() else frameHeight.toFloat() + + // Calculate scale preserving aspect ratio (CenterCrop equivalent) + val scaleX = canvasWidth / sourceWidth + val scaleY = canvasHeight / sourceHeight + val scale = max(scaleX, scaleY) + + // Calculate the dimensions of the scaled image + val scaledWidth = sourceWidth * scale + val scaledHeight = sourceHeight * scale + + // Calculate the offset to center the image + val dx = (canvasWidth - scaledWidth) / 2 + val dy = (canvasHeight - scaledHeight) / 2 + + // Draw framing box (debug/guide) + val targetHeight = Constants.VIDEO_SILHOETTE_FRAME_HEIGHT.toPx() + val targetTop = (canvasHeight - targetHeight) / 2 + + drawRect( + color = Color.Blue, + topLeft = Offset(0f, targetTop), + size = Size(canvasWidth, targetHeight), + style = Stroke(width = 2.dp.toPx()) + ) + + state.cowDetection?.let { result -> + val box = result.boundingBox + + val boxColor = if (state.feedback is FeedbackState.Optimal) Color.Green else Color.Yellow + + drawRect( + color = boxColor, + topLeft = Offset( + x = box.left * scale + dx, + y = box.top * scale + dy + ), + size = Size( + width = box.width() * scale, + height = box.height() * scale + ), + style = Stroke(width = 3.dp.toPx()) + ) + } + } + } } } } diff --git a/app/src/main/java/com/example/livingai/pages/camera/VideoViewModel.kt b/app/src/main/java/com/example/livingai/pages/camera/VideoViewModel.kt index 8fab93e..3122aae 100644 --- a/app/src/main/java/com/example/livingai/pages/camera/VideoViewModel.kt +++ b/app/src/main/java/com/example/livingai/pages/camera/VideoViewModel.kt @@ -1,15 +1,68 @@ package com.example.livingai.pages.camera +import android.graphics.Bitmap import android.net.Uri +import android.util.Log import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import com.example.livingai.domain.ml.FeedbackAnalyzer +import com.example.livingai.domain.ml.FeedbackState +import com.example.livingai.domain.ml.ObjectDetector +import com.example.livingai.utils.TiltSensorManager import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch -class VideoViewModel : ViewModel() { +class VideoViewModel( + private val objectDetector: ObjectDetector, + private val feedbackAnalyzer: FeedbackAnalyzer, + private val tiltSensorManager: TiltSensorManager +) : ViewModel() { private val _state = MutableStateFlow(VideoState()) val state = _state.asStateFlow() + init { + tiltSensorManager.start() + + viewModelScope.launch { + tiltSensorManager.tilt.collect { (pitch, _, _) -> + _state.value = _state.value.copy(tilt = pitch) + } + } + + objectDetector.onResults = { results, _ -> + if (results.isNotEmpty()) { + Log.d( + "VideoViewModel", + "Detected objects: ${results.joinToString { "${it.text} (${it.score})" }}" + ) + } + + val cow = results.firstOrNull { it.text.contains("cow", ignoreCase = true) } + val feedback = feedbackAnalyzer.analyze( + detection = cow, + frameWidth = state.value.frameWidth, + frameHeight = state.value.frameHeight, + tiltDegrees = state.value.tilt, + focalLengthPx = state.value.focalLength + ) + _state.value = _state.value.copy( + cowDetection = cow, + feedback = feedback + ) + } + + objectDetector.onError = { error -> + Log.e("VideoViewModel", "Detection error: $error") + } + } + + override fun onCleared() { + super.onCleared() + tiltSensorManager.stop() + } + fun onEvent(event: VideoEvent) { when (event) { is VideoEvent.VideoRecorded -> { @@ -24,13 +77,28 @@ class VideoViewModel : ViewModel() { is VideoEvent.ClearRecordedVideo -> { _state.value = _state.value.copy(recordedVideoUri = null) } + is VideoEvent.FrameReceived -> { + // Update frame dimensions and focal length, but let TiltSensorManager handle tilt + _state.value = _state.value.copy( + frameWidth = event.bitmap.width, + frameHeight = event.bitmap.height, + focalLength = event.focalLength + ) + objectDetector.detect(event.bitmap, event.rotation) + } } } } data class VideoState( val isRecording: Boolean = false, - val recordedVideoUri: Uri? = null + val recordedVideoUri: Uri? = null, + val cowDetection: ObjectDetector.DetectionResult? = null, + val feedback: FeedbackState = FeedbackState.Idle, + val frameWidth: Int = 0, + val frameHeight: Int = 0, + val tilt: Float = 0f, + val focalLength: Float = 0f ) sealed class VideoEvent { @@ -38,4 +106,10 @@ sealed class VideoEvent { object StartRecording : VideoEvent() object StopRecording : VideoEvent() object ClearRecordedVideo : VideoEvent() + data class FrameReceived( + val bitmap: Bitmap, + val rotation: Int, + val tilt: Float = 0f, + val focalLength: Float = 0f + ) : VideoEvent() } diff --git a/app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt b/app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt index 65b029e..3e8aeb1 100644 --- a/app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt +++ b/app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt @@ -18,7 +18,7 @@ import java.util.concurrent.Executors fun CameraPreview( modifier: Modifier = Modifier, controller: LifecycleCameraController? = null, - onFrame: (Bitmap, Int) -> Unit + onFrame: ((Bitmap, Int) -> Unit)? = null ) { val context = LocalContext.current val lifecycleOwner = LocalLifecycleOwner.current @@ -26,12 +26,14 @@ fun CameraPreview( val cameraController = controller ?: remember { LifecycleCameraController(context) } - LaunchedEffect(cameraController) { - cameraController.setImageAnalysisAnalyzer(cameraExecutor) { imageProxy -> - val bitmap = imageProxy.toBitmap() - val rotationDegrees = imageProxy.imageInfo.rotationDegrees - onFrame(bitmap, rotationDegrees) - imageProxy.close() + if (onFrame != null) { + LaunchedEffect(cameraController) { + cameraController.setImageAnalysisAnalyzer(cameraExecutor) { imageProxy -> + val bitmap = imageProxy.toBitmap() + val rotationDegrees = imageProxy.imageInfo.rotationDegrees + onFrame(bitmap, rotationDegrees) + imageProxy.close() + } } } diff --git a/app/src/main/java/com/example/livingai/utils/Constants.kt b/app/src/main/java/com/example/livingai/utils/Constants.kt index a2d2e9c..04d8d14 100644 --- a/app/src/main/java/com/example/livingai/utils/Constants.kt +++ b/app/src/main/java/com/example/livingai/utils/Constants.kt @@ -1,5 +1,7 @@ package com.example.livingai.utils +import androidx.compose.ui.unit.dp + object Constants { const val USER_SETTINGS = "USER_SETTINGS" const val APP_ENTRY = "APP_ENTRY" @@ -15,5 +17,6 @@ object Constants { "angleview" ) + val VIDEO_SILHOETTE_FRAME_HEIGHT = 300.dp const val JACCARD_THRESHOLD = 85 } \ No newline at end of file diff --git a/app/src/main/java/com/example/livingai/utils/TiltSensorManager.kt b/app/src/main/java/com/example/livingai/utils/TiltSensorManager.kt new file mode 100644 index 0000000..8fd4c80 --- /dev/null +++ b/app/src/main/java/com/example/livingai/utils/TiltSensorManager.kt @@ -0,0 +1,58 @@ +package com.example.livingai.utils + +import android.content.Context +import android.hardware.Sensor +import android.hardware.SensorEvent +import android.hardware.SensorEventListener +import android.hardware.SensorManager +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +class TiltSensorManager( + context: Context +) : SensorEventListener { + + private val sensorManager = context.getSystemService(Context.SENSOR_SERVICE) as SensorManager + private val rotationVectorSensor = sensorManager.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR) + + private val _tilt = MutableStateFlow(Triple(0f, 0f, 0f)) // pitch, roll, azimuth + val tilt: StateFlow> = _tilt.asStateFlow() + + fun start() { + rotationVectorSensor?.let { + sensorManager.registerListener( + this, + it, + SensorManager.SENSOR_DELAY_GAME + ) + } + } + + fun stop() { + sensorManager.unregisterListener(this) + } + + override fun onSensorChanged(event: SensorEvent?) { + event ?: return + if (event.sensor.type != Sensor.TYPE_ROTATION_VECTOR) return + + val rotationMatrix = FloatArray(9) + val orientationAngles = FloatArray(3) + + // Convert rotation vector to rotation matrix + SensorManager.getRotationMatrixFromVector(rotationMatrix, event.values) + + // Convert rotation matrix to orientation angles + SensorManager.getOrientation(rotationMatrix, orientationAngles) + + // orientationAngles: [azimuth, pitch, roll] in radians + val azimuth = Math.toDegrees(orientationAngles[0].toDouble()).toFloat() + val pitch = Math.toDegrees(orientationAngles[1].toDouble()).toFloat() + val roll = Math.toDegrees(orientationAngles[2].toDouble()).toFloat() + + _tilt.value = Triple(pitch, roll, azimuth) + } + + override fun onAccuracyChanged(sensor: Sensor?, accuracy: Int) {} +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3133e04..2f6fae2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,6 +17,9 @@ ui = "1.9.5" kotlinxSerialization = "1.6.3" koin = "4.1.1" window = "1.5.1" +tensorflowLiteSupport = "0.4.4" +tensorflowLite = "2.14.0" +tensorflowLiteTaskVision = "0.4.4" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } @@ -44,9 +47,13 @@ kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx- koin-android = { group = "io.insert-koin", name = "koin-android", version.ref = "koin" } koin-androidx-compose = { group = "io.insert-koin", name = "koin-androidx-compose", version.ref = "koin" } androidx-window = { group = "androidx.window", name = "window", version.ref = "window" } +tensorflow-lite-support = { group = "org.tensorflow", name = "tensorflow-lite-support", version.ref = "tensorflowLiteSupport" } +tensorflow-lite = { group = "org.tensorflow", name = "tensorflow-lite", version.ref = "tensorflowLite" } +tensorflow-lite-gpu = { group = "org.tensorflow", name = "tensorflow-lite-gpu", version.ref = "tensorflowLite" } +tensorflow-lite-task-vision = { group = "org.tensorflow", name = "tensorflow-lite-task-vision", version.ref = "tensorflowLiteTaskVision" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } kotlin-compose = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } -kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } \ No newline at end of file +kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" }