LivingAi
/
AnimalRating
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

video changes

This commit is contained in:
SaiD 2025-12-05 11:46:39 +05:30
parent 61f60ba11a
commit 4ba7aa398d
16 changed files with 573 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

0
.kotlin/sessions/kotlin-compiler-16408385281587638559.salive → .kotlin/sessions/kotlin-compiler-2318937954061801521.salive
View File

10
app/build.gradle.kts
View File

@ -39,6 +39,10 @@ android {
}
buildFeatures {
compose = true
mlModelBinding = true
}
aaptOptions {
noCompress += "tflite"
}
}
@ -61,6 +65,12 @@ dependencies {
implementation("com.google.mlkit:object-detection:17.0.2")
implementation("com.google.android.gms:play-services-mlkit-subject-segmentation:16.0.0-beta1")
// Tensorflow Lite
implementation(libs.tensorflow.lite.support)
implementation(libs.tensorflow.lite)
implementation(libs.tensorflow.lite.gpu)
implementation(libs.tensorflow.lite.task.vision)
//Koin
implementation(libs.koin.android)
implementation(libs.koin.androidx.compose)

5
app/src/main/AndroidManifest.xml
View File

@ -25,6 +25,11 @@
android:roundIcon="@mipmap/ic_launcher_round"
android:supportsRtl="true"
android:theme="@style/LivingAI.Starting.Theme">
<meta-data
android:name="com.google.mlkit.vision.DEPENDENCIES"
android:value="obj" />
<activity
android:name=".MainActivity"
android:exported="true"

BIN
app/src/main/assets/efficientdet-lite0.tflite Normal file
View File

Binary file not shown.

13
app/src/main/java/com/example/livingai/data/ml/AIModelImpl.kt
View File

@ -74,17 +74,4 @@ class AIModelImpl : AIModel {
}
return mask
}
fun jaccardIndex(maskA: BooleanArray, maskB: BooleanArray): Double {
val len = max(maskA.size, maskB.size)
var inter = 0
var union = 0
for (i in 0 until len) {
val a = if (i < maskA.size) maskA[i] else false
val b = if (i < maskB.size) maskB[i] else false
if (a || b) union++
if (a && b) inter++
}
return if (union == 0) 0.0 else inter.toDouble() / union.toDouble()
}
}

20
app/src/main/java/com/example/livingai/di/AppModule.kt
View File

@ -1,6 +1,7 @@
package com.example.livingai.di
import android.content.Context
import android.util.Log
import androidx.datastore.core.DataStore
import androidx.datastore.preferences.core.Preferences
import androidx.datastore.preferences.preferencesDataStore
@ -16,6 +17,11 @@ import com.example.livingai.data.repository.business.AnimalRatingRepositoryImpl
import com.example.livingai.data.repository.media.CameraRepositoryImpl
import com.example.livingai.data.repository.media.VideoRepositoryImpl
import com.example.livingai.domain.ml.AIModel
import com.example.livingai.domain.ml.AnalyzerThresholds
import com.example.livingai.domain.ml.FeedbackAnalyzer
import com.example.livingai.domain.ml.FeedbackAnalyzerImpl
import com.example.livingai.domain.ml.ObjectDetector
import com.example.livingai.domain.ml.ObjectDetectorImpl
import com.example.livingai.domain.repository.AppDataRepository
import com.example.livingai.domain.repository.CameraRepository
import com.example.livingai.domain.repository.VideoRepository
@ -49,6 +55,7 @@ import com.example.livingai.utils.CoroutineDispatchers
import com.example.livingai.utils.DefaultCoroutineDispatchers
import com.example.livingai.utils.ScreenDimensions
import com.example.livingai.utils.SilhouetteManager
import com.example.livingai.utils.TiltSensorManager
import org.koin.android.ext.koin.androidContext
import org.koin.core.module.dsl.viewModel
import org.koin.dsl.module
@ -105,6 +112,17 @@ val appModule = module {
// ML Model
single<AIModel> { AIModelImpl() }
single<ObjectDetector> {
ObjectDetectorImpl(
context = androidContext(),
onResults = { _, _ -> }, // Callback will be set by ViewModel
onError = { error -> Log.e("ObjectDetector", "Error: $error") }
)
}
single<FeedbackAnalyzer> { FeedbackAnalyzerImpl(AnalyzerThresholds()) }
single { TiltSensorManager(androidContext()) }
// Repositories
single<AnimalProfileRepository> { AnimalProfileRepositoryImpl(get()) }
@ -146,5 +164,5 @@ val appModule = module {
viewModel { SettingsViewModel(get()) }
viewModel { RatingViewModel(get(), get(), get(), get()) }
viewModel { CameraViewModel(get(), get(), get(), get()) }
viewModel { VideoViewModel() }
viewModel { VideoViewModel(get(), get(), get()) }
}

150
app/src/main/java/com/example/livingai/domain/ml/FeedbackAnalyzer.kt Normal file
View File

@ -0,0 +1,150 @@
package com.example.livingai.domain.ml
import android.graphics.RectF
import kotlin.math.abs
import kotlin.math.max
import kotlin.math.sin
// ------------------------------------------------------------
// CONFIG CLASS
// ------------------------------------------------------------
data class AnalyzerThresholds(
// Size thresholds
val minCoverage: Float = 0.15f,
val maxCoverage: Float = 0.70f,
// Vertical centering thresholds
val topMarginRatio: Float = 0.05f,
val bottomMarginRatio: Float = 0.05f,
val centerToleranceRatio: Float = 0.10f,
// Height estimation thresholds
val targetHeightMeters: Float = 0.70f,
val heightToleranceMeters: Float = 0.05f, // ±5 cm allowed
// Real height of subject (for estimating camera distance)
val subjectRealHeightMeters: Float = 1.70f // default human height or silhouette height
)
// ------------------------------------------------------------
// STATES
// ------------------------------------------------------------
sealed class FeedbackState(val message: String) {
object Idle : FeedbackState("")
object Searching : FeedbackState("Searching for subject...")
object TooFar : FeedbackState("Move closer")
object TooClose : FeedbackState("Move back")
object NotCentered : FeedbackState("Center the subject")
object TooLow : FeedbackState("Raise phone")
object TooHigh : FeedbackState("Lower phone")
object Optimal : FeedbackState("Hold still")
}
// ------------------------------------------------------------
// ANALYZER
// ------------------------------------------------------------
interface FeedbackAnalyzer {
fun analyze(
detection: ObjectDetector.DetectionResult?,
frameWidth: Int,
frameHeight: Int,
tiltDegrees: Float, // phone pitch for height estimation
focalLengthPx: Float // camera intrinsics for height estimation
): FeedbackState
}
// ------------------------------------------------------------
// IMPLEMENTATION
// ------------------------------------------------------------
class FeedbackAnalyzerImpl(
private val thresholds: AnalyzerThresholds
) : FeedbackAnalyzer {
override fun analyze(
detection: ObjectDetector.DetectionResult?,
frameWidth: Int,
frameHeight: Int,
tiltDegrees: Float,
focalLengthPx: Float
): FeedbackState {
if (detection == null) return FeedbackState.Searching
if (frameWidth <= 0 || frameHeight <= 0) return FeedbackState.Idle
val box = detection.boundingBox
val pc = Precomputed(box, frameWidth, frameHeight, thresholds)
val cameraHeight = estimateCameraHeight(pc, tiltDegrees, focalLengthPx)
return when {
isTooFar(pc) -> FeedbackState.TooFar
isTooClose(pc) -> FeedbackState.TooClose
isNotCentered(pc) -> FeedbackState.NotCentered
isHeightTooLow(cameraHeight) -> FeedbackState.TooLow
isHeightTooHigh(cameraHeight) -> FeedbackState.TooHigh
else -> FeedbackState.Optimal
}
}
private fun isTooFar(pc: Precomputed) =
pc.verticalCoverage < thresholds.minCoverage
private fun isTooClose(pc: Precomputed) =
pc.verticalCoverage > thresholds.maxCoverage
private fun isNotCentered(pc: Precomputed) =
!pc.topWithinMargin || !pc.bottomWithinMargin || !pc.centeredVertically
private fun isHeightTooLow(camHeight: Float) =
camHeight < (thresholds.targetHeightMeters - thresholds.heightToleranceMeters)
private fun isHeightTooHigh(camHeight: Float) =
camHeight > (thresholds.targetHeightMeters + thresholds.heightToleranceMeters)
private fun estimateCameraHeight(
pc: Precomputed,
tiltDegrees: Float,
focalLengthPx: Float
): Float {
val tiltRad = Math.toRadians(tiltDegrees.toDouble())
val realHeight = thresholds.subjectRealHeightMeters
val pixelHeight = pc.heightPx
if (pixelHeight <= 0f || focalLengthPx <= 0f) return -1f
val distance = (realHeight * focalLengthPx) / pixelHeight
val height = distance * sin(tiltRad)
return height.toFloat() // in meters
}
private data class Precomputed(
val box: RectF,
val frameWidth: Int,
val frameHeight: Int,
val t: AnalyzerThresholds
) {
val top = box.top
val bottom = box.bottom
val heightPx = max(0f, bottom - top)
val verticalCoverage: Float = heightPx / frameHeight
private val topMarginPx = frameHeight * t.topMarginRatio
private val bottomMarginPx = frameHeight * t.bottomMarginRatio
private val centerTolerancePx = frameHeight * t.centerToleranceRatio
val topWithinMargin = top >= topMarginPx
val bottomWithinMargin = bottom <= (frameHeight - bottomMarginPx)
val centerY = box.centerY()
val frameCenterY = frameHeight / 2f
val centeredVertically = abs(centerY - frameCenterY) <= centerTolerancePx
}
}

12
app/src/main/java/com/example/livingai/domain/ml/ObjectDetector.kt Normal file
View File

@ -0,0 +1,12 @@
package com.example.livingai.domain.ml
import android.graphics.Bitmap
import android.graphics.RectF
interface ObjectDetector {
var onResults: (List<DetectionResult>, Long) -> Unit
var onError: (String) -> Unit
fun detect(bitmap: Bitmap, imageRotation: Int)
data class DetectionResult(val boundingBox: RectF, val text: String, val score: Float)
}

119
app/src/main/java/com/example/livingai/domain/ml/ObjectDetectorImpl.kt Normal file
View File

@ -0,0 +1,119 @@
package com.example.livingai.domain.ml
import android.content.Context
import android.graphics.Bitmap
import android.graphics.RectF
import org.tensorflow.lite.DataType
import org.tensorflow.lite.Interpreter
import org.tensorflow.lite.support.image.ImageProcessor
import org.tensorflow.lite.support.image.TensorImage
import org.tensorflow.lite.support.image.ops.ResizeOp
import org.tensorflow.lite.support.image.ops.Rot90Op
import java.io.FileInputStream
import java.nio.channels.FileChannel
class ObjectDetectorImpl(
private val context: Context,
override var onResults: (List<ObjectDetector.DetectionResult>, Long) -> Unit,
override var onError: (String) -> Unit
) : ObjectDetector {
private var interpreter: Interpreter? = null
private val modelName = "efficientdet-lite0.tflite"
private val inputSize = 320 // EfficientDet-Lite0 expects 320x320
init {
setupInterpreter()
}
private fun setupInterpreter() {
try {
val assetFileDescriptor = context.assets.openFd(modelName)
val fileInputStream = FileInputStream(assetFileDescriptor.fileDescriptor)
val fileChannel = fileInputStream.channel
val startOffset = assetFileDescriptor.startOffset
val declaredLength = assetFileDescriptor.declaredLength
val modelBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength)
val options = Interpreter.Options()
options.setNumThreads(4)
interpreter = Interpreter(modelBuffer, options)
} catch (e: Exception) {
onError(e.message ?: "Error loading model")
}
}
override fun detect(bitmap: Bitmap, imageRotation: Int) {
val tflite = interpreter ?: return
val startTime = System.currentTimeMillis()
// 1. Preprocess Image
// Rotate -> Resize -> Convert to TensorImage (UINT8)
val imageProcessor = ImageProcessor.Builder()
.add(Rot90Op(-imageRotation / 90))
.add(ResizeOp(inputSize, inputSize, ResizeOp.ResizeMethod.BILINEAR))
.build()
var tensorImage = TensorImage(DataType.UINT8)
tensorImage.load(bitmap)
tensorImage = imageProcessor.process(tensorImage)
// 2. Prepare Output Buffers
val outputBoxes = Array(1) { Array(25) { FloatArray(4) } }
val outputClasses = Array(1) { FloatArray(25) }
val outputScores = Array(1) { FloatArray(25) }
val outputCount = FloatArray(1)
val outputs = mapOf(
0 to outputBoxes,
1 to outputClasses,
2 to outputScores,
3 to outputCount
)
// 3. Run Inference
try {
tflite.runForMultipleInputsOutputs(arrayOf(tensorImage.buffer), outputs)
} catch (e: Exception) {
onError(e.message ?: "Inference failed")
return
}
val inferenceTime = System.currentTimeMillis() - startTime
// 4. Parse Results
val results = mutableListOf<ObjectDetector.DetectionResult>()
// Calculate dimensions of the rotated image (the coordinate space of the detections)
val rotatedWidth = if (imageRotation % 180 == 0) bitmap.width else bitmap.height
val rotatedHeight = if (imageRotation % 180 == 0) bitmap.height else bitmap.width
for (i in 0 until 25) {
val score = outputScores[0][i]
if (score < 0.4f) continue
val classId = outputClasses[0][i].toInt()
val label = if (classId == 20) "Cow" else "Object $classId"
// Get box: [ymin, xmin, ymax, xmax] (normalized 0..1)
val box = outputBoxes[0][i]
val ymin = box[0]
val xmin = box[1]
val ymax = box[2]
val xmax = box[3]
// Scale to rotated image dimensions
val left = xmin * rotatedWidth
val top = ymin * rotatedHeight
val right = xmax * rotatedWidth
val bottom = ymax * rotatedHeight
val boundingBox = RectF(left, top, right, bottom)
results.add(ObjectDetector.DetectionResult(boundingBox, label, score))
}
onResults(results, inferenceTime)
}
}

103
app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt
View File

@ -4,7 +4,6 @@ import android.Manifest
import android.annotation.SuppressLint
import android.content.ContentValues
import android.content.pm.ActivityInfo
import android.net.Uri
import android.provider.MediaStore
import androidx.camera.core.CameraSelector
import androidx.camera.video.MediaStoreOutputOptions
@ -13,6 +12,7 @@ import androidx.camera.video.VideoRecordEvent
import androidx.camera.view.CameraController
import androidx.camera.view.LifecycleCameraController
import androidx.camera.view.video.AudioConfig
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
@ -28,7 +28,9 @@ import androidx.compose.material3.FabPosition
import androidx.compose.material3.FloatingActionButton
import androidx.compose.material3.FloatingActionButtonDefaults
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Scaffold
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
@ -38,18 +40,24 @@ import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import androidx.navigation.NavController
import com.example.livingai.domain.ml.FeedbackState
import com.example.livingai.pages.components.CameraPreview
import com.example.livingai.pages.components.PermissionWrapper
import com.example.livingai.pages.navigation.Route
import com.example.livingai.ui.theme.RecordingRed
import com.example.livingai.utils.Constants
import com.example.livingai.utils.SetScreenOrientation
import org.koin.androidx.compose.koinViewModel
import java.io.File
import kotlin.math.max
@SuppressLint("MissingPermission")
@Composable
@ -63,6 +71,10 @@ fun VideoRecordScreen(
val context = LocalContext.current
val state by viewModel.state.collectAsState()
var recording by remember { mutableStateOf<Recording?>(null) }
var frameWidth by remember { mutableStateOf(0) }
var frameHeight by remember { mutableStateOf(0) }
var rotation by remember { mutableStateOf(0) }
// We need RECORD_AUDIO permission for video with audio
PermissionWrapper(
@ -73,7 +85,7 @@ fun VideoRecordScreen(
) {
val controller = remember {
LifecycleCameraController(context).apply {
setEnabledUseCases(CameraController.VIDEO_CAPTURE)
setEnabledUseCases(CameraController.VIDEO_CAPTURE or CameraController.IMAGE_ANALYSIS)
cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
}
}
@ -147,7 +159,12 @@ fun VideoRecordScreen(
CameraPreview(
modifier = Modifier.fillMaxSize(),
controller = controller,
onFrame = { _, _ -> }
onFrame = { bitmap, imageRotation ->
frameWidth = bitmap.width
frameHeight = bitmap.height
rotation = imageRotation
viewModel.onEvent(VideoEvent.FrameReceived(bitmap, imageRotation))
}
)
// Overlay
@ -162,7 +179,7 @@ fun VideoRecordScreen(
)
Spacer(
modifier = Modifier
.height(300.dp) // Constant height for the transparent area
.height(Constants.VIDEO_SILHOETTE_FRAME_HEIGHT) // Constant height for the transparent area
.fillMaxWidth()
)
Box(
@ -172,6 +189,82 @@ fun VideoRecordScreen(
.background(Color.Black.copy(alpha = 0.5f))
)
}
// Feedback Text Overlay
if (state.feedback !is FeedbackState.Idle && state.feedback !is FeedbackState.Optimal) {
Box(
modifier = Modifier
.fillMaxSize()
.padding(top = 80.dp),
contentAlignment = Alignment.TopCenter
) {
Text(
text = state.feedback.message,
color = Color.White,
style = MaterialTheme.typography.headlineMedium,
textAlign = TextAlign.Center,
modifier = Modifier
.background(Color.Black.copy(alpha = 0.6f))
.padding(horizontal = 16.dp, vertical = 8.dp)
)
}
}
// Bounding Box Overlay
Canvas(modifier = Modifier.fillMaxSize()) {
val canvasWidth = size.width
val canvasHeight = size.height
if (frameWidth > 0 && frameHeight > 0) {
// Determine source dimensions based on rotation
val isRotated = rotation == 90 || rotation == 270
val sourceWidth = if (isRotated) frameHeight.toFloat() else frameWidth.toFloat()
val sourceHeight = if (isRotated) frameWidth.toFloat() else frameHeight.toFloat()
// Calculate scale preserving aspect ratio (CenterCrop equivalent)
val scaleX = canvasWidth / sourceWidth
val scaleY = canvasHeight / sourceHeight
val scale = max(scaleX, scaleY)
// Calculate the dimensions of the scaled image
val scaledWidth = sourceWidth * scale
val scaledHeight = sourceHeight * scale
// Calculate the offset to center the image
val dx = (canvasWidth - scaledWidth) / 2
val dy = (canvasHeight - scaledHeight) / 2
// Draw framing box (debug/guide)
val targetHeight = Constants.VIDEO_SILHOETTE_FRAME_HEIGHT.toPx()
val targetTop = (canvasHeight - targetHeight) / 2
drawRect(
color = Color.Blue,
topLeft = Offset(0f, targetTop),
size = Size(canvasWidth, targetHeight),
style = Stroke(width = 2.dp.toPx())
)
state.cowDetection?.let { result ->
val box = result.boundingBox
val boxColor = if (state.feedback is FeedbackState.Optimal) Color.Green else Color.Yellow
drawRect(
color = boxColor,
topLeft = Offset(
x = box.left * scale + dx,
y = box.top * scale + dy
),
size = Size(
width = box.width() * scale,
height = box.height() * scale
),
style = Stroke(width = 3.dp.toPx())
)
}
}
}
}
}
}

78
app/src/main/java/com/example/livingai/pages/camera/VideoViewModel.kt
View File

@ -1,15 +1,68 @@
package com.example.livingai.pages.camera
import android.graphics.Bitmap
import android.net.Uri
import android.util.Log
import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
import com.example.livingai.domain.ml.FeedbackAnalyzer
import com.example.livingai.domain.ml.FeedbackState
import com.example.livingai.domain.ml.ObjectDetector
import com.example.livingai.utils.TiltSensorManager
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
class VideoViewModel : ViewModel() {
class VideoViewModel(
private val objectDetector: ObjectDetector,
private val feedbackAnalyzer: FeedbackAnalyzer,
private val tiltSensorManager: TiltSensorManager
) : ViewModel() {
private val _state = MutableStateFlow(VideoState())
val state = _state.asStateFlow()
init {
tiltSensorManager.start()
viewModelScope.launch {
tiltSensorManager.tilt.collect { (pitch, _, _) ->
_state.value = _state.value.copy(tilt = pitch)
}
}
objectDetector.onResults = { results, _ ->
if (results.isNotEmpty()) {
Log.d(
"VideoViewModel",
"Detected objects: ${results.joinToString { "${it.text} (${it.score})" }}"
)
}
val cow = results.firstOrNull { it.text.contains("cow", ignoreCase = true) }
val feedback = feedbackAnalyzer.analyze(
detection = cow,
frameWidth = state.value.frameWidth,
frameHeight = state.value.frameHeight,
tiltDegrees = state.value.tilt,
focalLengthPx = state.value.focalLength
)
_state.value = _state.value.copy(
cowDetection = cow,
feedback = feedback
)
}
objectDetector.onError = { error ->
Log.e("VideoViewModel", "Detection error: $error")
}
}
override fun onCleared() {
super.onCleared()
tiltSensorManager.stop()
}
fun onEvent(event: VideoEvent) {
when (event) {
is VideoEvent.VideoRecorded -> {
@ -24,13 +77,28 @@ class VideoViewModel : ViewModel() {
is VideoEvent.ClearRecordedVideo -> {
_state.value = _state.value.copy(recordedVideoUri = null)
}
is VideoEvent.FrameReceived -> {
// Update frame dimensions and focal length, but let TiltSensorManager handle tilt
_state.value = _state.value.copy(
frameWidth = event.bitmap.width,
frameHeight = event.bitmap.height,
focalLength = event.focalLength
)
objectDetector.detect(event.bitmap, event.rotation)
}
}
}
}
data class VideoState(
val isRecording: Boolean = false,
val recordedVideoUri: Uri? = null
val recordedVideoUri: Uri? = null,
val cowDetection: ObjectDetector.DetectionResult? = null,
val feedback: FeedbackState = FeedbackState.Idle,
val frameWidth: Int = 0,
val frameHeight: Int = 0,
val tilt: Float = 0f,
val focalLength: Float = 0f
)
sealed class VideoEvent {
@ -38,4 +106,10 @@ sealed class VideoEvent {
object StartRecording : VideoEvent()
object StopRecording : VideoEvent()
object ClearRecordedVideo : VideoEvent()
data class FrameReceived(
val bitmap: Bitmap,
val rotation: Int,
val tilt: Float = 0f,
val focalLength: Float = 0f
) : VideoEvent()
}

16
app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt
View File

@ -18,7 +18,7 @@ import java.util.concurrent.Executors
fun CameraPreview(
modifier: Modifier = Modifier,
controller: LifecycleCameraController? = null,
onFrame: (Bitmap, Int) -> Unit
onFrame: ((Bitmap, Int) -> Unit)? = null
) {
val context = LocalContext.current
val lifecycleOwner = LocalLifecycleOwner.current
@ -26,12 +26,14 @@ fun CameraPreview(
val cameraController = controller ?: remember { LifecycleCameraController(context) }
LaunchedEffect(cameraController) {
cameraController.setImageAnalysisAnalyzer(cameraExecutor) { imageProxy ->
val bitmap = imageProxy.toBitmap()
val rotationDegrees = imageProxy.imageInfo.rotationDegrees
onFrame(bitmap, rotationDegrees)
imageProxy.close()
if (onFrame != null) {
LaunchedEffect(cameraController) {
cameraController.setImageAnalysisAnalyzer(cameraExecutor) { imageProxy ->
val bitmap = imageProxy.toBitmap()
val rotationDegrees = imageProxy.imageInfo.rotationDegrees
onFrame(bitmap, rotationDegrees)
imageProxy.close()
}
}
}

3
app/src/main/java/com/example/livingai/utils/Constants.kt
View File

@ -1,5 +1,7 @@
package com.example.livingai.utils
import androidx.compose.ui.unit.dp
object Constants {
const val USER_SETTINGS = "USER_SETTINGS"
const val APP_ENTRY = "APP_ENTRY"
@ -15,5 +17,6 @@ object Constants {
"angleview"
)
val VIDEO_SILHOETTE_FRAME_HEIGHT = 300.dp
const val JACCARD_THRESHOLD = 85
}

58
app/src/main/java/com/example/livingai/utils/TiltSensorManager.kt Normal file
View File

@ -0,0 +1,58 @@
package com.example.livingai.utils
import android.content.Context
import android.hardware.Sensor
import android.hardware.SensorEvent
import android.hardware.SensorEventListener
import android.hardware.SensorManager
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
class TiltSensorManager(
context: Context
) : SensorEventListener {
private val sensorManager = context.getSystemService(Context.SENSOR_SERVICE) as SensorManager
private val rotationVectorSensor = sensorManager.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR)
private val _tilt = MutableStateFlow(Triple(0f, 0f, 0f)) // pitch, roll, azimuth
val tilt: StateFlow<Triple<Float, Float, Float>> = _tilt.asStateFlow()
fun start() {
rotationVectorSensor?.let {
sensorManager.registerListener(
this,
it,
SensorManager.SENSOR_DELAY_GAME
)
}
}
fun stop() {
sensorManager.unregisterListener(this)
}
override fun onSensorChanged(event: SensorEvent?) {
event ?: return
if (event.sensor.type != Sensor.TYPE_ROTATION_VECTOR) return
val rotationMatrix = FloatArray(9)
val orientationAngles = FloatArray(3)
// Convert rotation vector to rotation matrix
SensorManager.getRotationMatrixFromVector(rotationMatrix, event.values)
// Convert rotation matrix to orientation angles
SensorManager.getOrientation(rotationMatrix, orientationAngles)
// orientationAngles: [azimuth, pitch, roll] in radians
val azimuth = Math.toDegrees(orientationAngles[0].toDouble()).toFloat()
val pitch = Math.toDegrees(orientationAngles[1].toDouble()).toFloat()
val roll = Math.toDegrees(orientationAngles[2].toDouble()).toFloat()
_tilt.value = Triple(pitch, roll, azimuth)
}
override fun onAccuracyChanged(sensor: Sensor?, accuracy: Int) {}
}

9
gradle/libs.versions.toml
View File

@ -17,6 +17,9 @@ ui = "1.9.5"
kotlinxSerialization = "1.6.3"
koin = "4.1.1"
window = "1.5.1"
tensorflowLiteSupport = "0.4.4"
tensorflowLite = "2.14.0"
tensorflowLiteTaskVision = "0.4.4"
[libraries]
androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" }
@ -44,9 +47,13 @@ kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-
koin-android = { group = "io.insert-koin", name = "koin-android", version.ref = "koin" }
koin-androidx-compose = { group = "io.insert-koin", name = "koin-androidx-compose", version.ref = "koin" }
androidx-window = { group = "androidx.window", name = "window", version.ref = "window" }
tensorflow-lite-support = { group = "org.tensorflow", name = "tensorflow-lite-support", version.ref = "tensorflowLiteSupport" }
tensorflow-lite = { group = "org.tensorflow", name = "tensorflow-lite", version.ref = "tensorflowLite" }
tensorflow-lite-gpu = { group = "org.tensorflow", name = "tensorflow-lite-gpu", version.ref = "tensorflowLite" }
tensorflow-lite-task-vision = { group = "org.tensorflow", name = "tensorflow-lite-task-vision", version.ref = "tensorflowLiteTaskVision" }
[plugins]
android-application = { id = "com.android.application", version.ref = "agp" }
kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" }
kotlin-compose = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" }
kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" }
kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" }