diff --git a/.kotlin/sessions/kotlin-compiler-18065896741356929687.salive b/.kotlin/sessions/kotlin-compiler-9759521387444280191.salive similarity index 100% rename from .kotlin/sessions/kotlin-compiler-18065896741356929687.salive rename to .kotlin/sessions/kotlin-compiler-9759521387444280191.salive diff --git a/app/src/main.zip b/app/src/main.zip new file mode 100644 index 0000000..d9eb531 Binary files /dev/null and b/app/src/main.zip differ diff --git a/app/src/main/java/com/example/livingai/data/ml/AIModelImpl.kt b/app/src/main/java/com/example/livingai/data/ml/AIModelImpl.kt index e74141f..703b115 100644 --- a/app/src/main/java/com/example/livingai/data/ml/AIModelImpl.kt +++ b/app/src/main/java/com/example/livingai/data/ml/AIModelImpl.kt @@ -2,6 +2,7 @@ package com.example.livingai.data.ml import android.graphics.Bitmap import android.graphics.Color +import android.graphics.Rect import com.example.livingai.domain.ml.AIModel import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.segmentation.subject.SubjectSegmentation @@ -9,13 +10,11 @@ import com.google.mlkit.vision.segmentation.subject.SubjectSegmenterOptions import kotlinx.coroutines.suspendCancellableCoroutine import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException -import kotlin.math.max -// Define a constant color for the segmentation mask -private const val MASK_COLOR = 0x5500FF00 // Semi-transparent green +private const val MASK_COLOR = 0x5500FF00 // semi-transparent green overlay class AIModelImpl : AIModel { - + private val segmenter by lazy { val options = SubjectSegmenterOptions.Builder() .enableForegroundBitmap() @@ -23,23 +22,21 @@ class AIModelImpl : AIModel { SubjectSegmentation.getClient(options) } - override fun deriveInference(bitmap: Bitmap): String { - return "Inference Result" - } + override fun deriveInference(bitmap: Bitmap): String = "Inference Result" - override suspend fun segmentImage(bitmap: Bitmap): Pair? { + override suspend fun segmentImage(bitmap: Bitmap): Triple? { return suspendCancellableCoroutine { cont -> val image = InputImage.fromBitmap(bitmap, 0) + segmenter.process(image) .addOnSuccessListener { result -> - val foregroundBitmap = result.foregroundBitmap - if (foregroundBitmap != null) { - val colorizedMask = createColorizedMask(foregroundBitmap) - val booleanMask = createBooleanMask(foregroundBitmap) - cont.resume(Pair(colorizedMask, booleanMask)) - } else { - cont.resume(null) - } + val fg = result.foregroundBitmap ?: return@addOnSuccessListener cont.resume(null) + + val booleanMask = createBooleanMask(fg) + val colorMask = createColorizedMask(fg) + val bbox = computeBoundingBox(booleanMask, fg.width, fg.height) + + cont.resume(Triple(colorMask, booleanMask, bbox)) } .addOnFailureListener { e -> cont.resumeWithException(e) @@ -48,10 +45,11 @@ class AIModelImpl : AIModel { } private fun createColorizedMask(maskBitmap: Bitmap): Bitmap { - val width = maskBitmap.width - val height = maskBitmap.height - val pixels = IntArray(width * height) - maskBitmap.getPixels(pixels, 0, width, 0, 0, width, height) + val w = maskBitmap.width + val h = maskBitmap.height + val pixels = IntArray(w * h) + + maskBitmap.getPixels(pixels, 0, w, 0, 0, w, h) for (i in pixels.indices) { if (Color.alpha(pixels[i]) > 0) { @@ -59,7 +57,7 @@ class AIModelImpl : AIModel { } } - return Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888) + return Bitmap.createBitmap(pixels, w, h, Bitmap.Config.ARGB_8888) } private fun createBooleanMask(bitmap: Bitmap): BooleanArray { @@ -67,11 +65,38 @@ class AIModelImpl : AIModel { val h = bitmap.height val mask = BooleanArray(w * h) val pixels = IntArray(w * h) + bitmap.getPixels(pixels, 0, w, 0, 0, w, h) + for (i in pixels.indices) { - val alpha = Color.alpha(pixels[i]) - mask[i] = alpha > 0 // Assuming foreground bitmap has transparent background + mask[i] = Color.alpha(pixels[i]) > 0 } + return mask } + + private fun computeBoundingBox(mask: BooleanArray, w: Int, h: Int): Rect { + var minX = Int.MAX_VALUE + var minY = Int.MAX_VALUE + var maxX = Int.MIN_VALUE + var maxY = Int.MIN_VALUE + + for (y in 0 until h) { + for (x in 0 until w) { + val idx = y * w + x + if (mask[idx]) { + if (x < minX) minX = x + if (y < minY) minY = y + if (x > maxX) maxX = x + if (y > maxY) maxY = y + } + } + } + + return if (minX == Int.MAX_VALUE) { + Rect(0, 0, 0, 0) + } else { + Rect(minX, minY, maxX, maxY) + } + } } diff --git a/app/src/main/java/com/example/livingai/data/ml/DistanceEstimatorImpl.kt b/app/src/main/java/com/example/livingai/data/ml/DistanceEstimatorImpl.kt new file mode 100644 index 0000000..3993a42 --- /dev/null +++ b/app/src/main/java/com/example/livingai/data/ml/DistanceEstimatorImpl.kt @@ -0,0 +1,49 @@ +package com.example.livingai.data.ml + +import com.example.livingai.domain.ml.ArcoreDepthEstimator +import com.example.livingai.domain.ml.CameraInfoData +import com.example.livingai.domain.ml.CameraInfoProvider +import com.example.livingai.domain.ml.DistanceEstimator +import com.example.livingai.domain.ml.DistanceRecommendation +import com.example.livingai.domain.ml.DistanceState +import com.example.livingai.domain.ml.FrameData +import com.example.livingai.domain.ml.KnownDimensionEstimator +import com.example.livingai.utils.Constants + +class DistanceEstimatorImpl( + private val mainEstimator: DistanceEstimator = ArcoreDepthEstimator(), + private val fallbackEstimator: DistanceEstimator = KnownDimensionEstimator() +) { + + fun processFrame(frame: FrameData): DistanceState { + // Fallback or retrieve camera info + val camInfo = CameraInfoProvider.tryGet() + ?: createFallbackCameraInfo(frame) + + val main = mainEstimator.analyze(frame, camInfo) + return main.distanceMeters?.let { main } + ?: fallbackEstimator.analyze(frame, camInfo) + } + + private fun createFallbackCameraInfo(frame: FrameData): CameraInfoData { + // Estimate focal length based on FOV if available, or a reasonable default + // For a typical phone: + // H-FOV ~ 60-70 degrees + // fx = (W/2) / tan(FOV/2) + val w = frame.imageBitmap?.width ?: 1080 + val h = frame.imageBitmap?.height ?: 1920 + + // Assume approx 60 degrees horizontal FOV as a fallback + val fovDegrees = 60.0 + val fovRadians = Math.toRadians(fovDegrees) + val focalLengthPx = (w / 2.0) / Math.tan(fovRadians / 2.0) + + return CameraInfoData( + focalLengthPixels = focalLengthPx.toFloat(), + sensorWidthPx = w, + sensorHeightPx = h, + principalPointX = w / 2f, + principalPointY = h / 2f + ) + } +} diff --git a/app/src/main/java/com/example/livingai/data/repository/media/CameraRepositoryImpl.kt b/app/src/main/java/com/example/livingai/data/repository/media/CameraRepositoryImpl.kt index 0763b38..70b0845 100644 --- a/app/src/main/java/com/example/livingai/data/repository/media/CameraRepositoryImpl.kt +++ b/app/src/main/java/com/example/livingai/data/repository/media/CameraRepositoryImpl.kt @@ -4,45 +4,67 @@ import android.content.ContentValues import android.content.Context import android.graphics.Bitmap import android.graphics.Matrix -import android.net.Uri import android.provider.MediaStore import androidx.camera.core.ImageProxy +import com.example.livingai.data.ml.DistanceEstimatorImpl import com.example.livingai.domain.ml.AIModel +import com.example.livingai.domain.ml.DistanceState +import com.example.livingai.domain.ml.FrameMetadataProvider +import com.example.livingai.domain.ml.FrameMetadataProvider.toFrameData import com.example.livingai.domain.repository.CameraRepository +import com.example.livingai.utils.TiltSensorManager import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext -import java.io.File -import java.io.FileOutputStream class CameraRepositoryImpl( private val aiModel: AIModel, + private val tiltSensorManager: TiltSensorManager, private val context: Context ) : CameraRepository { - override suspend fun captureImage(imageProxy: ImageProxy): Bitmap = withContext(Dispatchers.IO) { - val rotationDegrees = imageProxy.imageInfo.rotationDegrees - val bitmap = imageProxy.toBitmap() - imageProxy.close() + private val distanceEstimator = DistanceEstimatorImpl() - // Rotate bitmap if needed - if (rotationDegrees != 0) { - val matrix = Matrix().apply { postRotate(rotationDegrees.toFloat()) } - Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) - } else { - bitmap + init { + // inject dependencies into metadata provider + FrameMetadataProvider.aiModel = aiModel + FrameMetadataProvider.tiltSensorManager = tiltSensorManager + } + + override suspend fun captureImage(imageProxy: ImageProxy): Bitmap = + withContext(Dispatchers.IO) { + val rotation = imageProxy.imageInfo.rotationDegrees + val bitmap = imageProxy.toBitmap() + imageProxy.close() + + if (rotation != 0) { + val m = Matrix().apply { postRotate(rotation.toFloat()) } + Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, m, true) + } else bitmap } - } - override suspend fun processFrame(bitmap: Bitmap): String = withContext(Dispatchers.Default) { - aiModel.deriveInference(bitmap) - } + override suspend fun processFrame(bitmap: Bitmap): DistanceState = + withContext(Dispatchers.Default) { - override suspend fun saveImage(bitmap: Bitmap, animalId: String, orientation: String?): String = withContext(Dispatchers.IO) { - val orientationSuffix = orientation?.let { "_$it" } ?: "" - val filename = "${animalId}${orientationSuffix}.jpg" - - val contentValues = ContentValues().apply { - put(MediaStore.Images.Media.DISPLAY_NAME, filename) + // 1. Collect metadata + val meta = FrameMetadataProvider.collectMetadata(bitmap) + + // 2. Convert to FrameData + val frameData = meta.toFrameData(bitmap) + + // 3. Run distance estimator + distanceEstimator.processFrame(frameData) + } + + override suspend fun saveImage( + bitmap: Bitmap, + animalId: String, + orientation: String? + ): String = withContext(Dispatchers.IO) { + val suffix = orientation?.let { "_$it" } ?: "" + val fileName = "$animalId$suffix.jpg" + + val values = ContentValues().apply { + put(MediaStore.Images.Media.DISPLAY_NAME, fileName) put(MediaStore.Images.Media.MIME_TYPE, "image/jpeg") if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.Q) { put(MediaStore.Images.Media.RELATIVE_PATH, "Pictures/LivingAI/Media/$animalId") @@ -51,24 +73,24 @@ class CameraRepositoryImpl( } val resolver = context.contentResolver - val uri = resolver.insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, contentValues) - ?: throw RuntimeException("Failed to create image record") + val uri = resolver.insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, values) + ?: throw RuntimeException("Image insert failed") try { resolver.openOutputStream(uri)?.use { out -> bitmap.compress(Bitmap.CompressFormat.JPEG, 100, out) } - + if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.Q) { - contentValues.clear() - contentValues.put(MediaStore.Images.Media.IS_PENDING, 0) - resolver.update(uri, contentValues, null, null) + values.clear() + values.put(MediaStore.Images.Media.IS_PENDING, 0) + resolver.update(uri, values, null, null) } } catch (e: Exception) { - resolver.delete(uri, null, null) - throw e + resolver.delete(uri, null, null) + throw e } - + uri.toString() } } diff --git a/app/src/main/java/com/example/livingai/di/AppModule.kt b/app/src/main/java/com/example/livingai/di/AppModule.kt index 2687b81..36d14ef 100644 --- a/app/src/main/java/com/example/livingai/di/AppModule.kt +++ b/app/src/main/java/com/example/livingai/di/AppModule.kt @@ -128,7 +128,7 @@ val appModule = module { single { AnimalProfileRepositoryImpl(get()) } single { AnimalDetailsRepositoryImpl(get()) } single { AnimalRatingRepositoryImpl(get()) } - single { CameraRepositoryImpl(get(), androidContext()) } + single { CameraRepositoryImpl(get(), get(), androidContext()) } single { VideoRepositoryImpl(get()) } // Use Cases diff --git a/app/src/main/java/com/example/livingai/domain/ml/AIModel.kt b/app/src/main/java/com/example/livingai/domain/ml/AIModel.kt index ee66295..47a7d0f 100644 --- a/app/src/main/java/com/example/livingai/domain/ml/AIModel.kt +++ b/app/src/main/java/com/example/livingai/domain/ml/AIModel.kt @@ -1,8 +1,9 @@ package com.example.livingai.domain.ml import android.graphics.Bitmap +import android.graphics.Rect interface AIModel { fun deriveInference(bitmap: Bitmap): String - suspend fun segmentImage(bitmap: Bitmap): Pair? + suspend fun segmentImage(bitmap: Bitmap): Triple? } \ No newline at end of file diff --git a/app/src/main/java/com/example/livingai/domain/ml/ARCoreDepthEstimator.kt b/app/src/main/java/com/example/livingai/domain/ml/ARCoreDepthEstimator.kt new file mode 100644 index 0000000..cb444c1 --- /dev/null +++ b/app/src/main/java/com/example/livingai/domain/ml/ARCoreDepthEstimator.kt @@ -0,0 +1,152 @@ +package com.example.livingai.domain.ml + +import android.graphics.Rect +import com.example.livingai.utils.Constants +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.min + +class ArcoreDepthEstimator( + private val params: ArcoreDepthParams = ArcoreDepthParams() +) : DistanceEstimator { + + data class ArcoreDepthParams( + val targetDistanceMeters: Float = Constants.TARGET_DISTANCE_METERS, + val strictToleranceMeters: Float = Constants.DISTANCE_TOLERANCE_METERS_STRICT, + val relaxedToleranceMeters: Float = Constants.DISTANCE_TOLERANCE_METERS_RELAXED, + val depthConfidenceMin: Float = Constants.DEPTH_CONFIDENCE_MIN, + val minDepthSamples: Int = Constants.MIN_DEPTH_SAMPLES + ) + + override fun analyze(frame: FrameData, cameraInfo: CameraInfoData): DistanceState { + + val isTilted = abs(frame.imuPitchDegrees) > Constants.MAX_ACCEPTABLE_PITCH_DEGREES || + abs(frame.imuRollDegrees) > Constants.MAX_ACCEPTABLE_ROLL_DEGREES + + val isRotated = (frame.cameraRotationDegrees % 360) != 0 + val isOrientationCorrect = !isTilted && !isRotated + + val centered = checkCentered(frame, cameraInfo) + + val depthEstimate = sampleDepthMedian(frame) + val fallbackEstimate = computeKnownDimensionEstimate(frame, cameraInfo) + + val fusedDistance = when { + depthEstimate != null -> + Constants.WEIGHT_ARCORE * depthEstimate + + Constants.WEIGHT_KNOWN_DIM * (fallbackEstimate ?: depthEstimate) + + fallbackEstimate != null -> fallbackEstimate + else -> null + } + + val (recommendation, ready, conf) = + evaluateDistanceAndReadiness(fusedDistance, centered, isOrientationCorrect) + + return DistanceState( + distanceMeters = fusedDistance, + recommendation = recommendation, + isCameraTilted = isTilted, + isCameraRotated = isRotated, + isOrientationCorrect = isOrientationCorrect, + isObjectCentered = centered, + readyToCapture = ready, + confidenceScore = conf + ) + } + + private fun sampleDepthMedian(frame: FrameData): Float? { + val depth = frame.depthMapMeters ?: return null + val w = frame.depthWidth + val h = frame.depthHeight + if (w <= 0 || h <= 0) return null + + val box = frame.segmentationBox ?: return null + + val left = max(0, box.left) + val top = max(0, box.top) + val right = min(w - 1, box.right) + val bottom = min(h - 1, box.bottom) + + val conf = frame.depthConfidence + val samples = ArrayList() + + for (y in top..bottom) { + for (x in left..right) { + val idx = y * w + x + val d = depth[idx] + if (d <= 0f || d.isNaN()) continue + + if (conf != null && conf[idx] < params.depthConfidenceMin) + continue + + samples.add(d) + } + } + + if (samples.size < params.minDepthSamples) return null + samples.sort() + return samples[samples.size / 2] + } + + private fun computeKnownDimensionEstimate( + frame: FrameData, + cameraInfo: CameraInfoData + ): Float? { + val box = frame.segmentationBox ?: return null + val hPixels = box.height().toFloat() + if (hPixels <= 1f) return null + + val f = cameraInfo.focalLengthPixels + val Hreal = Constants.DEFAULT_OBJECT_REAL_HEIGHT_METERS + + return (f * Hreal) / hPixels + } + + private fun evaluateDistanceAndReadiness( + distMeters: Float?, + centered: Boolean, + orientationOk: Boolean + ): Triple { + + if (distMeters == null) + return Triple(DistanceRecommendation.DISTANCE_UNKNOWN, false, 0f) + + val diff = distMeters - params.targetDistanceMeters + val absDiff = abs(diff) + + val withinStrict = absDiff <= params.strictToleranceMeters + val withinRelaxed = absDiff <= params.relaxedToleranceMeters + + val recommendation = when { + withinStrict -> DistanceRecommendation.AT_OPTIMAL_DISTANCE + diff > 0f -> DistanceRecommendation.MOVE_CLOSER + else -> DistanceRecommendation.MOVE_AWAY + } + + val ready = withinRelaxed && centered && orientationOk + + val closenessScore = 1f - (absDiff / (params.relaxedToleranceMeters * 4f)) + val confidence = closenessScore * 0.8f + + (if (centered && orientationOk) 0.2f else 0f) + + return Triple(recommendation, ready, confidence) + } + + private fun checkCentered(frame: FrameData, cameraInfo: CameraInfoData): Boolean { + val box = frame.segmentationBox ?: return false + val imgW = cameraInfo.sensorWidthPx + val imgH = cameraInfo.sensorHeightPx + + val cxObj = (box.left + box.right) / 2f + val cyObj = (box.top + box.bottom) / 2f + val cx = imgW / 2f + val cy = imgH / 2f + + val dx = abs(cxObj - cx) / imgW + val dy = abs(cyObj - cy) / imgH + + return dx <= Constants.CENTER_TOLERANCE_X_FRACTION && + dy <= Constants.CENTER_TOLERANCE_Y_FRACTION + } +} diff --git a/app/src/main/java/com/example/livingai/domain/ml/CameraInfoProvider.kt b/app/src/main/java/com/example/livingai/domain/ml/CameraInfoProvider.kt new file mode 100644 index 0000000..29a436c --- /dev/null +++ b/app/src/main/java/com/example/livingai/domain/ml/CameraInfoProvider.kt @@ -0,0 +1,21 @@ +package com.example.livingai.domain.ml + +/** + * Singleton provider of camera intrinsic data. + * Must be initialized once per session. + */ +object CameraInfoProvider { + @Volatile + private var cameraInfoData: CameraInfoData? = null + + fun init(info: CameraInfoData) { + cameraInfoData = info + } + + fun get(): CameraInfoData { + return cameraInfoData + ?: throw IllegalStateException("CameraInfoProvider not initialized") + } + + fun tryGet(): CameraInfoData? = cameraInfoData +} diff --git a/app/src/main/java/com/example/livingai/domain/ml/DistanceEstimator.kt b/app/src/main/java/com/example/livingai/domain/ml/DistanceEstimator.kt new file mode 100644 index 0000000..af35413 --- /dev/null +++ b/app/src/main/java/com/example/livingai/domain/ml/DistanceEstimator.kt @@ -0,0 +1,74 @@ +package com.example.livingai.domain.ml + +import android.graphics.Bitmap +import android.graphics.Rect +import kotlin.math.abs + +/** + * Interface for all distance estimators. + */ +interface DistanceEstimator { + fun analyze( + frame: FrameData, + cameraInfo: CameraInfoData + ): DistanceState +} + +/** + * Frame-specific data for one inference cycle. + */ +data class FrameData( + val imageBitmap: Bitmap?, + val segmentationBox: Rect?, + val segmentationMaskBitmap: Bitmap?, + + // Optional ARCore depth inputs + val depthMapMeters: FloatArray?, // row-major R* C + val depthWidth: Int = 0, + val depthHeight: Int = 0, + val depthConfidence: FloatArray? = null, + + // IMU orientation + val imuPitchDegrees: Float = 0f, + val imuRollDegrees: Float = 0f, + val imuYawDegrees: Float = 0f, + + val cameraRotationDegrees: Int = 0, + val timestampMs: Long = System.currentTimeMillis() +) + +/** + * Singleton-provided camera intrinsics for metric calculations. + */ +data class CameraInfoData( + val focalLengthPixels: Float, // fx in pixels + val sensorWidthPx: Int, + val sensorHeightPx: Int, + val principalPointX: Float, + val principalPointY: Float, + val distortionCoeffs: FloatArray? = null, + val cameraModel: String? = null +) + +/** + * Output state describing computed distance and capture readiness. + */ +data class DistanceState( + val distanceMeters: Float?, + val recommendation: DistanceRecommendation, + + val isCameraTilted: Boolean, + val isCameraRotated: Boolean, + val isOrientationCorrect: Boolean, + val isObjectCentered: Boolean, + + val readyToCapture: Boolean, + val confidenceScore: Float = 0f +) + +enum class DistanceRecommendation { + MOVE_CLOSER, + MOVE_AWAY, + AT_OPTIMAL_DISTANCE, + DISTANCE_UNKNOWN +} diff --git a/app/src/main/java/com/example/livingai/domain/ml/FeedbackAnalyzer.kt b/app/src/main/java/com/example/livingai/domain/ml/FeedbackAnalyzer.kt index ddc6624..7add1f0 100644 --- a/app/src/main/java/com/example/livingai/domain/ml/FeedbackAnalyzer.kt +++ b/app/src/main/java/com/example/livingai/domain/ml/FeedbackAnalyzer.kt @@ -29,9 +29,10 @@ sealed class FeedbackState(val message: String) { object Searching : FeedbackState("Searching for subject...") object TooFar : FeedbackState("Move closer") object TooClose : FeedbackState("Move back") - object NotCentered : FeedbackState("Center the subject") object TooLow : FeedbackState("Raise phone") object TooHigh : FeedbackState("Lower phone") + object PhoneTooLow : FeedbackState("Raise phone to 60 to 70 cm from ground") + object PhoneTooHigh : FeedbackState("Lower phone to 60 to 70 cm from ground") object Optimal : FeedbackState("Hold still") } @@ -76,14 +77,8 @@ class FeedbackAnalyzerImpl( thresholds ) - val cameraHeight = estimateCameraHeight(pc, tiltDegrees, focalLengthPx) - - Log.d("FeedbackAnalyzer", "Camera Height: $cameraHeight, frame height: ${pc.frameHeight}, " + - "frame top: ${pc.frameTop}, frame bottom: ${pc.frameBottom}," + - " detection top: ${pc.detectionTop}, detection bottom: ${pc.detectionBottom}, " + - "isTopInside: ${pc.isTopInside}, isBottomInside: ${pc.isBottomInside}," + - " detection height: ${pc.detectionHeight}, tolerance: ${pc.tolerance}") - + val cameraHeight = estimateCameraHeight(pc.detectionHeight, tiltDegrees, focalLengthPx) + Log.d("FeedbackAnalyzerImpl", "Camera Height: $cameraHeight") return when { // ORDER MATTERS — evaluate alignment first isTooHigh(pc) -> FeedbackState.TooHigh @@ -92,8 +87,8 @@ class FeedbackAnalyzerImpl( isTooFar(pc) -> FeedbackState.TooFar // Height estimation last - isHeightTooLow(cameraHeight) -> FeedbackState.TooLow - isHeightTooHigh(cameraHeight) -> FeedbackState.TooHigh + isHeightTooLow(cameraHeight) -> FeedbackState.PhoneTooLow + isHeightTooHigh(cameraHeight) -> FeedbackState.PhoneTooHigh else -> FeedbackState.Optimal } @@ -122,18 +117,17 @@ class FeedbackAnalyzerImpl( heightMeters > (thresholds.maxTargetHeightMeters) private fun estimateCameraHeight( - pc: Precomputed, + pixelHeight: Float, tiltDegrees: Float, focalLengthPx: Float ): Float { val tiltRad = Math.toRadians(tiltDegrees.toDouble()) val realHeight = thresholds.subjectRealHeightMeters - val pixelHeight = pc.detectionHeight - if (pixelHeight <= 0f || focalLengthPx <= 0f) return -1f val distance = (realHeight * focalLengthPx) / pixelHeight + Log.d("FeedbackAnalyzerImpl", "Distance: $distance") return (distance * sin(tiltRad)).toFloat() } diff --git a/app/src/main/java/com/example/livingai/domain/ml/FrameMetadataProvider.kt b/app/src/main/java/com/example/livingai/domain/ml/FrameMetadataProvider.kt new file mode 100644 index 0000000..315c31e --- /dev/null +++ b/app/src/main/java/com/example/livingai/domain/ml/FrameMetadataProvider.kt @@ -0,0 +1,182 @@ +package com.example.livingai.domain.ml + +import android.graphics.Bitmap +import android.graphics.Rect +import com.example.livingai.utils.TiltSensorManager + +object FrameMetadataProvider { + + lateinit var aiModel: AIModel // injected once from AppModule + var tiltSensorManager: TiltSensorManager? = null + + // External data sources + var latestDepthResult: DepthResult? = null + var deviceRotation: Int = 0 + + suspend fun getSegmentation(bitmap: Bitmap): SegmentationResult? { + return try { + val (_, booleanMask, bbox) = aiModel.segmentImage(bitmap) ?: return null + SegmentationResult(booleanMask, bbox) + } catch (_: Exception) { + null + } + } + + data class SegmentationResult( + val mask: BooleanArray, + val boundingBox: Rect + ) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as SegmentationResult + + if (!mask.contentEquals(other.mask)) return false + if (boundingBox != other.boundingBox) return false + + return true + } + + override fun hashCode(): Int { + var result = mask.contentHashCode() + result = 31 * result + boundingBox.hashCode() + return result + } + } + + fun getDepthData(): DepthResult { + return latestDepthResult ?: DepthResult(null, 0, 0, null) + } + + data class DepthResult( + val depthMeters: FloatArray?, + val width: Int, + val height: Int, + val confidence: FloatArray? + ) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as DepthResult + + if (depthMeters != null) { + if (other.depthMeters == null) return false + if (!depthMeters.contentEquals(other.depthMeters)) return false + } else if (other.depthMeters != null) return false + if (width != other.width) return false + if (height != other.height) return false + if (confidence != null) { + if (other.confidence == null) return false + if (!confidence.contentEquals(other.confidence)) return false + } else if (other.confidence != null) return false + + return true + } + + override fun hashCode(): Int { + var result = depthMeters?.contentHashCode() ?: 0 + result = 31 * result + width + result = 31 * result + height + result = 31 * result + (confidence?.contentHashCode() ?: 0) + return result + } + } + + fun getIMU(): IMUResult { + val (pitch, roll, yaw) = tiltSensorManager?.tilt?.value ?: Triple(0f, 0f, 0f) + return IMUResult(pitch, roll, yaw) + } + + data class IMUResult(val pitch: Float, val roll: Float, val yaw: Float) + + fun getRotation(): Int { + return deviceRotation + } + + data class FrameCollectedMetadata( + val segmentationBox: Rect?, + val depthMeters: FloatArray?, + val depthWidth: Int, + val depthHeight: Int, + val depthConfidence: FloatArray?, + val pitch: Float, + val roll: Float, + val yaw: Float, + val rotationDegrees: Int + ) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as FrameCollectedMetadata + + if (segmentationBox != other.segmentationBox) return false + if (depthMeters != null) { + if (other.depthMeters == null) return false + if (!depthMeters.contentEquals(other.depthMeters)) return false + } else if (other.depthMeters != null) return false + if (depthWidth != other.depthWidth) return false + if (depthHeight != other.depthHeight) return false + if (depthConfidence != null) { + if (other.depthConfidence == null) return false + if (!depthConfidence.contentEquals(other.depthConfidence)) return false + } else if (other.depthConfidence != null) return false + if (pitch != other.pitch) return false + if (roll != other.roll) return false + if (yaw != other.yaw) return false + if (rotationDegrees != other.rotationDegrees) return false + + return true + } + + override fun hashCode(): Int { + var result = segmentationBox?.hashCode() ?: 0 + result = 31 * result + (depthMeters?.contentHashCode() ?: 0) + result = 31 * result + depthWidth + result = 31 * result + depthHeight + result = 31 * result + (depthConfidence?.contentHashCode() ?: 0) + result = 31 * result + pitch.hashCode() + result = 31 * result + roll.hashCode() + result = 31 * result + yaw.hashCode() + result = 31 * result + rotationDegrees + return result + } + } + + suspend fun collectMetadata(bitmap: Bitmap): FrameCollectedMetadata { + val seg = getSegmentation(bitmap) + val depth = getDepthData() + val imu = getIMU() + val rot = getRotation() + + return FrameCollectedMetadata( + segmentationBox = seg?.boundingBox, + depthMeters = depth.depthMeters, + depthWidth = depth.width, + depthHeight = depth.height, + depthConfidence = depth.confidence, + pitch = imu.pitch, + roll = imu.roll, + yaw = imu.yaw, + rotationDegrees = rot + ) + } + + fun FrameCollectedMetadata.toFrameData(bitmap: Bitmap): FrameData { + return FrameData( + imageBitmap = bitmap, + segmentationBox = segmentationBox, + segmentationMaskBitmap = null, + depthMapMeters = depthMeters, + depthWidth = depthWidth, + depthHeight = depthHeight, + depthConfidence = depthConfidence, + imuPitchDegrees = pitch, + imuRollDegrees = roll, + imuYawDegrees = yaw, + cameraRotationDegrees = rotationDegrees + ) + } +} diff --git a/app/src/main/java/com/example/livingai/domain/ml/KnownDistanceEstimator.kt b/app/src/main/java/com/example/livingai/domain/ml/KnownDistanceEstimator.kt new file mode 100644 index 0000000..5904332 --- /dev/null +++ b/app/src/main/java/com/example/livingai/domain/ml/KnownDistanceEstimator.kt @@ -0,0 +1,103 @@ +package com.example.livingai.domain.ml + +import com.example.livingai.utils.Constants +import kotlin.math.abs +import kotlin.math.min + +class KnownDimensionEstimator( + private val params: KnownDimensionParams = KnownDimensionParams() +) : DistanceEstimator { + + data class KnownDimensionParams( + val knownObjectHeightMeters: Float = Constants.DEFAULT_OBJECT_REAL_HEIGHT_METERS, + val targetDistanceMeters: Float = Constants.TARGET_DISTANCE_METERS, + val strictToleranceMeters: Float = Constants.DISTANCE_TOLERANCE_METERS_STRICT, + val relaxedToleranceMeters: Float = Constants.DISTANCE_TOLERANCE_METERS_RELAXED + ) + + override fun analyze(frame: FrameData, cameraInfo: CameraInfoData): DistanceState { + + val tilted = abs(frame.imuPitchDegrees) > Constants.MAX_ACCEPTABLE_PITCH_DEGREES || + abs(frame.imuRollDegrees) > Constants.MAX_ACCEPTABLE_ROLL_DEGREES + + val rotated = (frame.cameraRotationDegrees % 360) != 0 + val orientationOk = !tilted && !rotated + val centered = checkCentered(frame, cameraInfo) + + val distEstimate = computeDistance(frame, cameraInfo) + + val (recommendation, ready, conf) = + evaluateDistanceAndReadiness(distEstimate, centered, orientationOk) + + return DistanceState( + distanceMeters = distEstimate, + recommendation = recommendation, + isCameraTilted = tilted, + isCameraRotated = rotated, + isOrientationCorrect = orientationOk, + isObjectCentered = centered, + readyToCapture = ready, + confidenceScore = conf + ) + } + + private fun computeDistance( + frame: FrameData, + cameraInfo: CameraInfoData + ): Float? { + val box = frame.segmentationBox ?: return null + val hPx = box.height().toFloat() + if (hPx <= 1f) return null + + val f = cameraInfo.focalLengthPixels + return (f * params.knownObjectHeightMeters) / hPx + } + + private fun evaluateDistanceAndReadiness( + distMeters: Float?, + centered: Boolean, + orientationOk: Boolean + ): Triple { + + if (distMeters == null) + return Triple(DistanceRecommendation.DISTANCE_UNKNOWN, false, 0f) + + val diff = distMeters - params.targetDistanceMeters + val absDiff = abs(diff) + + val withinStrict = absDiff <= params.strictToleranceMeters + val withinRelaxed = absDiff <= params.relaxedToleranceMeters + + val recommendation = when { + withinStrict -> DistanceRecommendation.AT_OPTIMAL_DISTANCE + diff > 0f -> DistanceRecommendation.MOVE_CLOSER + else -> DistanceRecommendation.MOVE_AWAY + } + + val ready = withinRelaxed && centered && orientationOk + + val closenessScore = 1f - min(1f, absDiff / (params.relaxedToleranceMeters * 4f)) + val conf = closenessScore * 0.9f + + (if (centered && orientationOk) 0.1f else 0f) + + return Triple(recommendation, ready, conf) + } + + private fun checkCentered(frame: FrameData, cameraInfo: CameraInfoData): Boolean { + val box = frame.segmentationBox ?: return false + val imgW = cameraInfo.sensorWidthPx + val imgH = cameraInfo.sensorHeightPx + + val objCx = (box.left + box.right) / 2f + val objCy = (box.top + box.bottom) / 2f + + val cx = imgW / 2f + val cy = imgH / 2f + + val dx = abs(objCx - cx) / imgW + val dy = abs(objCy - cy) / imgH + + return dx <= Constants.CENTER_TOLERANCE_X_FRACTION && + dy <= Constants.CENTER_TOLERANCE_Y_FRACTION + } +} diff --git a/app/src/main/java/com/example/livingai/domain/repository/CameraRepository.kt b/app/src/main/java/com/example/livingai/domain/repository/CameraRepository.kt index 11f1a3e..1e852ea 100644 --- a/app/src/main/java/com/example/livingai/domain/repository/CameraRepository.kt +++ b/app/src/main/java/com/example/livingai/domain/repository/CameraRepository.kt @@ -2,9 +2,10 @@ package com.example.livingai.domain.repository import android.graphics.Bitmap import androidx.camera.core.ImageProxy +import com.example.livingai.domain.ml.DistanceState interface CameraRepository { suspend fun captureImage(imageProxy: ImageProxy): Bitmap - suspend fun processFrame(bitmap: Bitmap): String + suspend fun processFrame(bitmap: Bitmap): DistanceState suspend fun saveImage(bitmap: Bitmap, animalId: String, orientation: String?): String } diff --git a/app/src/main/java/com/example/livingai/pages/camera/CameraScreen.kt b/app/src/main/java/com/example/livingai/pages/camera/CameraScreen.kt index 44f30f1..ec5491d 100644 --- a/app/src/main/java/com/example/livingai/pages/camera/CameraScreen.kt +++ b/app/src/main/java/com/example/livingai/pages/camera/CameraScreen.kt @@ -6,15 +6,21 @@ import androidx.camera.core.ImageCaptureException import androidx.camera.core.ImageProxy import androidx.camera.view.LifecycleCameraController import androidx.compose.foundation.Image +import androidx.compose.foundation.background import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.Camera import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.FabPosition import androidx.compose.material3.FloatingActionButton import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.collectAsState @@ -22,9 +28,11 @@ import androidx.compose.runtime.getValue import androidx.compose.runtime.remember import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.dp import androidx.navigation.NavController import androidx.core.content.ContextCompat import com.example.livingai.pages.components.CameraPreview @@ -121,7 +129,7 @@ fun CameraScreen( CameraPreview( modifier = Modifier.fillMaxSize(), controller = controller, - onFrame = { bitmap, rotation -> + onFrame = { bitmap, rotation, _ -> viewModel.onEvent(CameraEvent.FrameReceived(bitmap, rotation)) } ) @@ -146,15 +154,26 @@ fun CameraScreen( alpha = 0.4f ) } + } - state.savedMaskBitmap?.let { - Image( - bitmap = it.asImageBitmap(), - contentDescription = "Silhouette Overlay", - modifier = Modifier.fillMaxSize(), - contentScale = ContentScale.Fit, - alpha = 0.4f - ) + // Debug Overlay + state.distanceState?.let { dist -> + Box( + modifier = Modifier + .align(Alignment.TopEnd) + .padding(16.dp) + .background(Color.Black.copy(alpha = 0.5f), RoundedCornerShape(8.dp)) + .padding(8.dp) + ) { + Column { + Text("Dist: ${dist.distanceMeters ?: "N/A"}", color = Color.White) + Text("Rec: ${dist.recommendation}", color = Color.White) + Text("Tilted: ${dist.isCameraTilted}", color = Color.White) + Text("Rotated: ${dist.isCameraRotated}", color = Color.White) + Text("Centered: ${dist.isObjectCentered}", color = Color.White) + Text("Ready: ${dist.readyToCapture}", color = Color.White) + Text("Conf: ${dist.confidenceScore}", color = Color.White) + } } } diff --git a/app/src/main/java/com/example/livingai/pages/camera/CameraViewModel.kt b/app/src/main/java/com/example/livingai/pages/camera/CameraViewModel.kt index 7eeebfa..abafb82 100644 --- a/app/src/main/java/com/example/livingai/pages/camera/CameraViewModel.kt +++ b/app/src/main/java/com/example/livingai/pages/camera/CameraViewModel.kt @@ -7,6 +7,7 @@ import androidx.camera.core.ImageProxy import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.example.livingai.domain.ml.AIModel +import com.example.livingai.domain.ml.DistanceState import com.example.livingai.domain.repository.CameraRepository import com.example.livingai.domain.usecases.AppDataUseCases import com.example.livingai.utils.ScreenDimensions @@ -94,6 +95,9 @@ class CameraViewModel( if (isProcessingFrame.compareAndSet(false, true)) { viewModelScope.launch { try { + // Process the frame for distance and metadata + val distanceState = cameraRepository.processFrame(bitmap) + val result = aiModel.segmentImage(bitmap) if (result != null) { val (maskBitmap, _) = result @@ -119,7 +123,8 @@ class CameraViewModel( fitImageToCrop(rotatedMask, screenDims.screenHeight, screenDims.screenWidth) _state.value = _state.value.copy( - segmentationMask = output + segmentationMask = output, + distanceState = distanceState ) if (_state.value.isAutoCaptureEnabled && @@ -139,7 +144,8 @@ class CameraViewModel( } } else { _state.value = _state.value.copy( - segmentationMask = null + segmentationMask = null, + distanceState = distanceState ) } } finally { @@ -161,7 +167,8 @@ data class CameraUiState( val isAutoCaptureEnabled: Boolean = false, val matchThreshold: Int = 50, val distanceMethod: String = "Jaccard", - val shouldAutoCapture: Boolean = false + val shouldAutoCapture: Boolean = false, + val distanceState: DistanceState? = null ) sealed class CameraEvent { diff --git a/app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt b/app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt index 10b35aa..ad57eae 100644 --- a/app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt +++ b/app/src/main/java/com/example/livingai/pages/camera/VideoRecordScreen.kt @@ -159,11 +159,11 @@ fun VideoRecordScreen( CameraPreview( modifier = Modifier.fillMaxSize(), controller = controller, - onFrame = { bitmap, imageRotation -> + onFrame = { bitmap, imageRotation, focalLength -> frameWidth = bitmap.width frameHeight = bitmap.height rotation = imageRotation - viewModel.onEvent(VideoEvent.FrameReceived(bitmap, imageRotation)) + viewModel.onEvent(VideoEvent.FrameReceived(bitmap, imageRotation, focalLength = focalLength)) } ) diff --git a/app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt b/app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt index 3e8aeb1..214e3e6 100644 --- a/app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt +++ b/app/src/main/java/com/example/livingai/pages/components/CameraPreview.kt @@ -1,24 +1,32 @@ package com.example.livingai.pages.components +import android.annotation.SuppressLint import android.graphics.Bitmap +import android.hardware.camera2.CameraCharacteristics import android.view.ViewGroup +import androidx.annotation.OptIn +import androidx.camera.camera2.interop.Camera2CameraInfo +import androidx.camera.camera2.interop.ExperimentalCamera2Interop import androidx.camera.core.CameraSelector import androidx.camera.view.LifecycleCameraController import androidx.camera.view.PreviewView import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalLifecycleOwner import androidx.compose.ui.viewinterop.AndroidView +import kotlinx.coroutines.delay import java.util.concurrent.Executors +@OptIn(ExperimentalCamera2Interop::class) @Composable fun CameraPreview( modifier: Modifier = Modifier, controller: LifecycleCameraController? = null, - onFrame: ((Bitmap, Int) -> Unit)? = null + onFrame: ((Bitmap, Int, Float) -> Unit)? = null ) { val context = LocalContext.current val lifecycleOwner = LocalLifecycleOwner.current @@ -26,12 +34,37 @@ fun CameraPreview( val cameraController = controller ?: remember { LifecycleCameraController(context) } + // State to hold the focal length. + // Updated on the Main thread, read by the analysis background thread. + val focalLengthState = remember { mutableStateOf(0f) } + + // Periodically check/update focal length on the Main thread + LaunchedEffect(cameraController) { + while(true) { + try { + val info = cameraController.cameraInfo + if (info != null) { + val camera2Info = Camera2CameraInfo.from(info) + val focalLengths = camera2Info.getCameraCharacteristic(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS) + val fl = focalLengths?.firstOrNull() ?: 0f + focalLengthState.value = fl + } + } catch (e: Exception) { + // Ignore errors, e.g. if camera is closing or not ready + } + // Check periodically in case the active camera changes + delay(2000) + } + } + if (onFrame != null) { LaunchedEffect(cameraController) { cameraController.setImageAnalysisAnalyzer(cameraExecutor) { imageProxy -> val bitmap = imageProxy.toBitmap() val rotationDegrees = imageProxy.imageInfo.rotationDegrees - onFrame(bitmap, rotationDegrees) + val currentFocalLength = focalLengthState.value + + onFrame(bitmap, rotationDegrees, currentFocalLength) imageProxy.close() } } diff --git a/app/src/main/java/com/example/livingai/utils/Constants.kt b/app/src/main/java/com/example/livingai/utils/Constants.kt index d694245..6d40c1b 100644 --- a/app/src/main/java/com/example/livingai/utils/Constants.kt +++ b/app/src/main/java/com/example/livingai/utils/Constants.kt @@ -20,4 +20,31 @@ object Constants { val VIDEO_SILHOETTE_FRAME_HEIGHT = 300.dp const val JACCARD_THRESHOLD = 85 const val MODEL_HEIGHT = 320F -} \ No newline at end of file + + // Target operational distance (meters) + const val TARGET_DISTANCE_METERS = 5.0f + + // Tolerances + const val DISTANCE_TOLERANCE_METERS_STRICT = 0.10f + const val DISTANCE_TOLERANCE_METERS_RELAXED = 0.25f + + // ARCore confidence threshold + const val DEPTH_CONFIDENCE_MIN = 0.6f + + const val MIN_DEPTH_SAMPLES = 20 + + // IMU orientation limits + const val MAX_ACCEPTABLE_PITCH_DEGREES = 5f + const val MAX_ACCEPTABLE_ROLL_DEGREES = 3f + + // Object centering thresholds + const val CENTER_TOLERANCE_X_FRACTION = 0.15f + const val CENTER_TOLERANCE_Y_FRACTION = 0.20f + + // Fallback known-object dimension + const val DEFAULT_OBJECT_REAL_HEIGHT_METERS = 1.2f + + // Fusion weights + const val WEIGHT_ARCORE = 0.8f + const val WEIGHT_KNOWN_DIM = 0.2f +}