full pipeline optimizations

This commit is contained in:
SaiD 2025-12-13 21:25:02 +05:30
parent 986c0da505
commit 3bbf128d9c
9 changed files with 577 additions and 147 deletions

View File

@ -18,8 +18,10 @@ import org.tensorflow.lite.support.common.FileUtil
import java.io.IOException import java.io.IOException
import java.nio.ByteBuffer import java.nio.ByteBuffer
import java.nio.ByteOrder import java.nio.ByteOrder
import java.util.concurrent.atomic.AtomicBoolean
import kotlin.coroutines.resume import kotlin.coroutines.resume
import kotlin.math.abs import kotlin.math.abs
import kotlin.math.max
import kotlin.math.min import kotlin.math.min
/* ============================================================= */ /* ============================================================= */
@ -60,6 +62,10 @@ class DefaultTiltChecker : TiltChecker {
override suspend fun analyze(input: PipelineInput): Instruction { override suspend fun analyze(input: PipelineInput): Instruction {
val tolerance = 25f val tolerance = 25f
var level_message = "Keep the phone straight"
Log.d("TiltChecker", "Device Roll: ${input.deviceRoll}, Device Pitch: ${input.devicePitch}, Device Azimuth: ${input.deviceAzimuth}")
val isLevel = when (input.requiredOrientation) { val isLevel = when (input.requiredOrientation) {
CameraOrientation.PORTRAIT -> CameraOrientation.PORTRAIT ->
@ -68,8 +74,21 @@ class DefaultTiltChecker : TiltChecker {
abs(input.devicePitch) <= tolerance abs(input.devicePitch) <= tolerance
} }
val standardPitch = when (input.requiredOrientation) {
CameraOrientation.PORTRAIT -> -90f
CameraOrientation.LANDSCAPE -> 0f
}
if (!isLevel) {
if (input.devicePitch > standardPitch) {
level_message = "Rotate the phone Right"
} else if (input.devicePitch < standardPitch) {
level_message = "Rotate the phone Left"
}
}
return Instruction( return Instruction(
message = if (isLevel) "Device is level" else "Keep the phone straight", message = if (isLevel) "Device is level" else level_message,
isValid = isLevel, isValid = isLevel,
result = TiltResult(input.deviceRoll, input.devicePitch, isLevel) result = TiltResult(input.deviceRoll, input.devicePitch, isLevel)
) )
@ -191,10 +210,6 @@ class TFLiteObjectDetector(context: Context) : ObjectDetector {
data class Detection(val label: String, val confidence: Float, val bounds: RectF) data class Detection(val label: String, val confidence: Float, val bounds: RectF)
} }
/* ============================================================= */
/* POSE ANALYZER (ALIGNMENT → CROP → SEGMENT) */
/* ============================================================= */
class MockPoseAnalyzer : PoseAnalyzer { class MockPoseAnalyzer : PoseAnalyzer {
private val segmenter by lazy { private val segmenter by lazy {
@ -205,51 +220,103 @@ class MockPoseAnalyzer : PoseAnalyzer {
) )
} }
private val isSegmentationRunning = AtomicBoolean(false)
private var lastSegmentationValid: Boolean? = null
override suspend fun analyze(input: PipelineInput): Instruction { override suspend fun analyze(input: PipelineInput): Instruction {
val detection = input.previousDetectionResult val detection = input.previousDetectionResult
?: return Instruction("No detection", isValid = false) ?: return invalidate("No detection")
val cowBox = detection.animalBounds val cowBoxImage = detection.animalBounds
?: return Instruction("Cow not detected", isValid = false) ?: return invalidate("Cow not detected")
val image = input.image val image = input.image
?: return Instruction("No image", isValid = false) ?: return invalidate("No image")
val silhouette = SilhouetteManager.getSilhouette(input.orientation) val silhouette = SilhouetteManager.getSilhouette(input.orientation)
?: return Instruction("Silhouette missing", isValid = false) ?: return invalidate("Silhouette missing")
val cowBoxScreen = imageToScreenRect(
box = cowBoxImage,
imageWidth = image.width,
imageHeight = image.height,
screenWidth = input.screenWidthPx,
screenHeight = input.screenHeightPx
)
val silhouetteBoxScreen = silhouette.boundingBox
val align = checkAlignment(
detected = cowBoxScreen,
reference = silhouetteBoxScreen,
toleranceRatio = 0.15f
)
val align = checkAlignment(cowBox, silhouette.boundingBox, 0.15f)
if (align.issue != AlignmentIssue.OK) { if (align.issue != AlignmentIssue.OK) {
lastSegmentationValid = false
return alignmentToInstruction(align) return alignmentToInstruction(align)
} }
val cropped = Bitmap.createBitmap( // If segmentation already running → reuse last result
image, if (!isSegmentationRunning.compareAndSet(false, true)) {
cowBox.left.toInt(),
cowBox.top.toInt(),
cowBox.width().toInt(),
cowBox.height().toInt()
)
val resized = Bitmap.createScaledBitmap( return Instruction(
cropped, message = if (lastSegmentationValid == true)
silhouette.croppedBitmap.width, "Pose Correct"
silhouette.croppedBitmap.height, else
true "Hold steady",
) isValid = lastSegmentationValid == true,
result = detection
)
}
val mask = segment(resized) try {
?: return Instruction("Segmentation failed", isValid = false) val cropped = Bitmap.createBitmap(
image,
cowBoxImage.left.toInt(),
cowBoxImage.top.toInt(),
cowBoxImage.width().toInt(),
cowBoxImage.height().toInt()
)
val score = similarity(mask, silhouette.signedMask) val resized = Bitmap.createScaledBitmap(
val valid = score >= 0.40f cropped,
silhouette.croppedBitmap.width,
silhouette.croppedBitmap.height,
true
)
return Instruction( val mask = segment(resized)
message = if (valid) "Pose Correct" else "Adjust Position",
isValid = valid, val valid = if (mask != null) {
result = detection val score = similarity(mask, silhouette.signedMask)
) score >= 0.40f
} else {
false
}
lastSegmentationValid = valid
return Instruction(
message = if (valid) "Pose Correct" else "Adjust Position",
isValid = valid,
result = detection
)
} finally {
isSegmentationRunning.set(false)
}
}
/* -------------------------------------------------- */
/* HELPERS */
/* -------------------------------------------------- */
private fun invalidate(reason: String): Instruction {
lastSegmentationValid = false
return Instruction(reason, isValid = false)
} }
private suspend fun segment(bitmap: Bitmap): ByteArray? = private suspend fun segment(bitmap: Bitmap): ByteArray? =
@ -258,12 +325,17 @@ class MockPoseAnalyzer : PoseAnalyzer {
.addOnSuccessListener { r -> .addOnSuccessListener { r ->
val buf = r.foregroundConfidenceMask val buf = r.foregroundConfidenceMask
?: return@addOnSuccessListener cont.resume(null) ?: return@addOnSuccessListener cont.resume(null)
buf.rewind() buf.rewind()
val out = ByteArray(bitmap.width * bitmap.height) val out = ByteArray(bitmap.width * bitmap.height)
for (i in out.indices) out[i] = if (buf.get() > 0.5f) 1 else 0 for (i in out.indices) {
out[i] = if (buf.get() > 0.5f) 1 else 0
}
cont.resume(out) cont.resume(out)
} }
.addOnFailureListener { cont.resume(null) } .addOnFailureListener {
cont.resume(null)
}
} }
private fun similarity(mask: ByteArray, ref: SignedMask): Float { private fun similarity(mask: ByteArray, ref: SignedMask): Float {
@ -276,43 +348,83 @@ class MockPoseAnalyzer : PoseAnalyzer {
} }
} }
/* ============================================================= */ /* ============================================================= */
/* ALIGNMENT HELPERS */ /* ALIGNMENT HELPERS (UNCHANGED) */
/* ============================================================= */ /* ============================================================= */
enum class AlignmentIssue { TOO_SMALL, TOO_LARGE, MOVE_LEFT, MOVE_RIGHT, MOVE_UP, MOVE_DOWN, OK } enum class AlignmentIssue { TOO_SMALL, TOO_LARGE, MOVE_LEFT, MOVE_RIGHT, MOVE_UP, MOVE_DOWN, OK }
data class AlignmentResult(val issue: AlignmentIssue, val scale: Float, val dx: Float, val dy: Float) data class AlignmentResult(val issue: AlignmentIssue, val scale: Float, val dx: Float, val dy: Float)
fun checkAlignment(d: RectF, s: RectF, tol: Float): AlignmentResult { fun checkAlignment(
detected: RectF,
reference: RectF,
toleranceRatio: Float
): AlignmentResult {
val scale = min(d.width() / s.width(), d.height() / s.height()) val tolX = reference.width() * toleranceRatio
val dx = d.centerX() - s.centerX() val tolY = reference.height() * toleranceRatio
val dy = d.centerY() - s.centerY()
if (scale < 1f - tol) return AlignmentResult(AlignmentIssue.TOO_SMALL, scale, dx, dy) if (detected.left < reference.left - tolX)
if (scale > 1f + tol) return AlignmentResult(AlignmentIssue.TOO_LARGE, scale, dx, dy) return AlignmentResult(AlignmentIssue.MOVE_RIGHT, detected.width() / reference.width(), detected.left - reference.left, 0f)
val tx = s.width() * tol if (detected.right > reference.right + tolX)
val ty = s.height() * tol return AlignmentResult(AlignmentIssue.MOVE_LEFT, detected.width() / reference.width(), detected.right - reference.right, 0f)
return when { if (detected.top < reference.top - tolY)
dx > tx -> AlignmentResult(AlignmentIssue.MOVE_LEFT, scale, dx, dy) return AlignmentResult(AlignmentIssue.MOVE_DOWN, detected.height() / reference.height(), 0f, detected.top - reference.top)
dx < -tx -> AlignmentResult(AlignmentIssue.MOVE_RIGHT, scale, dx, dy)
dy > ty -> AlignmentResult(AlignmentIssue.MOVE_UP, scale, dx, dy) if (detected.bottom > reference.bottom + tolY)
dy < -ty -> AlignmentResult(AlignmentIssue.MOVE_DOWN, scale, dx, dy) return AlignmentResult(AlignmentIssue.MOVE_UP, detected.height() / reference.height(), 0f, detected.bottom - reference.bottom)
else -> AlignmentResult(AlignmentIssue.OK, scale, dx, dy)
} val scale = min(
detected.width() / reference.width(),
detected.height() / reference.height()
)
if (scale < 1f - toleranceRatio)
return AlignmentResult(AlignmentIssue.TOO_SMALL, scale, 0f, 0f)
if (scale > 1f + toleranceRatio)
return AlignmentResult(AlignmentIssue.TOO_LARGE, scale, 0f, 0f)
return AlignmentResult(AlignmentIssue.OK, scale, 0f, 0f)
} }
fun alignmentToInstruction(a: AlignmentResult) = when (a.issue) { fun alignmentToInstruction(a: AlignmentResult) = when (a.issue) {
AlignmentIssue.TOO_SMALL -> Instruction("Move closer", isValid = false) AlignmentIssue.TOO_SMALL -> Instruction("Move closer", false)
AlignmentIssue.TOO_LARGE -> Instruction("Move backward", isValid = false) AlignmentIssue.TOO_LARGE -> Instruction("Move backward", false)
AlignmentIssue.MOVE_LEFT -> Instruction("Move right", isValid = false) AlignmentIssue.MOVE_LEFT -> Instruction("Move right", false)
AlignmentIssue.MOVE_RIGHT -> Instruction("Move left", isValid = false) AlignmentIssue.MOVE_RIGHT -> Instruction("Move left", false)
AlignmentIssue.MOVE_UP -> Instruction("Move down", isValid = false) AlignmentIssue.MOVE_UP -> Instruction("Move down", false)
AlignmentIssue.MOVE_DOWN -> Instruction("Move up", isValid = false) AlignmentIssue.MOVE_DOWN -> Instruction("Move up", false)
AlignmentIssue.OK -> Instruction("Hold steady", isValid = true) AlignmentIssue.OK -> Instruction("Hold steady", true)
}
private fun imageToScreenRect(
box: RectF,
imageWidth: Int,
imageHeight: Int,
screenWidth: Float,
screenHeight: Float
): RectF {
// EXACT SAME LOGIC AS DetectionOverlay
val widthRatio = screenWidth / imageWidth
val heightRatio = screenHeight / imageHeight
val scale = max(widthRatio, heightRatio)
val offsetX = (screenWidth - imageWidth * scale) / 2f
val offsetY = (screenHeight - imageHeight * scale) / 2f
return RectF(
box.left * scale + offsetX,
box.top * scale + offsetY,
box.right * scale + offsetX,
box.bottom * scale + offsetY
)
} }
/* ============================================================= */ /* ============================================================= */

View File

@ -8,6 +8,12 @@ import androidx.datastore.preferences.preferencesDataStore
import androidx.window.layout.WindowMetricsCalculator import androidx.window.layout.WindowMetricsCalculator
import coil.ImageLoader import coil.ImageLoader
import coil.decode.SvgDecoder import coil.decode.SvgDecoder
import com.example.livingai.data.camera.DefaultCaptureHandler
import com.example.livingai.data.camera.DefaultMeasurementCalculator
import com.example.livingai.data.camera.DefaultOrientationChecker
import com.example.livingai.data.camera.DefaultTiltChecker
import com.example.livingai.data.camera.MockPoseAnalyzer
import com.example.livingai.data.camera.TFLiteObjectDetector
import com.example.livingai.data.local.CSVDataSource import com.example.livingai.data.local.CSVDataSource
import com.example.livingai.data.ml.AIModelImpl import com.example.livingai.data.ml.AIModelImpl
import com.example.livingai.data.repository.AppDataRepositoryImpl import com.example.livingai.data.repository.AppDataRepositoryImpl
@ -16,6 +22,11 @@ import com.example.livingai.data.repository.business.AnimalProfileRepositoryImpl
import com.example.livingai.data.repository.business.AnimalRatingRepositoryImpl import com.example.livingai.data.repository.business.AnimalRatingRepositoryImpl
import com.example.livingai.data.repository.media.CameraRepositoryImpl import com.example.livingai.data.repository.media.CameraRepositoryImpl
import com.example.livingai.data.repository.media.VideoRepositoryImpl import com.example.livingai.data.repository.media.VideoRepositoryImpl
import com.example.livingai.domain.camera.CaptureHandler
import com.example.livingai.domain.camera.MeasurementCalculator
import com.example.livingai.domain.camera.OrientationChecker
import com.example.livingai.domain.camera.PoseAnalyzer
import com.example.livingai.domain.camera.TiltChecker
import com.example.livingai.domain.ml.AIModel import com.example.livingai.domain.ml.AIModel
import com.example.livingai.domain.ml.AnalyzerThresholds import com.example.livingai.domain.ml.AnalyzerThresholds
import com.example.livingai.domain.ml.FeedbackAnalyzer import com.example.livingai.domain.ml.FeedbackAnalyzer
@ -59,13 +70,12 @@ import com.example.livingai.utils.ScreenDimensions
import com.example.livingai.utils.SilhouetteManager import com.example.livingai.utils.SilhouetteManager
import com.example.livingai.utils.TiltSensorManager import com.example.livingai.utils.TiltSensorManager
import org.koin.android.ext.koin.androidContext import org.koin.android.ext.koin.androidContext
import org.koin.androidx.viewmodel.dsl.viewModel import org.koin.core.module.dsl.viewModel
import org.koin.dsl.module import org.koin.dsl.module
private val Context.dataStore: DataStore<Preferences> by preferencesDataStore(name = Constants.USER_SETTINGS) private val Context.dataStore: DataStore<Preferences> by preferencesDataStore(name = Constants.USER_SETTINGS)
val appModule = module { val appModule = module {
includes(cameraModule)
single<DataStore<Preferences>> { androidContext().dataStore } single<DataStore<Preferences>> { androidContext().dataStore }
@ -101,6 +111,15 @@ val appModule = module {
.build() .build()
} }
factory<OrientationChecker> { DefaultOrientationChecker() }
factory<TiltChecker> { DefaultTiltChecker() }
factory<com.example.livingai.domain.camera.ObjectDetector> { TFLiteObjectDetector(androidContext()) }
factory<PoseAnalyzer> { MockPoseAnalyzer() }
// Handlers
factory<CaptureHandler> { DefaultCaptureHandler() }
factory<MeasurementCalculator> { DefaultMeasurementCalculator() }
// Initialize silhouettes once // Initialize silhouettes once
single<ScreenDimensions>(createdAtStart = true) { single<ScreenDimensions>(createdAtStart = true) {
val ctx: Context = androidContext() val ctx: Context = androidContext()

View File

@ -2,17 +2,10 @@ package com.example.livingai.di
import com.example.livingai.data.camera.* import com.example.livingai.data.camera.*
import com.example.livingai.domain.camera.* import com.example.livingai.domain.camera.*
import com.example.livingai.utils.ScreenDimensions
import org.koin.android.ext.koin.androidContext import org.koin.android.ext.koin.androidContext
import org.koin.dsl.module import org.koin.dsl.module
val cameraModule = module { val cameraModule = module {
// Pipeline Steps // Pipeline Steps
factory<OrientationChecker> { DefaultOrientationChecker() }
factory<TiltChecker> { DefaultTiltChecker() }
factory<ObjectDetector> { TFLiteObjectDetector(androidContext()) }
factory<PoseAnalyzer> { MockPoseAnalyzer() }
// Handlers
factory<CaptureHandler> { DefaultCaptureHandler() }
factory<MeasurementCalculator> { DefaultMeasurementCalculator() }
} }

View File

@ -22,6 +22,8 @@ data class PipelineInput(
val devicePitch: Float, val devicePitch: Float,
val deviceAzimuth: Float, val deviceAzimuth: Float,
val requiredOrientation: CameraOrientation, val requiredOrientation: CameraOrientation,
val screenWidthPx: Float,
val screenHeightPx: Float,
val targetAnimal: String, // e.g., "Dog", "Cat" val targetAnimal: String, // e.g., "Dog", "Cat"
val orientation: String, // "front", "back", "side", etc. val orientation: String, // "front", "back", "side", etc.
val previousDetectionResult: DetectionResult? = null // To pass detection result to subsequent steps val previousDetectionResult: DetectionResult? = null // To pass detection result to subsequent steps

View File

@ -12,8 +12,8 @@ import android.graphics.RectF
*/ */
data class Instruction( data class Instruction(
val message: String, val message: String,
val animationResId: Int? = null,
val isValid: Boolean, val isValid: Boolean,
val animationResId: Int? = null,
val result: AnalysisResult? = null val result: AnalysisResult? = null
) )

View File

@ -24,8 +24,9 @@ import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.graphics.drawscope.Stroke import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.platform.LocalConfiguration
import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.res.painterResource import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
@ -77,26 +78,36 @@ fun ActiveCameraScreen(
viewModel: CameraViewModel, viewModel: CameraViewModel,
) { ) {
val context = LocalContext.current val context = LocalContext.current
val configuration = LocalConfiguration.current
val density = LocalDensity.current
val screenWidthPx = with(density) { configuration.screenWidthDp.dp.toPx() }
val screenHeightPx = with(density) { configuration.screenHeightDp.dp.toPx() }
val screenWidthState = rememberUpdatedState(screenWidthPx)
val screenHeightState = rememberUpdatedState(screenHeightPx)
var analysisImageSize by remember { mutableStateOf(Size(0f, 0f)) } var analysisImageSize by remember { mutableStateOf(Size(0f, 0f)) }
val controller = remember { val controller = remember {
LifecycleCameraController(context).apply { LifecycleCameraController(context).apply {
setEnabledUseCases(LifecycleCameraController.IMAGE_ANALYSIS or LifecycleCameraController.IMAGE_CAPTURE) setEnabledUseCases(LifecycleCameraController.IMAGE_ANALYSIS or LifecycleCameraController.IMAGE_CAPTURE)
setImageAnalysisAnalyzer( setImageAnalysisAnalyzer(
ContextCompat.getMainExecutor(context), ContextCompat.getMainExecutor(context)
{ imageProxy -> ) { imageProxy ->
val bitmap = imageProxy.toBitmap() val bitmap = imageProxy.toBitmap()
val rotation = imageProxy.imageInfo.rotationDegrees val rotation = imageProxy.imageInfo.rotationDegrees
val rotatedBitmap = rotateBitmap(bitmap, rotation) val rotatedBitmap = rotateBitmap(bitmap, rotation)
analysisImageSize = Size(rotatedBitmap.width.toFloat(), rotatedBitmap.height.toFloat()) analysisImageSize = Size(rotatedBitmap.width.toFloat(), rotatedBitmap.height.toFloat())
viewModel.processFrame( viewModel.processFrame(
image = rotatedBitmap, image = rotatedBitmap,
deviceOrientation = rotation deviceOrientation = rotation,
) screenWidth = screenWidthState.value,
imageProxy.close() screenHeight = screenHeightState.value
} )
) imageProxy.close()
}
} }
} }
@ -112,7 +123,9 @@ fun ActiveCameraScreen(
viewModel.onCaptureClicked( viewModel.onCaptureClicked(
image = rotatedBitmap, image = rotatedBitmap,
deviceOrientation = rotation deviceOrientation = rotation,
screenWidth = screenWidthState.value,
screenHeight = screenHeightState.value
) )
image.close() image.close()
} }
@ -132,8 +145,8 @@ fun ActiveCameraScreen(
} }
}, },
floatingActionButtonPosition = FabPosition.Center floatingActionButtonPosition = FabPosition.Center
) { paddingValues -> ) { _ ->
Box(modifier = Modifier.fillMaxSize().padding(paddingValues)) { Box(modifier = Modifier.fillMaxSize()) {
CameraPreview( CameraPreview(
modifier = Modifier.fillMaxSize(), modifier = Modifier.fillMaxSize(),
controller = controller controller = controller
@ -142,6 +155,8 @@ fun ActiveCameraScreen(
// Silhouette Overlay // Silhouette Overlay
if (uiState.targetOrientation.isNotEmpty()) { if (uiState.targetOrientation.isNotEmpty()) {
val silhouette = SilhouetteManager.getOriginal(uiState.targetOrientation) val silhouette = SilhouetteManager.getOriginal(uiState.targetOrientation)
val silhouetteData = SilhouetteManager.getSilhouette(uiState.targetOrientation)
if (silhouette != null) { if (silhouette != null) {
Image( Image(
bitmap = silhouette.asImageBitmap(), bitmap = silhouette.asImageBitmap(),
@ -150,6 +165,28 @@ fun ActiveCameraScreen(
contentScale = ContentScale.Fit contentScale = ContentScale.Fit
) )
} }
if (silhouetteData != null) {
if (silhouetteData.signedMask?.debugBitmap != null) {
val bbox = silhouetteData.boundingBox
Box(modifier = Modifier.fillMaxSize()) {
// Bounding box outline (same coordinate system)
Canvas(modifier = Modifier.fillMaxSize()) {
drawRect(
color = Color.Red,
topLeft = Offset(bbox.left, bbox.top),
size = Size(bbox.width(), bbox.height()),
style = Stroke(width = 2.dp.toPx())
)
}
}
}
}
} }
// Overlays // Overlays

View File

@ -50,7 +50,9 @@ class CameraViewModel(
fun processFrame( fun processFrame(
image: Bitmap, image: Bitmap,
deviceOrientation: Int deviceOrientation: Int,
screenWidth: Float,
screenHeight: Float
) { ) {
frameCounter = (frameCounter + 1) % frameSkipInterval frameCounter = (frameCounter + 1) % frameSkipInterval
if (frameCounter != 0) return if (frameCounter != 0) return
@ -64,6 +66,8 @@ class CameraViewModel(
devicePitch = currentTilt.first, devicePitch = currentTilt.first,
deviceAzimuth = currentTilt.third, deviceAzimuth = currentTilt.third,
requiredOrientation = _uiState.value.requiredOrientation, requiredOrientation = _uiState.value.requiredOrientation,
screenWidthPx = screenWidth,
screenHeightPx = screenHeight,
targetAnimal = "Cow", // Assuming Cow for now, can be parameter later targetAnimal = "Cow", // Assuming Cow for now, can be parameter later
orientation = _uiState.value.targetOrientation, orientation = _uiState.value.targetOrientation,
previousDetectionResult = _uiState.value.lastDetectionResult previousDetectionResult = _uiState.value.lastDetectionResult
@ -93,7 +97,7 @@ class CameraViewModel(
// Step 4: Check Pose (Silhouette matching) // Step 4: Check Pose (Silhouette matching)
val poseInstruction = poseAnalyzer.analyze(input.copy(previousDetectionResult = detectionInstruction.result as? DetectionResult)) val poseInstruction = poseAnalyzer.analyze(input.copy(previousDetectionResult = detectionInstruction.result as? DetectionResult))
if (!poseInstruction.isValid) { if (!poseInstruction.isValid) {
updateState(poseInstruction) updateState(poseInstruction, detectionInstruction.result as? DetectionResult)
return@launch return@launch
} }
@ -106,8 +110,8 @@ class CameraViewModel(
} }
} }
private fun updateState(instruction: Instruction) { private fun updateState(instruction: Instruction, latestDetectionResult: DetectionResult? = null) {
val detectionResult = instruction.result as? DetectionResult val detectionResult = (instruction.result as? DetectionResult) ?: latestDetectionResult
_uiState.value = _uiState.value.copy( _uiState.value = _uiState.value.copy(
currentInstruction = instruction, currentInstruction = instruction,
isReadyToCapture = false, isReadyToCapture = false,
@ -115,7 +119,12 @@ class CameraViewModel(
) )
} }
fun onCaptureClicked(image: Bitmap, deviceOrientation: Int) { fun onCaptureClicked(
image: Bitmap,
deviceOrientation: Int,
screenWidth: Float,
screenHeight: Float
) {
viewModelScope.launch { viewModelScope.launch {
val detectionResult = _uiState.value.lastDetectionResult ?: return@launch val detectionResult = _uiState.value.lastDetectionResult ?: return@launch
val currentTilt = tilt.value val currentTilt = tilt.value
@ -127,6 +136,8 @@ class CameraViewModel(
devicePitch = currentTilt.first, devicePitch = currentTilt.first,
deviceAzimuth = currentTilt.third, deviceAzimuth = currentTilt.third,
requiredOrientation = _uiState.value.requiredOrientation, requiredOrientation = _uiState.value.requiredOrientation,
screenWidthPx = screenWidth,
screenHeightPx = screenHeight,
targetAnimal = "Cow", targetAnimal = "Cow",
orientation = _uiState.value.targetOrientation orientation = _uiState.value.targetOrientation
) )

View File

@ -7,25 +7,48 @@ import com.example.livingai.R
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import kotlin.math.min import kotlin.math.min
/* ============================================================= */
/* CONFIG */
/* ============================================================= */
private const val TAG = "SilhouetteManager"
private const val DEBUG = true // toggle logs here
/* ============================================================= */
/* DATA MODELS */
/* ============================================================= */
data class SignedMask( data class SignedMask(
val mask: Array<FloatArray>, val mask: Array<FloatArray>,
val maxValue: Float val maxValue: Float,
val debugBitmap: Bitmap? = null // <-- NEW
) )
data class SilhouetteData( data class SilhouetteData(
val croppedBitmap: Bitmap, val croppedBitmap: Bitmap, // tightly cropped silhouette
val boundingBox: RectF, val boundingBox: RectF, // bbox in SCREEN coordinates
val signedMask: SignedMask val signedMask: SignedMask
) )
/* ============================================================= */
/* MANAGER */
/* ============================================================= */
object SilhouetteManager { object SilhouetteManager {
private val originals = ConcurrentHashMap<String, Bitmap>() private val originals = ConcurrentHashMap<String, Bitmap>()
private val inverted = ConcurrentHashMap<String, Bitmap>()
private val silhouettes = ConcurrentHashMap<String, SilhouetteData>() private val silhouettes = ConcurrentHashMap<String, SilhouetteData>()
fun getOriginal(name: String): Bitmap? = originals[name] fun getOriginal(name: String): Bitmap? = originals[name]
fun getInverted(name: String): Bitmap? = inverted[name]
fun getSilhouette(name: String): SilhouetteData? = silhouettes[name] fun getSilhouette(name: String): SilhouetteData? = silhouettes[name]
/* ========================================================= */
/* INITIALIZATION (RUN ONCE AT APP START) */
/* ========================================================= */
fun initialize(context: Context, screenW: Int, screenH: Int) { fun initialize(context: Context, screenW: Int, screenH: Int) {
val res = context.resources val res = context.resources
@ -42,27 +65,74 @@ object SilhouetteManager {
map.forEach { (name, resId) -> map.forEach { (name, resId) ->
val src = BitmapFactory.decodeResource(res, resId) if (DEBUG) Log.d(TAG, "---- Processing silhouette: $name ----")
originals[name] = src
val fitted = Bitmap.createScaledBitmap( val src = BitmapFactory.decodeResource(res, resId)
invertToPurple(src),
screenW, val isPortrait = name == "front" || name == "back"
screenH, val targetW = if (isPortrait) screenW else screenH
true val targetH = if (isPortrait) screenH else screenW
/* -------------------------------------------------- */
/* 1. FIT_CENTER (TRANSPARENT BACKGROUND!) */
/* -------------------------------------------------- */
val fitCentered = scaleFitCenter(
src = src,
targetW = targetW,
targetH = targetH,
backgroundColor = Color.BLACK
) )
val bbox = computeBoundingBox(fitted) originals[name] = fitCentered
if (DEBUG) {
Log.d(TAG, "fitCenter size=${fitCentered.width}x${fitCentered.height}")
}
/* -------------------------------------------------- */
/* 2. INVERT TO PURPLE MASK */
/* -------------------------------------------------- */
val invertedBmp = invertToPurple(fitCentered)
inverted[name] = invertedBmp
/* -------------------------------------------------- */
/* 3. COMPUTE BOUNDING BOX */
/* -------------------------------------------------- */
val bbox = computeBoundingBox(invertedBmp)
if (bbox.width() <= 0 || bbox.height() <= 0) {
Log.e(TAG, "❌ EMPTY bounding box for $name — skipping")
return@forEach
}
if (DEBUG) {
Log.d(
TAG,
"bbox = [${bbox.left}, ${bbox.top}, ${bbox.right}, ${bbox.bottom}] " +
"(${bbox.width()}x${bbox.height()})"
)
}
/* -------------------------------------------------- */
/* 4. CROP TO SILHOUETTE */
/* -------------------------------------------------- */
val cropped = Bitmap.createBitmap( val cropped = Bitmap.createBitmap(
fitted, invertedBmp,
bbox.left.toInt(), bbox.left.toInt(),
bbox.top.toInt(), bbox.top.toInt(),
bbox.width().toInt(), bbox.width().toInt(),
bbox.height().toInt() bbox.height().toInt()
) )
val signedMask = createSignedWeightedMask(cropped) /* -------------------------------------------------- */
/* 5. CREATE SIGNED WEIGHTED MASK */
/* -------------------------------------------------- */
val signedMask = createSignedWeightedMask(cropped, 50, 50)
silhouettes[name] = SilhouetteData( silhouettes[name] = SilhouetteData(
croppedBitmap = cropped, croppedBitmap = cropped,
@ -70,13 +140,130 @@ object SilhouetteManager {
signedMask = signedMask signedMask = signedMask
) )
Log.d("Silhouette", "Loaded $name (${bbox.width()} x ${bbox.height()})") if (DEBUG) {
Log.d(
TAG,
"✅ Loaded $name | cropped=${cropped.width}x${cropped.height} " +
"| maxWeight=${signedMask.maxValue}"
)
}
} }
} }
/* ---------------------------------------------------------- */ private fun signedMaskToDebugBitmap(
mask: Array<FloatArray>
): Bitmap {
val h = mask.size
val w = mask[0].size
val bmp = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888)
val pixels = IntArray(w * h)
fun clamp01(v: Float) = v.coerceIn(0f, 1f)
for (y in 0 until h) {
for (x in 0 until w) {
val v = mask[y][x]
val alpha = (clamp01(kotlin.math.abs(v)) * 255f).toInt()
val color =
if (v >= 0f) {
// PINK for positive
Color.argb(alpha, 255, 105, 180)
} else {
// GREEN for negative
Color.argb(alpha, 0, 200, 0)
}
pixels[y * w + x] = color
}
}
bmp.setPixels(pixels, 0, w, 0, 0, w, h)
return bmp
}
/* ========================================================= */
/* FIT_CENTER (TRANSPARENT, SAFE FOR GEOMETRY) */
/* ========================================================= */
/**
* Scales a bitmap using FIT_CENTER semantics into a target canvas.
*
* - Aspect ratio preserved
* - Image centered
* - Uncovered areas filled with backgroundColor
* - Image pixels preserved exactly (including transparency)
*/
fun scaleFitCenter(
src: Bitmap,
targetW: Int,
targetH: Int,
backgroundColor: Int = Color.BLACK
): Bitmap {
require(targetW > 0 && targetH > 0) {
"Target width/height must be > 0"
}
val scale = min(
targetW.toFloat() / src.width,
targetH.toFloat() / src.height
)
val scaledW = (src.width * scale).toInt()
val scaledH = (src.height * scale).toInt()
val left = (targetW - scaledW) / 2f
val top = (targetH - scaledH) / 2f
val edgeWidth = if (left == 0f) targetW else left.toInt()
val edgeHeight = if (top == 0f) targetH else top.toInt()
val edgeLeft = if (left == 0f) 0f else scaledW + left
val edgeTop = if (top == 0f) 0f else scaledH + top
Log.d("ScalingData", "Target H = ${targetH}, Target W = ${targetW}, Scaled H = ${scaledH}, Scaled W = ${scaledW}")
val scaled = Bitmap.createScaledBitmap(
src,
scaledW,
scaledH,
true
)
val output = Bitmap.createBitmap(
targetW,
targetH,
Bitmap.Config.ARGB_8888
)
val edgeBitmap = Bitmap.createBitmap(
edgeWidth,
edgeHeight,
Bitmap.Config.ARGB_8888
)
edgeBitmap.eraseColor(backgroundColor)
val canvas = Canvas(output)
// 3. Draw bitmap as-is
canvas.drawBitmap(scaled, left, top, null)
canvas.drawBitmap(edgeBitmap, 0f, 0f, null)
canvas.drawBitmap(edgeBitmap, edgeLeft, edgeTop, null)
return output
}
/* ========================================================= */
/* INVERT TO PURPLE (ALPHA IS SEMANTIC) */
/* ========================================================= */
private fun invertToPurple(src: Bitmap): Bitmap { private fun invertToPurple(src: Bitmap): Bitmap {
val w = src.width val w = src.width
val h = src.height val h = src.height
val pixels = IntArray(w * h) val pixels = IntArray(w * h)
@ -84,15 +271,30 @@ object SilhouetteManager {
val purple = Color.argb(255, 128, 0, 128) val purple = Color.argb(255, 128, 0, 128)
var opaque = 0
var transparent = 0
for (i in pixels.indices) { for (i in pixels.indices) {
pixels[i] = if ((pixels[i] ushr 24) == 0) {
if ((pixels[i] ushr 24) == 0) purple pixels[i] = purple
else 0x00000000 opaque++
} else {
pixels[i] = 0x00000000
transparent++
}
}
if (DEBUG) {
Log.d(TAG, "invertToPurple: opaque=$opaque transparent=$transparent")
} }
return Bitmap.createBitmap(pixels, w, h, Bitmap.Config.ARGB_8888) return Bitmap.createBitmap(pixels, w, h, Bitmap.Config.ARGB_8888)
} }
/* ========================================================= */
/* BOUNDING BOX */
/* ========================================================= */
private fun computeBoundingBox(bitmap: Bitmap): RectF { private fun computeBoundingBox(bitmap: Bitmap): RectF {
val w = bitmap.width val w = bitmap.width
@ -102,8 +304,9 @@ object SilhouetteManager {
var minX = w var minX = w
var minY = h var minY = h
var maxX = 0 var maxX = -1
var maxY = 0 var maxY = -1
var count = 0
for (y in 0 until h) { for (y in 0 until h) {
for (x in 0 until w) { for (x in 0 until w) {
@ -112,26 +315,33 @@ object SilhouetteManager {
minY = min(minY, y) minY = min(minY, y)
maxX = maxOf(maxX, x) maxX = maxOf(maxX, x)
maxY = maxOf(maxY, y) maxY = maxOf(maxY, y)
count++
} }
} }
} }
if (DEBUG) {
Log.d(TAG, "bbox pixel count=$count")
}
if (count == 0) return RectF()
return RectF( return RectF(
minX.toFloat(), minX.toFloat(),
minY.toFloat(), minY.toFloat(),
maxX.toFloat(), (maxX + 1).toFloat(),
maxY.toFloat() (maxY + 1).toFloat()
) )
} }
/* ---------------------------------------------------------- */ /* ========================================================= */
/* SIGNED WEIGHTED MASK */ /* SIGNED WEIGHTED MASK */
/* ---------------------------------------------------------- */ /* ========================================================= */
fun createSignedWeightedMask( fun createSignedWeightedMask(
bitmap: Bitmap, bitmap: Bitmap,
fadeInside: Int = 10, fadeInside: Int = 100,
fadeOutside: Int = 20 fadeOutside: Int = 200
): SignedMask { ): SignedMask {
val w = bitmap.width val w = bitmap.width
@ -140,54 +350,100 @@ object SilhouetteManager {
val pixels = IntArray(w * h) val pixels = IntArray(w * h)
bitmap.getPixels(pixels, 0, w, 0, 0, w, h) bitmap.getPixels(pixels, 0, w, 0, 0, w, h)
val inside = IntArray(w * h) // ------------------------------------------------------------
for (i in pixels.indices) // 1. Inside mask (alpha > 0)
inside[i] = if ((pixels[i] ushr 24) > 0) 1 else 0 // ------------------------------------------------------------
val inside = BooleanArray(w * h) {
(pixels[it] ushr 24) > 0
}
fun idx(x: Int, y: Int) = y * w + x fun idx(x: Int, y: Int) = y * w + x
val distIn = IntArray(w * h) { Int.MAX_VALUE } // ------------------------------------------------------------
val distOut = IntArray(w * h) { Int.MAX_VALUE } // 2. Distance to nearest boundary (multi-source BFS)
// ------------------------------------------------------------
for (i in inside.indices) { val dist = IntArray(w * h) { Int.MAX_VALUE }
if (inside[i] == 0) distIn[i] = 0 val queue = ArrayDeque<Int>()
else distOut[i] = 0
}
for (y in 0 until h) for (y in 0 until h) {
for (x in 0 until w) { for (x in 0 until w) {
val i = idx(x, y) val i = idx(x, y)
if (x > 0) distIn[i] = min(distIn[i], distIn[idx(x - 1, y)] + 1) val v = inside[i]
if (y > 0) distIn[i] = min(distIn[i], distIn[idx(x, y - 1)] + 1)
if (x > 0) distOut[i] = min(distOut[i], distOut[idx(x - 1, y)] + 1)
if (y > 0) distOut[i] = min(distOut[i], distOut[idx(x, y - 1)] + 1)
}
for (y in h - 1 downTo 0) val isBoundary =
for (x in w - 1 downTo 0) { (x > 0 && inside[idx(x - 1, y)] != v) ||
val i = idx(x, y) (y > 0 && inside[idx(x, y - 1)] != v) ||
if (x < w - 1) distIn[i] = min(distIn[i], distIn[idx(x + 1, y)] + 1) (x < w - 1 && inside[idx(x + 1, y)] != v) ||
if (y < h - 1) distIn[i] = min(distIn[i], distIn[idx(x, y + 1)] + 1) (y < h - 1 && inside[idx(x, y + 1)] != v)
if (x < w - 1) distOut[i] = min(distOut[i], distOut[idx(x + 1, y)] + 1)
if (y < h - 1) distOut[i] = min(distOut[i], distOut[idx(x, y + 1)] + 1) if (isBoundary) {
dist[i] = 0
queue.add(i)
}
} }
}
val dx = intArrayOf(-1, 1, 0, 0)
val dy = intArrayOf(0, 0, -1, 1)
while (queue.isNotEmpty()) {
val i = queue.removeFirst()
val x = i % w
val y = i / w
for (k in 0..3) {
val nx = x + dx[k]
val ny = y + dy[k]
if (nx in 0 until w && ny in 0 until h) {
val ni = idx(nx, ny)
if (dist[ni] > dist[i] + 1) {
dist[ni] = dist[i] + 1
queue.add(ni)
}
}
}
}
// ------------------------------------------------------------
// 3. Create signed weighted mask
// ------------------------------------------------------------
val mask = Array(h) { FloatArray(w) } val mask = Array(h) { FloatArray(w) }
var maxVal = Float.NEGATIVE_INFINITY var maxVal = Float.NEGATIVE_INFINITY
for (y in 0 until h) for (y in 0 until h) {
for (x in 0 until w) { for (x in 0 until w) {
val i = idx(x, y) val i = idx(x, y)
val d = dist[i].toFloat()
val v = val v =
if (inside[i] == 1) if (inside[i]) {
min(1f, distIn[i].toFloat() / fadeInside) // INSIDE → positive
else (d / fadeInside).coerceIn(0f, 1f)
maxOf(-1f, -distOut[i].toFloat() / fadeOutside) } else {
// OUTSIDE → negative everywhere
-(d / fadeOutside).coerceIn(0f, 1f)
}
mask[y][x] = v mask[y][x] = v
if (v > maxVal) maxVal = v if (v > maxVal) maxVal = v
} }
}
return SignedMask(mask, maxVal) // ------------------------------------------------------------
// 4. Debug bitmap (pink = +, green = -)
// ------------------------------------------------------------
val debugBitmap = signedMaskToDebugBitmap(mask)
return SignedMask(
mask = mask,
maxValue = maxVal,
debugBitmap = debugBitmap
)
} }
} }