full pipeline optimizations
This commit is contained in:
parent
986c0da505
commit
3bbf128d9c
|
|
@ -18,8 +18,10 @@ import org.tensorflow.lite.support.common.FileUtil
|
|||
import java.io.IOException
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import kotlin.coroutines.resume
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.max
|
||||
import kotlin.math.min
|
||||
|
||||
/* ============================================================= */
|
||||
|
|
@ -60,6 +62,10 @@ class DefaultTiltChecker : TiltChecker {
|
|||
override suspend fun analyze(input: PipelineInput): Instruction {
|
||||
|
||||
val tolerance = 25f
|
||||
var level_message = "Keep the phone straight"
|
||||
|
||||
Log.d("TiltChecker", "Device Roll: ${input.deviceRoll}, Device Pitch: ${input.devicePitch}, Device Azimuth: ${input.deviceAzimuth}")
|
||||
|
||||
|
||||
val isLevel = when (input.requiredOrientation) {
|
||||
CameraOrientation.PORTRAIT ->
|
||||
|
|
@ -68,8 +74,21 @@ class DefaultTiltChecker : TiltChecker {
|
|||
abs(input.devicePitch) <= tolerance
|
||||
}
|
||||
|
||||
val standardPitch = when (input.requiredOrientation) {
|
||||
CameraOrientation.PORTRAIT -> -90f
|
||||
CameraOrientation.LANDSCAPE -> 0f
|
||||
}
|
||||
|
||||
if (!isLevel) {
|
||||
if (input.devicePitch > standardPitch) {
|
||||
level_message = "Rotate the phone Right"
|
||||
} else if (input.devicePitch < standardPitch) {
|
||||
level_message = "Rotate the phone Left"
|
||||
}
|
||||
}
|
||||
|
||||
return Instruction(
|
||||
message = if (isLevel) "Device is level" else "Keep the phone straight",
|
||||
message = if (isLevel) "Device is level" else level_message,
|
||||
isValid = isLevel,
|
||||
result = TiltResult(input.deviceRoll, input.devicePitch, isLevel)
|
||||
)
|
||||
|
|
@ -191,10 +210,6 @@ class TFLiteObjectDetector(context: Context) : ObjectDetector {
|
|||
data class Detection(val label: String, val confidence: Float, val bounds: RectF)
|
||||
}
|
||||
|
||||
/* ============================================================= */
|
||||
/* POSE ANALYZER (ALIGNMENT → CROP → SEGMENT) */
|
||||
/* ============================================================= */
|
||||
|
||||
class MockPoseAnalyzer : PoseAnalyzer {
|
||||
|
||||
private val segmenter by lazy {
|
||||
|
|
@ -205,31 +220,65 @@ class MockPoseAnalyzer : PoseAnalyzer {
|
|||
)
|
||||
}
|
||||
|
||||
private val isSegmentationRunning = AtomicBoolean(false)
|
||||
|
||||
private var lastSegmentationValid: Boolean? = null
|
||||
|
||||
override suspend fun analyze(input: PipelineInput): Instruction {
|
||||
|
||||
val detection = input.previousDetectionResult
|
||||
?: return Instruction("No detection", isValid = false)
|
||||
?: return invalidate("No detection")
|
||||
|
||||
val cowBox = detection.animalBounds
|
||||
?: return Instruction("Cow not detected", isValid = false)
|
||||
val cowBoxImage = detection.animalBounds
|
||||
?: return invalidate("Cow not detected")
|
||||
|
||||
val image = input.image
|
||||
?: return Instruction("No image", isValid = false)
|
||||
?: return invalidate("No image")
|
||||
|
||||
val silhouette = SilhouetteManager.getSilhouette(input.orientation)
|
||||
?: return Instruction("Silhouette missing", isValid = false)
|
||||
?: return invalidate("Silhouette missing")
|
||||
|
||||
val cowBoxScreen = imageToScreenRect(
|
||||
box = cowBoxImage,
|
||||
imageWidth = image.width,
|
||||
imageHeight = image.height,
|
||||
screenWidth = input.screenWidthPx,
|
||||
screenHeight = input.screenHeightPx
|
||||
)
|
||||
|
||||
val silhouetteBoxScreen = silhouette.boundingBox
|
||||
|
||||
val align = checkAlignment(
|
||||
detected = cowBoxScreen,
|
||||
reference = silhouetteBoxScreen,
|
||||
toleranceRatio = 0.15f
|
||||
)
|
||||
|
||||
val align = checkAlignment(cowBox, silhouette.boundingBox, 0.15f)
|
||||
if (align.issue != AlignmentIssue.OK) {
|
||||
lastSegmentationValid = false
|
||||
return alignmentToInstruction(align)
|
||||
}
|
||||
|
||||
// If segmentation already running → reuse last result
|
||||
if (!isSegmentationRunning.compareAndSet(false, true)) {
|
||||
|
||||
return Instruction(
|
||||
message = if (lastSegmentationValid == true)
|
||||
"Pose Correct"
|
||||
else
|
||||
"Hold steady",
|
||||
isValid = lastSegmentationValid == true,
|
||||
result = detection
|
||||
)
|
||||
}
|
||||
|
||||
try {
|
||||
val cropped = Bitmap.createBitmap(
|
||||
image,
|
||||
cowBox.left.toInt(),
|
||||
cowBox.top.toInt(),
|
||||
cowBox.width().toInt(),
|
||||
cowBox.height().toInt()
|
||||
cowBoxImage.left.toInt(),
|
||||
cowBoxImage.top.toInt(),
|
||||
cowBoxImage.width().toInt(),
|
||||
cowBoxImage.height().toInt()
|
||||
)
|
||||
|
||||
val resized = Bitmap.createScaledBitmap(
|
||||
|
|
@ -240,16 +289,34 @@ class MockPoseAnalyzer : PoseAnalyzer {
|
|||
)
|
||||
|
||||
val mask = segment(resized)
|
||||
?: return Instruction("Segmentation failed", isValid = false)
|
||||
|
||||
val valid = if (mask != null) {
|
||||
val score = similarity(mask, silhouette.signedMask)
|
||||
val valid = score >= 0.40f
|
||||
score >= 0.40f
|
||||
} else {
|
||||
false
|
||||
}
|
||||
|
||||
lastSegmentationValid = valid
|
||||
|
||||
return Instruction(
|
||||
message = if (valid) "Pose Correct" else "Adjust Position",
|
||||
isValid = valid,
|
||||
result = detection
|
||||
)
|
||||
|
||||
} finally {
|
||||
isSegmentationRunning.set(false)
|
||||
}
|
||||
}
|
||||
|
||||
/* -------------------------------------------------- */
|
||||
/* HELPERS */
|
||||
/* -------------------------------------------------- */
|
||||
|
||||
private fun invalidate(reason: String): Instruction {
|
||||
lastSegmentationValid = false
|
||||
return Instruction(reason, isValid = false)
|
||||
}
|
||||
|
||||
private suspend fun segment(bitmap: Bitmap): ByteArray? =
|
||||
|
|
@ -258,12 +325,17 @@ class MockPoseAnalyzer : PoseAnalyzer {
|
|||
.addOnSuccessListener { r ->
|
||||
val buf = r.foregroundConfidenceMask
|
||||
?: return@addOnSuccessListener cont.resume(null)
|
||||
|
||||
buf.rewind()
|
||||
val out = ByteArray(bitmap.width * bitmap.height)
|
||||
for (i in out.indices) out[i] = if (buf.get() > 0.5f) 1 else 0
|
||||
for (i in out.indices) {
|
||||
out[i] = if (buf.get() > 0.5f) 1 else 0
|
||||
}
|
||||
cont.resume(out)
|
||||
}
|
||||
.addOnFailureListener { cont.resume(null) }
|
||||
.addOnFailureListener {
|
||||
cont.resume(null)
|
||||
}
|
||||
}
|
||||
|
||||
private fun similarity(mask: ByteArray, ref: SignedMask): Float {
|
||||
|
|
@ -276,43 +348,83 @@ class MockPoseAnalyzer : PoseAnalyzer {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* ============================================================= */
|
||||
/* ALIGNMENT HELPERS */
|
||||
/* ALIGNMENT HELPERS (UNCHANGED) */
|
||||
/* ============================================================= */
|
||||
|
||||
enum class AlignmentIssue { TOO_SMALL, TOO_LARGE, MOVE_LEFT, MOVE_RIGHT, MOVE_UP, MOVE_DOWN, OK }
|
||||
|
||||
data class AlignmentResult(val issue: AlignmentIssue, val scale: Float, val dx: Float, val dy: Float)
|
||||
|
||||
fun checkAlignment(d: RectF, s: RectF, tol: Float): AlignmentResult {
|
||||
fun checkAlignment(
|
||||
detected: RectF,
|
||||
reference: RectF,
|
||||
toleranceRatio: Float
|
||||
): AlignmentResult {
|
||||
|
||||
val scale = min(d.width() / s.width(), d.height() / s.height())
|
||||
val dx = d.centerX() - s.centerX()
|
||||
val dy = d.centerY() - s.centerY()
|
||||
val tolX = reference.width() * toleranceRatio
|
||||
val tolY = reference.height() * toleranceRatio
|
||||
|
||||
if (scale < 1f - tol) return AlignmentResult(AlignmentIssue.TOO_SMALL, scale, dx, dy)
|
||||
if (scale > 1f + tol) return AlignmentResult(AlignmentIssue.TOO_LARGE, scale, dx, dy)
|
||||
if (detected.left < reference.left - tolX)
|
||||
return AlignmentResult(AlignmentIssue.MOVE_RIGHT, detected.width() / reference.width(), detected.left - reference.left, 0f)
|
||||
|
||||
val tx = s.width() * tol
|
||||
val ty = s.height() * tol
|
||||
if (detected.right > reference.right + tolX)
|
||||
return AlignmentResult(AlignmentIssue.MOVE_LEFT, detected.width() / reference.width(), detected.right - reference.right, 0f)
|
||||
|
||||
return when {
|
||||
dx > tx -> AlignmentResult(AlignmentIssue.MOVE_LEFT, scale, dx, dy)
|
||||
dx < -tx -> AlignmentResult(AlignmentIssue.MOVE_RIGHT, scale, dx, dy)
|
||||
dy > ty -> AlignmentResult(AlignmentIssue.MOVE_UP, scale, dx, dy)
|
||||
dy < -ty -> AlignmentResult(AlignmentIssue.MOVE_DOWN, scale, dx, dy)
|
||||
else -> AlignmentResult(AlignmentIssue.OK, scale, dx, dy)
|
||||
}
|
||||
if (detected.top < reference.top - tolY)
|
||||
return AlignmentResult(AlignmentIssue.MOVE_DOWN, detected.height() / reference.height(), 0f, detected.top - reference.top)
|
||||
|
||||
if (detected.bottom > reference.bottom + tolY)
|
||||
return AlignmentResult(AlignmentIssue.MOVE_UP, detected.height() / reference.height(), 0f, detected.bottom - reference.bottom)
|
||||
|
||||
val scale = min(
|
||||
detected.width() / reference.width(),
|
||||
detected.height() / reference.height()
|
||||
)
|
||||
|
||||
if (scale < 1f - toleranceRatio)
|
||||
return AlignmentResult(AlignmentIssue.TOO_SMALL, scale, 0f, 0f)
|
||||
|
||||
if (scale > 1f + toleranceRatio)
|
||||
return AlignmentResult(AlignmentIssue.TOO_LARGE, scale, 0f, 0f)
|
||||
|
||||
return AlignmentResult(AlignmentIssue.OK, scale, 0f, 0f)
|
||||
}
|
||||
|
||||
fun alignmentToInstruction(a: AlignmentResult) = when (a.issue) {
|
||||
AlignmentIssue.TOO_SMALL -> Instruction("Move closer", isValid = false)
|
||||
AlignmentIssue.TOO_LARGE -> Instruction("Move backward", isValid = false)
|
||||
AlignmentIssue.MOVE_LEFT -> Instruction("Move right", isValid = false)
|
||||
AlignmentIssue.MOVE_RIGHT -> Instruction("Move left", isValid = false)
|
||||
AlignmentIssue.MOVE_UP -> Instruction("Move down", isValid = false)
|
||||
AlignmentIssue.MOVE_DOWN -> Instruction("Move up", isValid = false)
|
||||
AlignmentIssue.OK -> Instruction("Hold steady", isValid = true)
|
||||
AlignmentIssue.TOO_SMALL -> Instruction("Move closer", false)
|
||||
AlignmentIssue.TOO_LARGE -> Instruction("Move backward", false)
|
||||
AlignmentIssue.MOVE_LEFT -> Instruction("Move right", false)
|
||||
AlignmentIssue.MOVE_RIGHT -> Instruction("Move left", false)
|
||||
AlignmentIssue.MOVE_UP -> Instruction("Move down", false)
|
||||
AlignmentIssue.MOVE_DOWN -> Instruction("Move up", false)
|
||||
AlignmentIssue.OK -> Instruction("Hold steady", true)
|
||||
}
|
||||
|
||||
private fun imageToScreenRect(
|
||||
box: RectF,
|
||||
imageWidth: Int,
|
||||
imageHeight: Int,
|
||||
screenWidth: Float,
|
||||
screenHeight: Float
|
||||
): RectF {
|
||||
|
||||
// EXACT SAME LOGIC AS DetectionOverlay
|
||||
val widthRatio = screenWidth / imageWidth
|
||||
val heightRatio = screenHeight / imageHeight
|
||||
val scale = max(widthRatio, heightRatio)
|
||||
|
||||
val offsetX = (screenWidth - imageWidth * scale) / 2f
|
||||
val offsetY = (screenHeight - imageHeight * scale) / 2f
|
||||
|
||||
return RectF(
|
||||
box.left * scale + offsetX,
|
||||
box.top * scale + offsetY,
|
||||
box.right * scale + offsetX,
|
||||
box.bottom * scale + offsetY
|
||||
)
|
||||
}
|
||||
|
||||
/* ============================================================= */
|
||||
|
|
|
|||
|
|
@ -8,6 +8,12 @@ import androidx.datastore.preferences.preferencesDataStore
|
|||
import androidx.window.layout.WindowMetricsCalculator
|
||||
import coil.ImageLoader
|
||||
import coil.decode.SvgDecoder
|
||||
import com.example.livingai.data.camera.DefaultCaptureHandler
|
||||
import com.example.livingai.data.camera.DefaultMeasurementCalculator
|
||||
import com.example.livingai.data.camera.DefaultOrientationChecker
|
||||
import com.example.livingai.data.camera.DefaultTiltChecker
|
||||
import com.example.livingai.data.camera.MockPoseAnalyzer
|
||||
import com.example.livingai.data.camera.TFLiteObjectDetector
|
||||
import com.example.livingai.data.local.CSVDataSource
|
||||
import com.example.livingai.data.ml.AIModelImpl
|
||||
import com.example.livingai.data.repository.AppDataRepositoryImpl
|
||||
|
|
@ -16,6 +22,11 @@ import com.example.livingai.data.repository.business.AnimalProfileRepositoryImpl
|
|||
import com.example.livingai.data.repository.business.AnimalRatingRepositoryImpl
|
||||
import com.example.livingai.data.repository.media.CameraRepositoryImpl
|
||||
import com.example.livingai.data.repository.media.VideoRepositoryImpl
|
||||
import com.example.livingai.domain.camera.CaptureHandler
|
||||
import com.example.livingai.domain.camera.MeasurementCalculator
|
||||
import com.example.livingai.domain.camera.OrientationChecker
|
||||
import com.example.livingai.domain.camera.PoseAnalyzer
|
||||
import com.example.livingai.domain.camera.TiltChecker
|
||||
import com.example.livingai.domain.ml.AIModel
|
||||
import com.example.livingai.domain.ml.AnalyzerThresholds
|
||||
import com.example.livingai.domain.ml.FeedbackAnalyzer
|
||||
|
|
@ -59,13 +70,12 @@ import com.example.livingai.utils.ScreenDimensions
|
|||
import com.example.livingai.utils.SilhouetteManager
|
||||
import com.example.livingai.utils.TiltSensorManager
|
||||
import org.koin.android.ext.koin.androidContext
|
||||
import org.koin.androidx.viewmodel.dsl.viewModel
|
||||
import org.koin.core.module.dsl.viewModel
|
||||
import org.koin.dsl.module
|
||||
|
||||
private val Context.dataStore: DataStore<Preferences> by preferencesDataStore(name = Constants.USER_SETTINGS)
|
||||
|
||||
val appModule = module {
|
||||
includes(cameraModule)
|
||||
|
||||
single<DataStore<Preferences>> { androidContext().dataStore }
|
||||
|
||||
|
|
@ -101,6 +111,15 @@ val appModule = module {
|
|||
.build()
|
||||
}
|
||||
|
||||
factory<OrientationChecker> { DefaultOrientationChecker() }
|
||||
factory<TiltChecker> { DefaultTiltChecker() }
|
||||
factory<com.example.livingai.domain.camera.ObjectDetector> { TFLiteObjectDetector(androidContext()) }
|
||||
factory<PoseAnalyzer> { MockPoseAnalyzer() }
|
||||
|
||||
// Handlers
|
||||
factory<CaptureHandler> { DefaultCaptureHandler() }
|
||||
factory<MeasurementCalculator> { DefaultMeasurementCalculator() }
|
||||
|
||||
// Initialize silhouettes once
|
||||
single<ScreenDimensions>(createdAtStart = true) {
|
||||
val ctx: Context = androidContext()
|
||||
|
|
|
|||
|
|
@ -2,17 +2,10 @@ package com.example.livingai.di
|
|||
|
||||
import com.example.livingai.data.camera.*
|
||||
import com.example.livingai.domain.camera.*
|
||||
import com.example.livingai.utils.ScreenDimensions
|
||||
import org.koin.android.ext.koin.androidContext
|
||||
import org.koin.dsl.module
|
||||
|
||||
val cameraModule = module {
|
||||
// Pipeline Steps
|
||||
factory<OrientationChecker> { DefaultOrientationChecker() }
|
||||
factory<TiltChecker> { DefaultTiltChecker() }
|
||||
factory<ObjectDetector> { TFLiteObjectDetector(androidContext()) }
|
||||
factory<PoseAnalyzer> { MockPoseAnalyzer() }
|
||||
|
||||
// Handlers
|
||||
factory<CaptureHandler> { DefaultCaptureHandler() }
|
||||
factory<MeasurementCalculator> { DefaultMeasurementCalculator() }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ data class PipelineInput(
|
|||
val devicePitch: Float,
|
||||
val deviceAzimuth: Float,
|
||||
val requiredOrientation: CameraOrientation,
|
||||
val screenWidthPx: Float,
|
||||
val screenHeightPx: Float,
|
||||
val targetAnimal: String, // e.g., "Dog", "Cat"
|
||||
val orientation: String, // "front", "back", "side", etc.
|
||||
val previousDetectionResult: DetectionResult? = null // To pass detection result to subsequent steps
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ import android.graphics.RectF
|
|||
*/
|
||||
data class Instruction(
|
||||
val message: String,
|
||||
val animationResId: Int? = null,
|
||||
val isValid: Boolean,
|
||||
val animationResId: Int? = null,
|
||||
val result: AnalysisResult? = null
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -24,8 +24,9 @@ import androidx.compose.ui.graphics.Color
|
|||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.graphics.drawscope.Stroke
|
||||
import androidx.compose.ui.layout.ContentScale
|
||||
import androidx.compose.ui.platform.LocalConfiguration
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.res.painterResource
|
||||
import androidx.compose.ui.platform.LocalDensity
|
||||
import androidx.compose.ui.text.style.TextAlign
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.core.content.ContextCompat
|
||||
|
|
@ -77,14 +78,23 @@ fun ActiveCameraScreen(
|
|||
viewModel: CameraViewModel,
|
||||
) {
|
||||
val context = LocalContext.current
|
||||
val configuration = LocalConfiguration.current
|
||||
val density = LocalDensity.current
|
||||
|
||||
val screenWidthPx = with(density) { configuration.screenWidthDp.dp.toPx() }
|
||||
val screenHeightPx = with(density) { configuration.screenHeightDp.dp.toPx() }
|
||||
|
||||
val screenWidthState = rememberUpdatedState(screenWidthPx)
|
||||
val screenHeightState = rememberUpdatedState(screenHeightPx)
|
||||
|
||||
var analysisImageSize by remember { mutableStateOf(Size(0f, 0f)) }
|
||||
|
||||
val controller = remember {
|
||||
LifecycleCameraController(context).apply {
|
||||
setEnabledUseCases(LifecycleCameraController.IMAGE_ANALYSIS or LifecycleCameraController.IMAGE_CAPTURE)
|
||||
setImageAnalysisAnalyzer(
|
||||
ContextCompat.getMainExecutor(context),
|
||||
{ imageProxy ->
|
||||
ContextCompat.getMainExecutor(context)
|
||||
) { imageProxy ->
|
||||
val bitmap = imageProxy.toBitmap()
|
||||
val rotation = imageProxy.imageInfo.rotationDegrees
|
||||
val rotatedBitmap = rotateBitmap(bitmap, rotation)
|
||||
|
|
@ -92,11 +102,12 @@ fun ActiveCameraScreen(
|
|||
|
||||
viewModel.processFrame(
|
||||
image = rotatedBitmap,
|
||||
deviceOrientation = rotation
|
||||
deviceOrientation = rotation,
|
||||
screenWidth = screenWidthState.value,
|
||||
screenHeight = screenHeightState.value
|
||||
)
|
||||
imageProxy.close()
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -112,7 +123,9 @@ fun ActiveCameraScreen(
|
|||
|
||||
viewModel.onCaptureClicked(
|
||||
image = rotatedBitmap,
|
||||
deviceOrientation = rotation
|
||||
deviceOrientation = rotation,
|
||||
screenWidth = screenWidthState.value,
|
||||
screenHeight = screenHeightState.value
|
||||
)
|
||||
image.close()
|
||||
}
|
||||
|
|
@ -132,8 +145,8 @@ fun ActiveCameraScreen(
|
|||
}
|
||||
},
|
||||
floatingActionButtonPosition = FabPosition.Center
|
||||
) { paddingValues ->
|
||||
Box(modifier = Modifier.fillMaxSize().padding(paddingValues)) {
|
||||
) { _ ->
|
||||
Box(modifier = Modifier.fillMaxSize()) {
|
||||
CameraPreview(
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
controller = controller
|
||||
|
|
@ -142,6 +155,8 @@ fun ActiveCameraScreen(
|
|||
// Silhouette Overlay
|
||||
if (uiState.targetOrientation.isNotEmpty()) {
|
||||
val silhouette = SilhouetteManager.getOriginal(uiState.targetOrientation)
|
||||
val silhouetteData = SilhouetteManager.getSilhouette(uiState.targetOrientation)
|
||||
|
||||
if (silhouette != null) {
|
||||
Image(
|
||||
bitmap = silhouette.asImageBitmap(),
|
||||
|
|
@ -150,6 +165,28 @@ fun ActiveCameraScreen(
|
|||
contentScale = ContentScale.Fit
|
||||
)
|
||||
}
|
||||
|
||||
if (silhouetteData != null) {
|
||||
|
||||
if (silhouetteData.signedMask?.debugBitmap != null) {
|
||||
|
||||
val bbox = silhouetteData.boundingBox
|
||||
|
||||
Box(modifier = Modifier.fillMaxSize()) {
|
||||
|
||||
// Bounding box outline (same coordinate system)
|
||||
Canvas(modifier = Modifier.fillMaxSize()) {
|
||||
drawRect(
|
||||
color = Color.Red,
|
||||
topLeft = Offset(bbox.left, bbox.top),
|
||||
size = Size(bbox.width(), bbox.height()),
|
||||
style = Stroke(width = 2.dp.toPx())
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Overlays
|
||||
|
|
|
|||
|
|
@ -50,7 +50,9 @@ class CameraViewModel(
|
|||
|
||||
fun processFrame(
|
||||
image: Bitmap,
|
||||
deviceOrientation: Int
|
||||
deviceOrientation: Int,
|
||||
screenWidth: Float,
|
||||
screenHeight: Float
|
||||
) {
|
||||
frameCounter = (frameCounter + 1) % frameSkipInterval
|
||||
if (frameCounter != 0) return
|
||||
|
|
@ -64,6 +66,8 @@ class CameraViewModel(
|
|||
devicePitch = currentTilt.first,
|
||||
deviceAzimuth = currentTilt.third,
|
||||
requiredOrientation = _uiState.value.requiredOrientation,
|
||||
screenWidthPx = screenWidth,
|
||||
screenHeightPx = screenHeight,
|
||||
targetAnimal = "Cow", // Assuming Cow for now, can be parameter later
|
||||
orientation = _uiState.value.targetOrientation,
|
||||
previousDetectionResult = _uiState.value.lastDetectionResult
|
||||
|
|
@ -93,7 +97,7 @@ class CameraViewModel(
|
|||
// Step 4: Check Pose (Silhouette matching)
|
||||
val poseInstruction = poseAnalyzer.analyze(input.copy(previousDetectionResult = detectionInstruction.result as? DetectionResult))
|
||||
if (!poseInstruction.isValid) {
|
||||
updateState(poseInstruction)
|
||||
updateState(poseInstruction, detectionInstruction.result as? DetectionResult)
|
||||
return@launch
|
||||
}
|
||||
|
||||
|
|
@ -106,8 +110,8 @@ class CameraViewModel(
|
|||
}
|
||||
}
|
||||
|
||||
private fun updateState(instruction: Instruction) {
|
||||
val detectionResult = instruction.result as? DetectionResult
|
||||
private fun updateState(instruction: Instruction, latestDetectionResult: DetectionResult? = null) {
|
||||
val detectionResult = (instruction.result as? DetectionResult) ?: latestDetectionResult
|
||||
_uiState.value = _uiState.value.copy(
|
||||
currentInstruction = instruction,
|
||||
isReadyToCapture = false,
|
||||
|
|
@ -115,7 +119,12 @@ class CameraViewModel(
|
|||
)
|
||||
}
|
||||
|
||||
fun onCaptureClicked(image: Bitmap, deviceOrientation: Int) {
|
||||
fun onCaptureClicked(
|
||||
image: Bitmap,
|
||||
deviceOrientation: Int,
|
||||
screenWidth: Float,
|
||||
screenHeight: Float
|
||||
) {
|
||||
viewModelScope.launch {
|
||||
val detectionResult = _uiState.value.lastDetectionResult ?: return@launch
|
||||
val currentTilt = tilt.value
|
||||
|
|
@ -127,6 +136,8 @@ class CameraViewModel(
|
|||
devicePitch = currentTilt.first,
|
||||
deviceAzimuth = currentTilt.third,
|
||||
requiredOrientation = _uiState.value.requiredOrientation,
|
||||
screenWidthPx = screenWidth,
|
||||
screenHeightPx = screenHeight,
|
||||
targetAnimal = "Cow",
|
||||
orientation = _uiState.value.targetOrientation
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,25 +7,48 @@ import com.example.livingai.R
|
|||
import java.util.concurrent.ConcurrentHashMap
|
||||
import kotlin.math.min
|
||||
|
||||
/* ============================================================= */
|
||||
/* CONFIG */
|
||||
/* ============================================================= */
|
||||
|
||||
private const val TAG = "SilhouetteManager"
|
||||
private const val DEBUG = true // toggle logs here
|
||||
|
||||
/* ============================================================= */
|
||||
/* DATA MODELS */
|
||||
/* ============================================================= */
|
||||
|
||||
data class SignedMask(
|
||||
val mask: Array<FloatArray>,
|
||||
val maxValue: Float
|
||||
val maxValue: Float,
|
||||
val debugBitmap: Bitmap? = null // <-- NEW
|
||||
)
|
||||
|
||||
|
||||
data class SilhouetteData(
|
||||
val croppedBitmap: Bitmap,
|
||||
val boundingBox: RectF,
|
||||
val croppedBitmap: Bitmap, // tightly cropped silhouette
|
||||
val boundingBox: RectF, // bbox in SCREEN coordinates
|
||||
val signedMask: SignedMask
|
||||
)
|
||||
|
||||
/* ============================================================= */
|
||||
/* MANAGER */
|
||||
/* ============================================================= */
|
||||
|
||||
object SilhouetteManager {
|
||||
|
||||
private val originals = ConcurrentHashMap<String, Bitmap>()
|
||||
private val inverted = ConcurrentHashMap<String, Bitmap>()
|
||||
private val silhouettes = ConcurrentHashMap<String, SilhouetteData>()
|
||||
|
||||
fun getOriginal(name: String): Bitmap? = originals[name]
|
||||
fun getInverted(name: String): Bitmap? = inverted[name]
|
||||
fun getSilhouette(name: String): SilhouetteData? = silhouettes[name]
|
||||
|
||||
/* ========================================================= */
|
||||
/* INITIALIZATION (RUN ONCE AT APP START) */
|
||||
/* ========================================================= */
|
||||
|
||||
fun initialize(context: Context, screenW: Int, screenH: Int) {
|
||||
|
||||
val res = context.resources
|
||||
|
|
@ -42,27 +65,74 @@ object SilhouetteManager {
|
|||
|
||||
map.forEach { (name, resId) ->
|
||||
|
||||
val src = BitmapFactory.decodeResource(res, resId)
|
||||
originals[name] = src
|
||||
if (DEBUG) Log.d(TAG, "---- Processing silhouette: $name ----")
|
||||
|
||||
val fitted = Bitmap.createScaledBitmap(
|
||||
invertToPurple(src),
|
||||
screenW,
|
||||
screenH,
|
||||
true
|
||||
val src = BitmapFactory.decodeResource(res, resId)
|
||||
|
||||
val isPortrait = name == "front" || name == "back"
|
||||
val targetW = if (isPortrait) screenW else screenH
|
||||
val targetH = if (isPortrait) screenH else screenW
|
||||
|
||||
/* -------------------------------------------------- */
|
||||
/* 1. FIT_CENTER (TRANSPARENT BACKGROUND!) */
|
||||
/* -------------------------------------------------- */
|
||||
|
||||
val fitCentered = scaleFitCenter(
|
||||
src = src,
|
||||
targetW = targetW,
|
||||
targetH = targetH,
|
||||
backgroundColor = Color.BLACK
|
||||
)
|
||||
|
||||
val bbox = computeBoundingBox(fitted)
|
||||
originals[name] = fitCentered
|
||||
|
||||
if (DEBUG) {
|
||||
Log.d(TAG, "fitCenter size=${fitCentered.width}x${fitCentered.height}")
|
||||
}
|
||||
|
||||
/* -------------------------------------------------- */
|
||||
/* 2. INVERT TO PURPLE MASK */
|
||||
/* -------------------------------------------------- */
|
||||
|
||||
val invertedBmp = invertToPurple(fitCentered)
|
||||
inverted[name] = invertedBmp
|
||||
|
||||
/* -------------------------------------------------- */
|
||||
/* 3. COMPUTE BOUNDING BOX */
|
||||
/* -------------------------------------------------- */
|
||||
|
||||
val bbox = computeBoundingBox(invertedBmp)
|
||||
|
||||
if (bbox.width() <= 0 || bbox.height() <= 0) {
|
||||
Log.e(TAG, "❌ EMPTY bounding box for $name — skipping")
|
||||
return@forEach
|
||||
}
|
||||
|
||||
if (DEBUG) {
|
||||
Log.d(
|
||||
TAG,
|
||||
"bbox = [${bbox.left}, ${bbox.top}, ${bbox.right}, ${bbox.bottom}] " +
|
||||
"(${bbox.width()}x${bbox.height()})"
|
||||
)
|
||||
}
|
||||
|
||||
/* -------------------------------------------------- */
|
||||
/* 4. CROP TO SILHOUETTE */
|
||||
/* -------------------------------------------------- */
|
||||
|
||||
val cropped = Bitmap.createBitmap(
|
||||
fitted,
|
||||
invertedBmp,
|
||||
bbox.left.toInt(),
|
||||
bbox.top.toInt(),
|
||||
bbox.width().toInt(),
|
||||
bbox.height().toInt()
|
||||
)
|
||||
|
||||
val signedMask = createSignedWeightedMask(cropped)
|
||||
/* -------------------------------------------------- */
|
||||
/* 5. CREATE SIGNED WEIGHTED MASK */
|
||||
/* -------------------------------------------------- */
|
||||
|
||||
val signedMask = createSignedWeightedMask(cropped, 50, 50)
|
||||
|
||||
silhouettes[name] = SilhouetteData(
|
||||
croppedBitmap = cropped,
|
||||
|
|
@ -70,13 +140,130 @@ object SilhouetteManager {
|
|||
signedMask = signedMask
|
||||
)
|
||||
|
||||
Log.d("Silhouette", "Loaded $name (${bbox.width()} x ${bbox.height()})")
|
||||
if (DEBUG) {
|
||||
Log.d(
|
||||
TAG,
|
||||
"✅ Loaded $name | cropped=${cropped.width}x${cropped.height} " +
|
||||
"| maxWeight=${signedMask.maxValue}"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------- */
|
||||
private fun signedMaskToDebugBitmap(
|
||||
mask: Array<FloatArray>
|
||||
): Bitmap {
|
||||
|
||||
val h = mask.size
|
||||
val w = mask[0].size
|
||||
|
||||
val bmp = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888)
|
||||
val pixels = IntArray(w * h)
|
||||
|
||||
fun clamp01(v: Float) = v.coerceIn(0f, 1f)
|
||||
|
||||
for (y in 0 until h) {
|
||||
for (x in 0 until w) {
|
||||
|
||||
val v = mask[y][x]
|
||||
val alpha = (clamp01(kotlin.math.abs(v)) * 255f).toInt()
|
||||
|
||||
val color =
|
||||
if (v >= 0f) {
|
||||
// PINK for positive
|
||||
Color.argb(alpha, 255, 105, 180)
|
||||
} else {
|
||||
// GREEN for negative
|
||||
Color.argb(alpha, 0, 200, 0)
|
||||
}
|
||||
|
||||
pixels[y * w + x] = color
|
||||
}
|
||||
}
|
||||
|
||||
bmp.setPixels(pixels, 0, w, 0, 0, w, h)
|
||||
return bmp
|
||||
}
|
||||
|
||||
/* ========================================================= */
|
||||
/* FIT_CENTER (TRANSPARENT, SAFE FOR GEOMETRY) */
|
||||
/* ========================================================= */
|
||||
|
||||
/**
|
||||
* Scales a bitmap using FIT_CENTER semantics into a target canvas.
|
||||
*
|
||||
* - Aspect ratio preserved
|
||||
* - Image centered
|
||||
* - Uncovered areas filled with backgroundColor
|
||||
* - Image pixels preserved exactly (including transparency)
|
||||
*/
|
||||
fun scaleFitCenter(
|
||||
src: Bitmap,
|
||||
targetW: Int,
|
||||
targetH: Int,
|
||||
backgroundColor: Int = Color.BLACK
|
||||
): Bitmap {
|
||||
|
||||
require(targetW > 0 && targetH > 0) {
|
||||
"Target width/height must be > 0"
|
||||
}
|
||||
|
||||
val scale = min(
|
||||
targetW.toFloat() / src.width,
|
||||
targetH.toFloat() / src.height
|
||||
)
|
||||
|
||||
val scaledW = (src.width * scale).toInt()
|
||||
val scaledH = (src.height * scale).toInt()
|
||||
|
||||
val left = (targetW - scaledW) / 2f
|
||||
val top = (targetH - scaledH) / 2f
|
||||
|
||||
val edgeWidth = if (left == 0f) targetW else left.toInt()
|
||||
val edgeHeight = if (top == 0f) targetH else top.toInt()
|
||||
|
||||
val edgeLeft = if (left == 0f) 0f else scaledW + left
|
||||
val edgeTop = if (top == 0f) 0f else scaledH + top
|
||||
|
||||
Log.d("ScalingData", "Target H = ${targetH}, Target W = ${targetW}, Scaled H = ${scaledH}, Scaled W = ${scaledW}")
|
||||
|
||||
val scaled = Bitmap.createScaledBitmap(
|
||||
src,
|
||||
scaledW,
|
||||
scaledH,
|
||||
true
|
||||
)
|
||||
|
||||
val output = Bitmap.createBitmap(
|
||||
targetW,
|
||||
targetH,
|
||||
Bitmap.Config.ARGB_8888
|
||||
)
|
||||
|
||||
val edgeBitmap = Bitmap.createBitmap(
|
||||
edgeWidth,
|
||||
edgeHeight,
|
||||
Bitmap.Config.ARGB_8888
|
||||
)
|
||||
edgeBitmap.eraseColor(backgroundColor)
|
||||
|
||||
val canvas = Canvas(output)
|
||||
|
||||
// 3. Draw bitmap as-is
|
||||
canvas.drawBitmap(scaled, left, top, null)
|
||||
canvas.drawBitmap(edgeBitmap, 0f, 0f, null)
|
||||
canvas.drawBitmap(edgeBitmap, edgeLeft, edgeTop, null)
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
|
||||
/* ========================================================= */
|
||||
/* INVERT TO PURPLE (ALPHA IS SEMANTIC) */
|
||||
/* ========================================================= */
|
||||
|
||||
private fun invertToPurple(src: Bitmap): Bitmap {
|
||||
|
||||
val w = src.width
|
||||
val h = src.height
|
||||
val pixels = IntArray(w * h)
|
||||
|
|
@ -84,15 +271,30 @@ object SilhouetteManager {
|
|||
|
||||
val purple = Color.argb(255, 128, 0, 128)
|
||||
|
||||
var opaque = 0
|
||||
var transparent = 0
|
||||
|
||||
for (i in pixels.indices) {
|
||||
pixels[i] =
|
||||
if ((pixels[i] ushr 24) == 0) purple
|
||||
else 0x00000000
|
||||
if ((pixels[i] ushr 24) == 0) {
|
||||
pixels[i] = purple
|
||||
opaque++
|
||||
} else {
|
||||
pixels[i] = 0x00000000
|
||||
transparent++
|
||||
}
|
||||
}
|
||||
|
||||
if (DEBUG) {
|
||||
Log.d(TAG, "invertToPurple: opaque=$opaque transparent=$transparent")
|
||||
}
|
||||
|
||||
return Bitmap.createBitmap(pixels, w, h, Bitmap.Config.ARGB_8888)
|
||||
}
|
||||
|
||||
/* ========================================================= */
|
||||
/* BOUNDING BOX */
|
||||
/* ========================================================= */
|
||||
|
||||
private fun computeBoundingBox(bitmap: Bitmap): RectF {
|
||||
|
||||
val w = bitmap.width
|
||||
|
|
@ -102,8 +304,9 @@ object SilhouetteManager {
|
|||
|
||||
var minX = w
|
||||
var minY = h
|
||||
var maxX = 0
|
||||
var maxY = 0
|
||||
var maxX = -1
|
||||
var maxY = -1
|
||||
var count = 0
|
||||
|
||||
for (y in 0 until h) {
|
||||
for (x in 0 until w) {
|
||||
|
|
@ -112,26 +315,33 @@ object SilhouetteManager {
|
|||
minY = min(minY, y)
|
||||
maxX = maxOf(maxX, x)
|
||||
maxY = maxOf(maxY, y)
|
||||
count++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (DEBUG) {
|
||||
Log.d(TAG, "bbox pixel count=$count")
|
||||
}
|
||||
|
||||
if (count == 0) return RectF()
|
||||
|
||||
return RectF(
|
||||
minX.toFloat(),
|
||||
minY.toFloat(),
|
||||
maxX.toFloat(),
|
||||
maxY.toFloat()
|
||||
(maxX + 1).toFloat(),
|
||||
(maxY + 1).toFloat()
|
||||
)
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------- */
|
||||
/* ========================================================= */
|
||||
/* SIGNED WEIGHTED MASK */
|
||||
/* ---------------------------------------------------------- */
|
||||
/* ========================================================= */
|
||||
|
||||
fun createSignedWeightedMask(
|
||||
bitmap: Bitmap,
|
||||
fadeInside: Int = 10,
|
||||
fadeOutside: Int = 20
|
||||
fadeInside: Int = 100,
|
||||
fadeOutside: Int = 200
|
||||
): SignedMask {
|
||||
|
||||
val w = bitmap.width
|
||||
|
|
@ -140,54 +350,100 @@ object SilhouetteManager {
|
|||
val pixels = IntArray(w * h)
|
||||
bitmap.getPixels(pixels, 0, w, 0, 0, w, h)
|
||||
|
||||
val inside = IntArray(w * h)
|
||||
for (i in pixels.indices)
|
||||
inside[i] = if ((pixels[i] ushr 24) > 0) 1 else 0
|
||||
// ------------------------------------------------------------
|
||||
// 1. Inside mask (alpha > 0)
|
||||
// ------------------------------------------------------------
|
||||
|
||||
val inside = BooleanArray(w * h) {
|
||||
(pixels[it] ushr 24) > 0
|
||||
}
|
||||
|
||||
fun idx(x: Int, y: Int) = y * w + x
|
||||
|
||||
val distIn = IntArray(w * h) { Int.MAX_VALUE }
|
||||
val distOut = IntArray(w * h) { Int.MAX_VALUE }
|
||||
// ------------------------------------------------------------
|
||||
// 2. Distance to nearest boundary (multi-source BFS)
|
||||
// ------------------------------------------------------------
|
||||
|
||||
for (i in inside.indices) {
|
||||
if (inside[i] == 0) distIn[i] = 0
|
||||
else distOut[i] = 0
|
||||
}
|
||||
val dist = IntArray(w * h) { Int.MAX_VALUE }
|
||||
val queue = ArrayDeque<Int>()
|
||||
|
||||
for (y in 0 until h)
|
||||
for (y in 0 until h) {
|
||||
for (x in 0 until w) {
|
||||
val i = idx(x, y)
|
||||
if (x > 0) distIn[i] = min(distIn[i], distIn[idx(x - 1, y)] + 1)
|
||||
if (y > 0) distIn[i] = min(distIn[i], distIn[idx(x, y - 1)] + 1)
|
||||
if (x > 0) distOut[i] = min(distOut[i], distOut[idx(x - 1, y)] + 1)
|
||||
if (y > 0) distOut[i] = min(distOut[i], distOut[idx(x, y - 1)] + 1)
|
||||
val v = inside[i]
|
||||
|
||||
val isBoundary =
|
||||
(x > 0 && inside[idx(x - 1, y)] != v) ||
|
||||
(y > 0 && inside[idx(x, y - 1)] != v) ||
|
||||
(x < w - 1 && inside[idx(x + 1, y)] != v) ||
|
||||
(y < h - 1 && inside[idx(x, y + 1)] != v)
|
||||
|
||||
if (isBoundary) {
|
||||
dist[i] = 0
|
||||
queue.add(i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (y in h - 1 downTo 0)
|
||||
for (x in w - 1 downTo 0) {
|
||||
val i = idx(x, y)
|
||||
if (x < w - 1) distIn[i] = min(distIn[i], distIn[idx(x + 1, y)] + 1)
|
||||
if (y < h - 1) distIn[i] = min(distIn[i], distIn[idx(x, y + 1)] + 1)
|
||||
if (x < w - 1) distOut[i] = min(distOut[i], distOut[idx(x + 1, y)] + 1)
|
||||
if (y < h - 1) distOut[i] = min(distOut[i], distOut[idx(x, y + 1)] + 1)
|
||||
val dx = intArrayOf(-1, 1, 0, 0)
|
||||
val dy = intArrayOf(0, 0, -1, 1)
|
||||
|
||||
while (queue.isNotEmpty()) {
|
||||
val i = queue.removeFirst()
|
||||
val x = i % w
|
||||
val y = i / w
|
||||
|
||||
for (k in 0..3) {
|
||||
val nx = x + dx[k]
|
||||
val ny = y + dy[k]
|
||||
|
||||
if (nx in 0 until w && ny in 0 until h) {
|
||||
val ni = idx(nx, ny)
|
||||
if (dist[ni] > dist[i] + 1) {
|
||||
dist[ni] = dist[i] + 1
|
||||
queue.add(ni)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// 3. Create signed weighted mask
|
||||
// ------------------------------------------------------------
|
||||
|
||||
val mask = Array(h) { FloatArray(w) }
|
||||
var maxVal = Float.NEGATIVE_INFINITY
|
||||
|
||||
for (y in 0 until h)
|
||||
for (y in 0 until h) {
|
||||
for (x in 0 until w) {
|
||||
val i = idx(x, y)
|
||||
val d = dist[i].toFloat()
|
||||
|
||||
val v =
|
||||
if (inside[i] == 1)
|
||||
min(1f, distIn[i].toFloat() / fadeInside)
|
||||
else
|
||||
maxOf(-1f, -distOut[i].toFloat() / fadeOutside)
|
||||
if (inside[i]) {
|
||||
// INSIDE → positive
|
||||
(d / fadeInside).coerceIn(0f, 1f)
|
||||
} else {
|
||||
// OUTSIDE → negative everywhere
|
||||
-(d / fadeOutside).coerceIn(0f, 1f)
|
||||
}
|
||||
|
||||
mask[y][x] = v
|
||||
if (v > maxVal) maxVal = v
|
||||
}
|
||||
|
||||
return SignedMask(mask, maxVal)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// 4. Debug bitmap (pink = +, green = -)
|
||||
// ------------------------------------------------------------
|
||||
|
||||
val debugBitmap = signedMaskToDebugBitmap(mask)
|
||||
|
||||
return SignedMask(
|
||||
mask = mask,
|
||||
maxValue = maxVal,
|
||||
debugBitmap = debugBitmap
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue