feat: integrate YOLO runtime, update screens, and sync docs

walkguide-mobile/walkguide_app/lib/core/ai/yolo_detector.dart

@@ -1,19 +1,32 @@
+import 'dart:math' as math;
+
+import 'package:camera/camera.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart';
 
 import '../constants/app_constants.dart';
 import 'obstacle_analyzer.dart';
+import 'yolo_runtime.dart';
 
 class YoloDetector {
   final ObstacleAnalyzer _analyzer;
   List<String> _labels = const [];
+  YoloRuntime? _runtime;
+  List<int> _inputShape = const [];
+  List<int> _outputShape = const [];
+  YoloTensorType? _inputType;
+  YoloTensorType? _outputType;
   bool _ready = false;
+  String? _lastError;
 
   YoloDetector(this._analyzer);
 
-  bool get isReady => _ready;
+  bool get isReady => _ready && _runtime != null;
+  String? get lastError => _lastError;
 
   Future<void> init() async {
+    dispose();
+    _lastError = null;
     try {
       _labels = (await rootBundle.loadString(AppConstants.yoloLabelsPath))
           .split('\n')
@@ -24,24 +37,489 @@ class YoloDetector {
         _ready = false;
         return;
       }
-      await rootBundle.load(await AppConstants.getSelectedYoloModelPath());
+      _runtime = await YoloRuntime.load(
+        await AppConstants.getSelectedYoloModelPath(),
+      );
+      _inputShape = _runtime!.inputShape;
+      _outputShape = _runtime!.outputShape;
+      _inputType = _runtime!.inputType;
+      _outputType = _runtime!.outputType;
       _ready = true;
     } catch (e) {
+      _lastError = e.toString();
       debugPrint('YOLO fallback mode: $e');
       _ready = false;
     }
   }
 
-  Future<DetectionResult?> detectFallback() async {
-    if (_ready) {
-      // Full tensor pre/post-processing belongs here once yolov8n.tflite is present.
-      // The app keeps a deterministic fallback so demo flows remain testable.
+  Future<DetectionResult?> detect(
+    CameraImage image, {
+    double confidenceThreshold = 0.45,
+  }) async {
+    if (!isReady) return detectFallback();
+    try {
+      final input = _buildCameraInput(image);
+      final output = Uint8List(_runtime!.outputByteLength);
+      _runtime!.run(input, output);
+      final detections = _decodeDetections(output);
+      final filtered =
+          _analyzer.filterByConfidence(detections, confidenceThreshold);
+      return _analyzer.prioritize(filtered);
+    } catch (e) {
+      _lastError = e.toString();
+      debugPrint('YOLO inference fallback: $e');
+      return detectFallback();
     }
+  }
+
+  Future<DetectionResult?> detectSynthetic({
+    double confidenceThreshold = 0.25,
+  }) async {
+    if (!isReady) return detectFallback();
+    try {
+      final input = _buildSyntheticInput();
+      final output = Uint8List(_runtime!.outputByteLength);
+      _runtime!.run(input, output);
+      final detections = _decodeDetections(output);
+      final filtered =
+          _analyzer.filterByConfidence(detections, confidenceThreshold);
+      return _analyzer.prioritize(filtered) ?? detectFallback();
+    } catch (e) {
+      _lastError = e.toString();
+      debugPrint('YOLO synthetic fallback: $e');
+      return detectFallback();
+    }
+  }
+
+  Future<DetectionResult?> detectFallback() async {
     final label = _labels.isNotEmpty ? _labels.first : 'person';
     return _analyzer.analyzeFallback(label: label);
   }
 
   void dispose() {
+    _runtime?.close();
+    _runtime = null;
+    _inputShape = const [];
+    _outputShape = const [];
+    _inputType = null;
+    _outputType = null;
     _ready = false;
   }
+
+  Uint8List _buildCameraInput(CameraImage image) {
+    final inputInfo = _inputInfo();
+    final bytes = Uint8List(inputInfo.byteLength);
+    final data = ByteData.view(bytes.buffer);
+
+    for (var y = 0; y < inputInfo.height; y++) {
+      final srcY = (y * image.height / inputInfo.height).floor();
+      for (var x = 0; x < inputInfo.width; x++) {
+        final srcX = (x * image.width / inputInfo.width).floor();
+        final rgb = _rgbAt(image, srcX, srcY);
+        _writeInputValue(data, bytes, inputInfo, x, y, 0, (rgb >> 16) & 0xFF);
+        _writeInputValue(data, bytes, inputInfo, x, y, 1, (rgb >> 8) & 0xFF);
+        _writeInputValue(data, bytes, inputInfo, x, y, 2, rgb & 0xFF);
+      }
+    }
+
+    return bytes;
+  }
+
+  Uint8List _buildSyntheticInput() {
+    final inputInfo = _inputInfo();
+    final bytes = Uint8List(inputInfo.byteLength);
+    final data = ByteData.view(bytes.buffer);
+
+    for (var y = 0; y < inputInfo.height; y++) {
+      for (var x = 0; x < inputInfo.width; x++) {
+        final shade = ((x / inputInfo.width) * 64 + 96).round();
+        _writeInputValue(data, bytes, inputInfo, x, y, 0, shade);
+        _writeInputValue(data, bytes, inputInfo, x, y, 1, shade);
+        _writeInputValue(data, bytes, inputInfo, x, y, 2, shade);
+      }
+    }
+
+    return bytes;
+  }
+
+  _InputInfo _inputInfo() {
+    if (_inputShape.length != 4) {
+      throw StateError('Unsupported YOLO input shape: $_inputShape');
+    }
+    final type = _inputType;
+    if (type == null) throw StateError('YOLO input type is unavailable');
+
+    final channelsLast = _inputShape.last == 3;
+    final channelsFirst = _inputShape[1] == 3;
+    if (!channelsLast && !channelsFirst) {
+      throw StateError('Unsupported YOLO channel layout: $_inputShape');
+    }
+    if (type != YoloTensorType.float32 &&
+        type != YoloTensorType.uint8 &&
+        type != YoloTensorType.int8) {
+      throw StateError('Unsupported YOLO input type: $type');
+    }
+
+    final height = channelsLast ? _inputShape[1] : _inputShape[2];
+    final width = channelsLast ? _inputShape[2] : _inputShape[3];
+    final bytesPerValue = type == YoloTensorType.float32 ? 4 : 1;
+    return _InputInfo(
+      width: width,
+      height: height,
+      channelsLast: channelsLast,
+      type: type,
+      bytesPerValue: bytesPerValue,
+    );
+  }
+
+  void _writeInputValue(
+    ByteData data,
+    Uint8List bytes,
+    _InputInfo info,
+    int x,
+    int y,
+    int channel,
+    int value,
+  ) {
+    final elementIndex = info.channelsLast
+        ? ((y * info.width + x) * 3 + channel)
+        : (channel * info.width * info.height + y * info.width + x);
+    final byteOffset = elementIndex * info.bytesPerValue;
+
+    if (info.type == YoloTensorType.float32) {
+      data.setFloat32(byteOffset, value / 255.0, Endian.little);
+    } else if (info.type == YoloTensorType.uint8) {
+      bytes[byteOffset] = value;
+    } else if (info.type == YoloTensorType.int8) {
+      data.setInt8(byteOffset, value - 128);
+    }
+  }
+
+  int _rgbAt(CameraImage image, int x, int y) {
+    final group = image.format.group;
+    if (group == ImageFormatGroup.bgra8888) {
+      final plane = image.planes.first;
+      final index = y * plane.bytesPerRow + x * 4;
+      if (index + 2 >= plane.bytes.length) return 0;
+      final b = plane.bytes[index];
+      final g = plane.bytes[index + 1];
+      final r = plane.bytes[index + 2];
+      return (r << 16) | (g << 8) | b;
+    }
+
+    if (image.planes.length < 3) {
+      final plane = image.planes.first;
+      final index = math.min(y * plane.bytesPerRow + x, plane.bytes.length - 1);
+      final luma = plane.bytes[index];
+      return (luma << 16) | (luma << 8) | luma;
+    }
+
+    final yPlane = image.planes[0];
+    final uPlane = image.planes[1];
+    final vPlane = image.planes[2];
+
+    final yIndex =
+        math.min(y * yPlane.bytesPerRow + x, yPlane.bytes.length - 1);
+    final uvX = x ~/ 2;
+    final uvY = y ~/ 2;
+    final uvPixelStride = uPlane.bytesPerPixel ?? 1;
+    final uvIndex = math.min(
+      uvY * uPlane.bytesPerRow + uvX * uvPixelStride,
+      math.min(uPlane.bytes.length, vPlane.bytes.length) - 1,
+    );
+
+    final yy = yPlane.bytes[yIndex].toDouble();
+    final uu = uPlane.bytes[uvIndex].toDouble() - 128.0;
+    final vv = vPlane.bytes[uvIndex].toDouble() - 128.0;
+
+    final r = _clampByte(yy + 1.402 * vv);
+    final g = _clampByte(yy - 0.344136 * uu - 0.714136 * vv);
+    final b = _clampByte(yy + 1.772 * uu);
+    return (r << 16) | (g << 8) | b;
+  }
+
+  int _clampByte(double value) => value.round().clamp(0, 255);
+
+  List<DetectionResult> _decodeDetections(Uint8List outputBuffer) {
+    final values = _readOutput(outputBuffer);
+    if (values.isEmpty || _outputShape.isEmpty) return const [];
+
+    final dims = _outputShape.where((d) => d > 0).toList();
+    final effectiveShape =
+        dims.isNotEmpty && dims.first == 1 ? dims.sublist(1) : dims;
+
+    if (effectiveShape.length == 2) {
+      final a = effectiveShape[0];
+      final b = effectiveShape[1];
+      if (a >= 5 && b > a) {
+        return _decodeFeatureFirst(values, a, b);
+      }
+      if (b >= 5) {
+        return _decodeRowFirst(values, a, b);
+      }
+    }
+
+    if (effectiveShape.length == 3 && effectiveShape[0] == 1) {
+      final a = effectiveShape[1];
+      final b = effectiveShape[2];
+      if (a >= 5 && b > a) {
+        return _decodeFeatureFirst(values, a, b);
+      }
+      if (b >= 5) {
+        return _decodeRowFirst(values, a, b);
+      }
+    }
+
+    return const [];
+  }
+
+  List<double> _readOutput(Uint8List bytes) {
+    final type = _outputType;
+    if (type == null) return const [];
+    final data = ByteData.view(bytes.buffer);
+
+    if (type == YoloTensorType.float32) {
+      return [
+        for (var i = 0; i < bytes.length; i += 4)
+          data.getFloat32(i, Endian.little),
+      ];
+    }
+    if (type == YoloTensorType.uint8) {
+      return [for (final byte in bytes) byte / 255.0];
+    }
+    if (type == YoloTensorType.int8) {
+      return [for (var i = 0; i < bytes.length; i++) data.getInt8(i) / 127.0];
+    }
+    if (type == YoloTensorType.float16) {
+      // Most exported YOLOv8 TFLite files use float32 output. Float16 is rare;
+      // fallback keeps the app usable if an unsupported variant is selected.
+      throw StateError('Float16 YOLO output is not supported yet');
+    }
+    throw StateError('Unsupported YOLO output type: $type');
+  }
+
+  List<DetectionResult> _decodeFeatureFirst(
+    List<double> values,
+    int featureCount,
+    int boxCount,
+  ) {
+    final detections = <DetectionResult>[];
+    final classStart = _classStart(featureCount);
+    for (var i = 0; i < boxCount; i++) {
+      final cx = values[i];
+      final cy = values[boxCount + i];
+      final w = values[2 * boxCount + i];
+      final h = values[3 * boxCount + i];
+      final scored = _bestClassFeatureFirst(values, featureCount, boxCount,
+          classStart: classStart, boxIndex: i);
+      if (scored == null) continue;
+      detections.add(_fromCenterBox(
+        label: scored.label,
+        confidence: scored.score,
+        cx: cx,
+        cy: cy,
+        width: w,
+        height: h,
+      ));
+    }
+    return detections;
+  }
+
+  List<DetectionResult> _decodeRowFirst(
+    List<double> values,
+    int rowCount,
+    int featureCount,
+  ) {
+    final detections = <DetectionResult>[];
+    if (featureCount == 6 || featureCount == 7) {
+      for (var row = 0; row < rowCount; row++) {
+        final offset = row * featureCount;
+        final score = values[offset + 4].clamp(0.0, 1.0);
+        final classIndex = values[offset + 5].round();
+        detections.add(_fromCornerBox(
+          label: _labelFor(classIndex),
+          confidence: score,
+          left: values[offset],
+          top: values[offset + 1],
+          right: values[offset + 2],
+          bottom: values[offset + 3],
+        ));
+      }
+      return detections;
+    }
+
+    final classStart = _classStart(featureCount);
+    for (var row = 0; row < rowCount; row++) {
+      final offset = row * featureCount;
+      final scored =
+          _bestClassRow(values, offset, featureCount, classStart: classStart);
+      if (scored == null) continue;
+      detections.add(_fromCenterBox(
+        label: scored.label,
+        confidence: scored.score,
+        cx: values[offset],
+        cy: values[offset + 1],
+        width: values[offset + 2],
+        height: values[offset + 3],
+      ));
+    }
+    return detections;
+  }
+
+  int _classStart(int featureCount) {
+    if (featureCount == 85) return 5; // YOLOv5 COCO: xywh + obj + 80 cls
+    if (_labels.isNotEmpty && featureCount == _labels.length + 5) return 5;
+    return 4;
+  }
+
+  _ScoredLabel? _bestClassFeatureFirst(
+    List<double> values,
+    int featureCount,
+    int boxCount, {
+    required int classStart,
+    required int boxIndex,
+  }) {
+    final objectness = classStart == 5 ? values[4 * boxCount + boxIndex] : 1.0;
+    var bestScore = 0.0;
+    var bestClass = 0;
+    for (var feature = classStart; feature < featureCount; feature++) {
+      final score = objectness * values[feature * boxCount + boxIndex];
+      if (score > bestScore) {
+        bestScore = score;
+        bestClass = feature - classStart;
+      }
+    }
+    if (bestScore <= 0) return null;
+    return _ScoredLabel(
+      _labelFor(bestClass, classCount: featureCount - classStart),
+      bestScore.clamp(0.0, 1.0),
+    );
+  }
+
+  _ScoredLabel? _bestClassRow(
+    List<double> values,
+    int offset,
+    int featureCount, {
+    required int classStart,
+  }) {
+    final objectness = classStart == 5 ? values[offset + 4] : 1.0;
+    var bestScore = 0.0;
+    var bestClass = 0;
+    for (var feature = classStart; feature < featureCount; feature++) {
+      final score = objectness * values[offset + feature];
+      if (score > bestScore) {
+        bestScore = score;
+        bestClass = feature - classStart;
+      }
+    }
+    if (bestScore <= 0) return null;
+    return _ScoredLabel(
+      _labelFor(bestClass, classCount: featureCount - classStart),
+      bestScore.clamp(0.0, 1.0),
+    );
+  }
+
+  String _labelFor(int classIndex, {int? classCount}) {
+    if (classCount != null && classCount > _labels.length) {
+      return _cocoObstacleLabels[classIndex] ?? 'object';
+    }
+    if (classIndex >= 0 && classIndex < _labels.length) {
+      return _labels[classIndex];
+    }
+    return 'object';
+  }
+
+  DetectionResult _fromCenterBox({
+    required String label,
+    required double confidence,
+    required double cx,
+    required double cy,
+    required double width,
+    required double height,
+  }) {
+    return _fromCornerBox(
+      label: label,
+      confidence: confidence,
+      left: cx - width / 2,
+      top: cy - height / 2,
+      right: cx + width / 2,
+      bottom: cy + height / 2,
+    );
+  }
+
+  DetectionResult _fromCornerBox({
+    required String label,
+    required double confidence,
+    required double left,
+    required double top,
+    required double right,
+    required double bottom,
+  }) {
+    final inputInfo = _inputInfo();
+    final normalized =
+        [left, top, right, bottom].every((value) => value >= 0 && value <= 1.5);
+
+    final scaledLeft = normalized
+        ? left * ObstacleAnalyzer.frameWidth
+        : left / inputInfo.width * ObstacleAnalyzer.frameWidth;
+    final scaledRight = normalized
+        ? right * ObstacleAnalyzer.frameWidth
+        : right / inputInfo.width * ObstacleAnalyzer.frameWidth;
+    final scaledTop = normalized
+        ? top * ObstacleAnalyzer.frameHeight
+        : top / inputInfo.height * ObstacleAnalyzer.frameHeight;
+    final scaledBottom = normalized
+        ? bottom * ObstacleAnalyzer.frameHeight
+        : bottom / inputInfo.height * ObstacleAnalyzer.frameHeight;
+
+    final box = BoundingBox(
+      left: scaledLeft.clamp(0.0, ObstacleAnalyzer.frameWidth),
+      top: scaledTop.clamp(0.0, ObstacleAnalyzer.frameHeight),
+      right: scaledRight.clamp(0.0, ObstacleAnalyzer.frameWidth),
+      bottom: scaledBottom.clamp(0.0, ObstacleAnalyzer.frameHeight),
+    );
+
+    return DetectionResult(
+      label: label,
+      confidence: confidence,
+      direction: _analyzer.analyzeDirection(box),
+      estimatedDistance: _analyzer.estimateDistance(box),
+    );
+  }
+}
+
+const Map<int, String> _cocoObstacleLabels = {
+  0: 'person',
+  1: 'bicycle',
+  2: 'car',
+  3: 'motorcycle',
+  5: 'bus',
+  7: 'truck',
+  13: 'bench',
+  56: 'chair',
+};
+
+class _InputInfo {
+  final int width;
+  final int height;
+  final bool channelsLast;
+  final YoloTensorType type;
+  final int bytesPerValue;
+
+  const _InputInfo({
+    required this.width,
+    required this.height,
+    required this.channelsLast,
+    required this.type,
+    required this.bytesPerValue,
+  });
+
+  int get byteLength => width * height * 3 * bytesPerValue;
+}
+
+class _ScoredLabel {
+  final String label;
+  final double score;
+
+  const _ScoredLabel(this.label, this.score);
 }

walkguide-mobile/walkguide_app/lib/core/ai/yolo_runtime.dart

@@ -0,0 +1 @@
+export 'yolo_runtime_stub.dart' if (dart.library.io) 'yolo_runtime_native.dart';

walkguide-mobile/walkguide_app/lib/core/ai/yolo_runtime_native.dart

@@ -0,0 +1,57 @@
+import 'dart:typed_data';
+
+import 'package:tflite_flutter/tflite_flutter.dart';
+
+enum YoloTensorType { float32, uint8, int8, float16, unsupported }
+
+class YoloRuntime {
+  final Interpreter _interpreter;
+  final List<int> inputShape;
+  final List<int> outputShape;
+  final YoloTensorType inputType;
+  final YoloTensorType outputType;
+  final int outputByteLength;
+
+  YoloRuntime._({
+    required Interpreter interpreter,
+    required this.inputShape,
+    required this.outputShape,
+    required this.inputType,
+    required this.outputType,
+    required this.outputByteLength,
+  }) : _interpreter = interpreter;
+
+  static Future<YoloRuntime> load(String assetPath) async {
+    final options = InterpreterOptions()..threads = 2;
+    final interpreter =
+        await Interpreter.fromAsset(assetPath, options: options);
+    final input = interpreter.getInputTensor(0);
+    final output = interpreter.getOutputTensor(0);
+    return YoloRuntime._(
+      interpreter: interpreter,
+      inputShape: input.shape,
+      outputShape: output.shape,
+      inputType: _mapTensorType(input.type),
+      outputType: _mapTensorType(output.type),
+      outputByteLength: output.numBytes(),
+    );
+  }
+
+  void run(Uint8List input, Uint8List output) {
+    _interpreter.run(input, output);
+  }
+
+  void close() {
+    _interpreter.close();
+  }
+}
+
+YoloTensorType _mapTensorType(TensorType type) {
+  return switch (type) {
+    TensorType.float32 => YoloTensorType.float32,
+    TensorType.uint8 => YoloTensorType.uint8,
+    TensorType.int8 => YoloTensorType.int8,
+    TensorType.float16 => YoloTensorType.float16,
+    _ => YoloTensorType.unsupported,
+  };
+}

walkguide-mobile/walkguide_app/lib/core/ai/yolo_runtime_stub.dart

@@ -0,0 +1,23 @@
+import 'dart:typed_data';
+
+enum YoloTensorType { float32, uint8, int8, float16, unsupported }
+
+class YoloRuntime {
+  const YoloRuntime();
+
+  static Future<YoloRuntime> load(String assetPath) async {
+    throw UnsupportedError('YOLO TFLite runtime is only available on mobile.');
+  }
+
+  List<int> get inputShape => const [];
+  List<int> get outputShape => const [];
+  YoloTensorType get inputType => YoloTensorType.unsupported;
+  YoloTensorType get outputType => YoloTensorType.unsupported;
+  int get outputByteLength => 0;
+
+  void run(Uint8List input, Uint8List output) {
+    throw UnsupportedError('YOLO TFLite runtime is only available on mobile.');
+  }
+
+  void close() {}
+}