┌─────────────────────────────────────────────────────────────┐
│                      应用服务层                              │
│  会议仪表盘 · 实时预警 · 会后报告 · 开放API(SDK)             │
├─────────────────────────────────────────────────────────────┤
│                      分析引擎层                              │
│  ┌─────────────┐ ┌─────────────┐ ┌─────────────┐             │
│  │ 情绪计算模块 │ │ 参会度量模块 │ │ 语义交互模块 │             │
│  └─────────────┘ └─────────────┘ └─────────────┘             │
├─────────────────────────────────────────────────────────────┤
│                      特征融合层                              │
│  视觉特征 · 音频特征 · 文本特征 · 元数据(时长/角色/屏共)       │
├─────────────────────────────────────────────────────────────┤
│                      数据采集与预处理层                        │
│  客户端采集 SDK · 流媒体转发 · 脱敏管道 · 合规审计日志         │
└─────────────────────────────────────────────────────────────┘

# 伪代码：多模态融合推理流程
def multimodal_fusion(visual_feat, audio_feat, text_feat, meta):
    # 1. 单模态置信度校准
    v_conf = calibrate(visual_feat.confidence, meta.lighting, meta.occlusion)
    a_conf = calibrate(audio_feat.confidence, meta.snr, meta.vad)
    t_conf = calibrate(text_feat.confidence, meta.asr_wer)
    
    # 2. 动态权重分配(基于环境质量自适应)
    weights = softmax([v_conf, a_conf, t_conf] * meta.modality_importance)
    
    # 3. 特征级注意力融合(仅当三模态均可用)
    if all_modalities_available:
        fused = cross_attention(visual_feat, audio_feat, text_feat)
        emotion_logits = classifier(fused)
    else:
        # 降级：加权投票
        emotion_logits = sum(w * m.logits for w, m in zip(weights, modalities))
    
    # 4. 时序平滑(滑动窗口 + Kalman/指数移动平均)
    smoothed = temporal_smooth(emotion_logits, window=5s, alpha=0.3)
    return EmotionResult(label=argmax(smoothed), distribution=softmax(smoothed))

-- Flink SQL 示例：滑动窗口下参会度实时聚合
CREATE VIEW engagement_1min AS
SELECT 
    meeting_id,
    user_id,
    TUMBLE_START(proc_time, INTERVAL '1' MINUTE) AS window_start,
    -- 行为特征
    SUM(CASE WHEN event='video_on' THEN 1 ELSE 0 END) / COUNT(*) AS video_on_ratio,
    COUNT(CASE WHEN event='screen_share' THEN 1 END) AS share_count,
    -- 认知特征(来自客户端上报)
    AVG(gaze_on_screen_ratio) AS avg_gaze,
    -- 情绪特征(来自情绪模块推流)
    STDDEV(arousal) AS arousal_stability,
    AVG(valence) AS avg_valence,
    -- 交互特征
    COUNT(CASE WHEN event='speak' THEN 1 END) AS speak_turns,
    SUM(CASE WHEN event='reaction' THEN 1 ELSE 0 END) AS reaction_count
FROM unified_event_stream
GROUP BY meeting_id, user_id, TUMBLE(proc_time, INTERVAL '1' MINUTE);

graph LR
    A[原始音视频流] --> B{客户端/边缘网关}
    B -->|仅特征向量| C[分析引擎]
    B -.->|可选: 脱敏缩略图| D[合规审计存储]
    C --> E[聚合指标/报告]
    D --> F[定期自动清理/加密归档]
    
    style A fill:#ffebee
    style B fill:#e8f5e9
    style C fill:#e3f2fd

实验分层：
Layer 1: 模型版本(v1.2 vs v1.3)
Layer 2: 融合策略(决策级 vs 特征级)
Layer 3: 预警阈值(敏感/平衡/宽容)

核心北极星指标：
- 会议有效时长占比 ↑
- 参会者主观满意度(会后1题微调研) ↑
- 主席干预采纳率 ↑
- 误报投诉率 ↓

// 核心调度策略伪代码
class InferenceScheduler {
    enum Backend { TENSORRT, COREML, NCNN, MNN, ONNX_RUNTIME, TFLITE, RKNN };
    
    struct DeviceProfile {
        bool has_npu; bool has_gpu; int gpu_mem_mb;
        int cpu_cores; bool support_fp16; bool support_int8;
        ThermalState thermal; // NOMINAL / FAIR / SERIOUS / CRITICAL
    };

    Backend select_backend(const ModelSpec& model, const DeviceProfile& dev) {
        // 1. 热力学约束优先：温度高强制降级CPU INT8
        if (dev.thermal >= ThermalState::SERIOUS) return Backend::NCNN; 
        
        // 2. 模型算子支持度匹配（算子覆盖率>95%才走硬件加速）
        auto support = query_operator_support(model, dev);
        if (support.coverage < 0.95) return Backend::ONNX_RUNTIME; // CPU兜底
        
        // 3. 延迟/功耗帕累托最优选择（离线Profiling建表）
        return lookup_pareto_optimal(model.id, dev);
    }
};

# 动态Letterbox + 元数据归一化
def preprocess_adaptive(frame, target_size=224):
    h, w = frame.shape[:2]
    scale = min(target_size / h, target_size / w)
    new_w, new_h = int(w * scale), int(h * scale)
    
    # 1. 高质量缩放 (LANCZOS4 / GPU Bicubic)
    resized = cv2.resize(frame, (new_w, new_h), interpolation=cv2.INTER_LANCZOS4)
    
    # 2. 灰条填充 (114, 114, 114) - 避免黑边引入伪边缘
    canvas = np.full((target_size, target_size, 3), 114, dtype=np.uint8)
    dx, dy = (target_size - new_w) // 2, (target_size - new_h) // 2
    canvas[dy:dy+new_h, dx:dx+new_w] = resized
    
    # 3. 归一化参数随分辨率自适应 (可选: 训练时引入Scale Augmentation)
    tensor = (canvas.astype(np.float32) / 255.0 - mean) / std
    
    # 4. 关键：输出反变换矩阵，供后处理(关键点回映射/目标检测框还原)使用
    inv_matrix = np.array([[1/scale, 0, -dx/scale], [0, 1/scale, -dy/scale], [0,0,1]])
    return tensor, inv_matrix

# 核心结构：Pairwise Cross-Modal Attention
class CrossModalAttention(nn.Module):
    def __init__(self, d_model, n_heads, dropout=0.1):
        super().__init__()
        self.q_proj = nn.Linear(d_model, d_model)
        self.kv_proj = nn.Linear(d_model, d_model * 2)
        self.attn = nn.MultiheadAttention(d_model, n_heads, dropout, batch_first=True)
        self.norm = nn.LayerNorm(d_model)
        self.ffn = nn.Sequential(nn.Linear(d_model, d_model*4), nn.GELU(), nn.Linear(d_model*4, d_model))

    def forward(self, query_modality, key_value_modality, key_padding_mask=None):
        # query: [B, T_q, D], kv: [B, T_kv, D]
        q = self.q_proj(query_modality)
        k, v = self.kv_proj(key_value_modality).chunk(2, dim=-1)
        
        # 关键：允许 T_q != T_kv (通过插值或原生支持变长)
        attn_out, _ = self.attn(q, k, v, key_padding_mask=key_padding_mask)
        x = self.norm(query_modality + attn_out)
        x = self.norm(x + self.ffn(x))
        return x

# 融合流程：V->A, A->V, T->V, T->A ... 多轮交互后 Concat -> Classifier

┌────────────────────────────────────────────────────────────────────┐
│                      会议智能体 Orchestrator                        │
│  (Planning + Memory + Tool Use + Reflection)                       │
├──────────────────┬─────────────────────────────────────────────────┤
│   感知引擎        │              认知引擎                            │
│  (上篇核心引擎)   │            (LLM + RAG + Knowledge Graph)       │
│  - 实时情绪/参会度 │  - 会议纪要生成 (结构化: 决策/行动项/风险/共识)  │
│  - 关键词/语义流   │  - 实时干预建议 (话术/议程调整/破冰)           │
│  - 发言人拓扑图    │  - 个人/团队复盘报告 (优势/盲区/成长建议)       │
│  - 异常事件流      │  - 跨会议知识沉淀 (决策溯源/经验复用)           │
└──────────────────┴─────────────────────────────────────────────────┘

// tools_schema.json (OpenAI Function Calling / Anthropic Tool Use 兼容)
[
  {
    "name": "query_engagement_timeline",
    "description": "查询指定时间窗内全员/个人参会度曲线、情绪分布、发言占比",
    "parameters": {
      "type": "object",
      "properties": {
        "meeting_id": {"type": "string"},
        "user_ids": {"type": "array", "items": {"type": "string"}},
        "start_ts": {"type": "number", "description": "Unix毫秒时间戳"},
        "end_ts": {"type": "number"},
        "metrics": {"type": "array", "items": {"type": "string", "enum": ["engagement", "valence", "arousal", "speak_ratio", "gaze_ratio"]}}
      },
      "required": ["meeting_id", "start_ts", "end_ts"]
    }
  },
  {
    "name": "get_semantic_segments",
    "description": "获取语义分段: 议题、发言人、摘要、情绪标签、关键词",
    "parameters": { ... }
  },
  {
    "name": "send_realtime_nudge",
    "description": "【慎用】向主持人/个人发送实时干预提示 (需权限校验)",
    "parameters": {
      "type": "object",
      "properties": {
        "target": {"type": "string", "enum": ["host", "user_id", "all"]},
        "nudge_type": {"type": "string", "enum": ["pace_slow", "low_engagement", "conflict_detected", "topic_drift", "time_warning"]},
        "suggested_script": {"type": "string", "description": "建议话术，如：'我们在预算议题上停留了20分钟，建议先确认核心结论，细节放到线下小组讨论'"}
      },
      "required": ["target", "nudge_type", "suggested_script"]
    }
  },
  {
    "name": "search_historical_meetings",
    "description": "跨会议知识检索: 相似议题、历史决策、专家发言",
    "parameters": { ... }
  }
]

sequenceDiagram
    participant Perception as 感知引擎
    participant Agent as 会议智能体
    participant LLM as 大模型
    participant Host as 主持人客户端
    
    loop 每30秒/触发式
        Perception->>Agent: 事件流: {topic_drift_score: 0.85, current_topic: "技术选型", agenda_topic: "预算审批", duration: 1200s}
        Agent->>LLM: Prompt + Tools: "检测到议题偏离，当前讨论技术选型，议程要求预算审批。请分析偏离必要性并给出干预建议。"
        LLM->>Agent: 调用 search_historical_meetings("技术选型 预算 关联")
        Agent-->>LLM: 返回历史案例: "技术选型直接影响预算上限，属必要偏离"
        LLM->>Agent: 决策: "允许延长15分钟，但需在T+15min强制收敛至预算结论"
        Agent->>Host: send_realtime_nudge(host, time_warning, "技术选型讨论有价值，建议再讨论15分钟聚焦成本影响，随后必须切回预算审批表决")
    end

graph LR
    A[线上推理日志<br/>(特征/预测/置信度)] --> B{数据分流}
    B -->|高置信度+一致性校验通过| C[伪标签自动入库<br/>Auto-Labeling]
    B -->|低置信度/分布漂移/OOD| D[主动学习采样池<br/>Active Learning Pool]
    B -->|用户投诉/人工纠偏| E[高价值硬例集<br/>Golden Set]
    
    C --> F[周度增量训练<br/>Incremental Training]
    D --> G[人工标注优先队列<br/>(不确定性/多样性采样)]
    E --> F
    G --> H[月度全量重训<br/>Full Retrain]
    
    F --> I[模型评估看板<br/>回归测试/公平性/鲁棒性]
    H --> I
    I -->|通过| J[灰度发布<br/>Canary 5% -> 100%]
    I -->|不通过| K[阻断/回滚/报警]

# config/engine_defaults.yaml
inference:
  visual:
    model: "mobilenetv3_small_pfe_au66_int8.onnx"
    input_size: [224, 224]
    fps_target: 10
    confidence_threshold: 0.45
    temporal_smooth_alpha: 0.3
    occlusion_fallback: true
  audio:
    model: "ecapa_tdnn_192dim_int8.onnx + wav2vec2_base_chinese_int8.onnx"
    sample_rate: 16000
    window_ms: 2000
    hop_ms: 500
    vad_threshold: 0.6
    snr_min_db: 5.0
  text:
    model: "distilbert_base_chinese_int8.onnx"
    max_seq_len: 128
    asr_latency_budget_ms: 1500
fusion:
  strategy: "cross_attention_mult"
  modality_dropout_p: 0.1 # 训练时随机丢弃模态，增强鲁棒性
  weights_adaptive: true
engagement:
  factors:
    behavior: 0.25
    cognitive: 0.30
    emotional: 0.20
    interaction: 0.25
  window_sec: 60
  anomaly:
    idle_threshold_sec: 300
    low_quality_score: 0.25
    distress_arousal: 0.8
    distress_valence: -0.5
    distress_duration_sec: 180
privacy:
  local_inference_default: true
  feature_vector_only_upload: true
  dp_epsilon_team_report: 1.0
  data_retention_days: 30
  audit_log_encryption: "AES-256-GCM"
deployment:
  client:
    max_memory_mb: 300
    max_cpu_percent: 15
    thermal_throttle_enabled: true
  edge:
    batch_size: 8
    gpu_memory_fraction: 0.7
  cloud:
    autoscaling:
      min_replicas: 3
      max_replicas: 200
      target_gpu_util: 65