移动端Android SDK

    /**
     * 当前配置是否是全双工模式
     */
    fun isDuplexMode(): Boolean

    /**
     * Push2Talk 用户取消说话
     */
    fun cancelSpeech()

    /**
     * 发送语音回复结束
     * */
    fun sendResponseEnded()

    /**
     * 发送语音回复开始
     * */
    fun sendResponseStarted()

    /**
     * 推送视频帧
     *
     * @param videoFrame 视频帧
     * @param callback: 送帧结果回调
     */
    fun sendVideoFrame(videoFrame: TYVideoFrame,callback: IVideoChatPushFrameCallback)

{
    "header": {
        "action":"run-task",
        "task_id": "f894c16f-f20e-4c1d-837e-89e0fbc63a43",
        "streaming":"duplex"
    },
    "payload": {
        "task_group":"aigc",
        "task":"multimodal-generation", // 任务类型：多模态生成
        "function":"generation",
        "model":"multimodal-dialog", // 模型名称：多模态对话（注意与task不同）
        "input":{
          "directive": "Start",
          "workspace_id": "llm-***********",
          "app_id": "****************"
        },
        "parameters":{
          "upstream":{
            "type": "AudioOnly",
            "mode": "duplex"
          },
          "downstream":{
            "voice": "longxiaochun_v2",
            "sample_rate": 24000
          },
          "client_info":{
            "user_id": "bin********207",
            "device":{
              "uuid": "432k*********k449"
            },
            "network":{
              "ip": "203.0.113.10"
            },
            "location":{
              "city_name": "北京市"
            }
          },
          "biz_params":{
            "user_defined_params": {
                "extra_config": {
                    "enable_web_search": false,
                    "agent_timeout": 10
                },
                "agent_id_xxxxx": {
                    "name": "value"
                },
                "mcp_server_id_xxxxx":{
                    "name": "value"
                }
            },
            "user_prompt_params": {
                "name": "value"
            },
            "user_query_params": {
                "name": "value"
            }
          }
        }
    }
}

    /**
     * 对话启动结果，鉴权、客户端加入通道成功后，会回调该消息
     */
    fun onStartResult(isSuccess: Boolean, errorInfo: TYError?)

    /**
     * 主动打断的回调，包括手动打断和语音打断
     */
    fun onInterruptResult(isSuccess: Boolean, errorInfo: TYError?)

    /**
     * 对话准备完成，多端（客户端/VoiceChat/视频理解）均加入通道后，会回调该消息，此时才能调用业务接口
     */
    fun onReadyToSpeech()

    /**
     * 状态切换回调
     * 包含DIALOG_IDLE，DIALOG_LISTENING，DIALOG_RESPONDING，DIALOG_THINKING
     */
    fun onConvStateChangedCallback(state: DialogState)

    /**
     * 音量强度回调
     * @param audioType 参考 {@link Constant.TYVolumeSourceType}
     * @param audioLevel 0-100
     */
    fun onConvSoundLevelCallback(audioLevel: Float, audioType: Constant.TYVolumeSourceType)

    /**
     * 超出用户有效输入超时时间，收到此事件，用户需要重新启动或者结束对话
     * @param timeout 超时时间
     */
    fun onSpeechTimeout(timeout: Long)

    /**
     * 回路音频，用于客户端AEC回声消除
     * @param bytes 音频数据
     */
    fun onPlaybackAudioData(bytes: ByteArray)

    /**
     * 对话过程中的异常信息
     * @param errorInfo 异常信息
     */
    fun onErrorReceived(errorInfo: TYError)

    /**
     * 对话过程中的事件信息
     * @param errorInfo 事件信息
     */
    fun onConvEventCallback(var1: ConvEvent?)

    /**
     * 运行过程中回调关键日志信息给上层
     * @param level 日志级别，参考Android.Log
     * @param type 日志类型
     * @param debugInfo 日志信息
     */
    fun onDebugInfoTrack(level: Int, type: Constant.TYDebugInfoType, debugInfo: String)

private void executeCmd(String cmd,String dialogId,String taskId){
  String cmdName = null;
  String cmdParamValue = null;
  try {
    cmdName = new JSONArray(cmd).getJSONObject(0).getString("name");
  }catch (Exception ex) {
    ex.printStackTrace();
  }
  switch (cmdName){
     case "visual_qa":
       JSONObject extraObject = new JSONObject();
       extraObject.put("images", imagesArray);
       multimodalConversation.requestToRespond(type, text, extraObject.toString());
       break;
  ......
}

//上传拍照
public static JSONArray getMockOSSImage() {
    JSONObject imageObject = new JSONObject();
    JSONArray images = new JSONArray();
    try{
      if (vqaUseUrl){
        imageObject.put("type", "url");
        imageObject.put("value", "https://help-static-aliyun-doc.aliyuncs.com/assets/img/zh-CN/7043267371/p909896.png");
      }else {
        imageObject.put("type", "base64");
        imageObject.put("value", getLocalImageBase64());
      }
      images.put(imageObject);

    }catch (Exception e){
      e.printStackTrace();
    }
    return images;
  }

//1. 设置交互类型为AudioAndVideo
MultiModalRequestParam.UpStream.builder().mode("duplex").type("AudioAndVideo").build();

//2. 在建联后发送voicechat_video_channel 请求
private void startVideoMode() {
    if (isVideoMode) return;

    try {
        MultiModalRequestParam updateParams = MultiModalRequestParam.builder().build();
        JSONObject videoObj = new JSONObject();
        videoObj.put("action", "connect");
        videoObj.put("type", "voicechat_video_channel");

        List<JSONObject> videos = new ArrayList<>();
        videos.add(videoObj);

        updateParams.setBizParams(MultiModalRequestParam.BizParams.builder()
                .videos(videos).build());

        multiModalDialog.requestToRespond("prompt", "", updateParams.getParametersAsJson());
        multiModalDialog.setVideoContainer(videoContainer, uiHandler);

        isVideoMode = true;
        runOnUiThread(() -> videoContainer.setVisibility(View.VISIBLE));

    } catch (JSONException e) {
        Log.e(TAG, "启动视频模式失败", e);
    }
}

//3. 每 500ms 提交一张视频帧照片数据，注意图片尺寸小于 180 KB。
/**
 * DEMO 流程未调用。演示在 websocket 链路中实现 liveAI
 * 启动视频帧流，每 500ms 发送一次图片帧
 */
private void startVideoFrameStreaming() {
    Thread videoStreamingThread = new Thread(() -> {
        try {
            while ( !Thread.currentThread().isInterrupted()) {
                Thread.sleep(500);

                JSONObject extraObject = new JSONObject();
                extraObject.put("images", getMockOSSImage());

                multiModalDialog.updateInfo(extraObject);
                }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        } catch (JSONException e) {
            throw new RuntimeException(e);
        }
    });

    videoStreamingThread.setDaemon(true);
    videoStreamingThread.setName("LiveAI-VideoStreaming");
    videoStreamingThread.start();

}

/**
 * build images list request
 * */
private static JSONArray getMockOSSImage() {
    JSONObject imageObject = new JSONObject();
    JSONArray images = new JSONArray();
    try{

        imageObject.put("type", "base64");
        imageObject.put("value", getLocalImageBase64());

        images.put(imageObject);

    }catch (Exception e){
        e.printStackTrace();
    }
    return images;
}

//read local image to base64
private static String getLocalImageBase64(){
    return "";
}