移动端Android Lite SDK

/**
  * 鉴权通过，Websocket连接建立
  */
fun onConnected()

/**
  * 对话建立，开始对话流程
  */
fun onStarted(dialogId: String)

/**
   * 状态切换
   * 包含DIALOG_IDLE，DIALOG_LISTENING，DIALOG_RESPONDING，DIALOG_THINKING
   */
fun onConvStateChangedCallback(state: State.DialogState)

/**
   * 超出用户有效输入超时时间的超时时间，收到此事件，用户需要重新启动或者结束对话
   * @param timeout 超时时间
   */
fun onSpeechTimeout(timeout: Long)

/**
   * 对话过程中的异常信息
   * @param errorInfo 异常信息
   */
fun onErrorReceived(errorCode: Int, errorMessage: String)

/**
   * 合成tts音频回调
   * @param bytes 音频数据
   */
fun onSynthesizedSpeech(bytes: ByteArray)

/**
   * 对话结束
   */
fun onStopped()

/**
   * 检测到用户开始说话
   */
fun onSpeechStarted()

/**
   * 检测到用户说话结束
   */
fun onSpeechEnded()

/**
   * 打断请求被接受
   */
fun onRequestAccepted()

/**
   * 云端开始下发tts回复
   */
fun onRespondingStarted()

/**
   * 云端下发tts回复结束
   */
fun onRespondingEnded(output: Map<String, Any>)

/**
   * 语音识别内容
   */
fun onSpeechContent(output: Map<String, Any>)

/**
   * 对话响应内容
   */
fun onRespondingContent(output: Map<String, Any>)

/**
   * websocket连接关闭
   */
fun onClosed(closeCode: Int, reason: String)

@Override
public void onRespondingContent(@NotNull final Map<String, ?> output) {
  Log.i(TAG,"onRespondingContent output:{"+output+"}");
  if (output.containsKey("extra_info")) {
    JSONObject extraInfo = (JSONObject) output.get("extra_info");
    try {
        if (extraInfo.containsKey("commands") && extraInfo.getString("commands") != null && extraInfo.getString("commands").contains("visual_qa")) {
        //包含vqa指令，这里简单实现
            uploadVQAImg();
            }
        } catch (JSONException e) {
            throw new RuntimeException(e);
        }
    }
}

//上传拍照
private void uploadVQAImg(){
    MultiModalRequestParam updateParams = null;
    JSONObject imageObject = new JSONObject();
    try{
        imageObject.put("type", "url");
        imageObject.put("value", vqaImgLink);
        List<JSONObject> images = new ArrayList<>();
        images.add(imageObject);
            updateParams= MultiModalRequestParam
                .builder()
                .images(images)
                    .build();
    }catch (JSONException e){
        e.printStackTrace();
    }
    Log.i(TAG, "uploadVQAImg: " + JSON.toJSONString(updateParams));
    multimodalDialog.requestToRespond("prompt", "",updateParams);
}

//1. 设置交互类型为AudioAndVideo
MultiModalRequestParam.UpStream.builder().mode(this.dialogMode.getValue()).type("AudioAndVideo").build();

//2. 在建联后发送voicechat_video_channel 请求
private void sendVideoChatCommand(){
    try {
        MultiModalRequestParam updateParams = MultiModalRequestParam.builder().build();
        org.json.JSONObject videoObj = new org.json.JSONObject();
        videoObj.put("action", "connect");
        videoObj.put("type", "voicechat_video_channel");

        List<org.json.JSONObject> videos = new ArrayList<>();
        videos.add(videoObj);

        updateParams.setBizParams(MultiModalRequestParam.BizParams.builder()
                .videos(videos).build());

        multimodalDialog.requestToRespond("prompt", "", updateParams);

    } catch (org.json.JSONException e) {
        Log.e(TAG, "启动视频模式失败", e);
    }
}

//3. 每 500ms 提交一张视频帧照片数据，注意图片尺寸小于 180KB。
/**
 * 演示在 websocket 链路中实现 liveAI
 * 启动视频帧流，每 500ms 发送一次图片帧
 */
private void startVideoFrameStreaming() {
    Thread videoStreamingThread = new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                while (isVideoChatting && !Thread.currentThread().isInterrupted()) {
                    Thread.sleep(500);

                    MultiModalRequestParam updateParams = null;
                    try {

                        updateParams = MultiModalRequestParam
                                .builder()
                                .images(MultimodalDialogActivity.this.getImageList())
                                .build();
                    } catch (JSONException e) {
                        e.printStackTrace();
                    }

                    multimodalDialog.updateInfo(updateParams);
                }
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            } catch (JSONException e) {
                throw new RuntimeException(e);
            }
        }
    });

    videoStreamingThread.setDaemon(true);
    videoStreamingThread.setName("LiveAI-VideoStreaming");
    videoStreamingThread.start();

}

/**
 * build images list request
 * */
private List getImageList() {
    AssetManager assetManager = getAssets();
    org.json.JSONObject imageObject = new org.json.JSONObject();
    List<org.json.JSONObject> images = new ArrayList<>();
    try{

        imageObject.put("type", "base64");
        imageObject.put("value", getImageBase64(assetManager.open("jpeg-bridge.jpg")));

        images.add(imageObject);

    }catch (Exception e){
        e.printStackTrace();
    }
    return images;
}

private String getImageBase64(InputStream file) { //200k以内大小图片
    try {

        byte[] bytes = new byte[file.available()];
        file.read(bytes);
        return Base64.encodeToString(bytes, Base64.NO_WRAP); //使用NO_WRAP 参数，避免换行
    } catch (Exception e) {
        e.printStackTrace();
    }
    return null;
}