服务端 Go SDK

// 配置上行参数
upstream := &dashscope.Upstream{
    Type:        "AudioOnly",
    Mode:        "push2talk",
    AudioFormat: "pcm",
}

// 配置下行参数
downstream := &dashscope.Downstream{
    Voice:            "longanhuan",
    SampleRate:       16000,
    IntermediateText: "transcript,dialog",
    AudioFormat:      "pcm",
}

// 配置客户端信息
clientInfo := &dashscope.ClientInfo{
    UserID: "user123",
    Device: &dashscope.Device{
        UUID: "1234567890",
    },
}

requestParams := &dashscope.RequestParameters{
    Upstream:   upstream,
    Downstream: downstream,
    ClientInfo: clientInfo,
}

type MultiModalCallback interface {
    // OnStarted 通知对话开始
    //   - dialogID: 回调对话ID
    OnStarted(dialogID string)

    // OnStopped 通知对话停止
    OnStopped()

    // OnStateChanged 对话状态改变
    //   - state: 新的对话状态
    //   - inputType: 触发状态变更的输入来源
    OnStateChanged(state DialogState, inputType InputType)

    // OnSpeechAudioData 合成音频数据回调
    //   - data: 音频二进制数据
    OnSpeechAudioData(data []byte)

    // OnError 发生错误时调用此方法
    //   - err: 错误信息
    OnError(err error)

    // OnConnected 成功连接到服务器后调用此方法
    OnConnected()

    // OnRespondingStarted 回复开始回调
    OnRespondingStarted()

    // OnRespondingEnded 回复结束
    //   - payload: 回复结束时的附加信息
    OnRespondingEnded(payload map[string]interface{})

    // OnSpeechStarted 检测到语音输入开始
    OnSpeechStarted()

    // OnSpeechEnded 检测到语音输入结束
    OnSpeechEnded()

    // OnSpeechContent 语音识别文本
    //   - payload: 包含识别文本的数据
    OnSpeechContent(payload map[string]interface{})

    // OnRespondingContent 大模型回复文本
    //   - payload: 包含回复文本的数据
    OnRespondingContent(payload map[string]interface{})

    // OnRequestAccepted 打断请求被接受
    OnRequestAccepted()

    // OnClose 连接关闭时调用此方法
    //   - closeStatusCode: 关闭状态码
    //   - closeMsg: 关闭消息
    OnClose(closeStatusCode int, closeMsg string)
}

// callback 中处理 visual_qa 意图
  func (c *MyCallback) OnRespondingContent(payload map[string]interface{}) {
      if output, ok := payload["output"].(map[string]interface{}); ok {
          if extraInfo, ok := output["extra_info"].(map[string]interface{}); ok {
              if commands, ok := extraInfo["commands"].(string); ok {
                  if strings.Contains(commands, "visual_qa") {
                      // 触发 VQA
                      sendImageVQA(dialog)
                  }
              }
          }
      }
  }

  // 发送 VQA 请求
  func sendImageVQA(dialog *dashscope.MultiModalDialog) {
      image := map[string]interface{}{
          "type":  "base64",
          "value": imageBase64Data,
      }
      // 或者使用 URL 方式
      // image := map[string]interface{}{
      //     "type":  "url",
      //     "value": imageURL,
      // }

      imagesParams := &dashscope.RequestToRespondParameters{
          Images: []interface{}{image},
      }

      // 使用语音调用 VQA，text 传空字符串即可
      dialog.RequestToRespond("prompt", "", imagesParams, false)
  }

// 1. 设置请求模式为 AudioAndVideo
upstream := &dashscope.Upstream{
    Type:        "AudioAndVideo",
    Mode:        "duplex",
    AudioFormat: "pcm",
}

// 2. 发送连接到视频对话 Agent 请求
func sendConnectVideoCommand(dialog *dashscope.MultiModalDialog) {
    videoConnectCommand := []interface{}{
        map[string]interface{}{
            "action": "connect",
            "type":   "voicechat_video_channel",
        },
    }

    params := &dashscope.RequestToRespondParameters{
        BizParams: &dashscope.BizParams{
            Videos: videoConnectCommand,
        },
    }

    dialog.RequestToRespond("prompt", "", params, false)
}

// 3. 间隔 500ms 发送一张 180KB 以下的视频帧图片
func sendVideoFrameLoop(dialog *dashscope.MultiModalDialog, stopChan chan bool) {
    ticker := time.NewTicker(500 * time.Millisecond)
    defer ticker.Stop()

    for {
        select {
        case <-stopChan:
            return
        case <-ticker.C:
            image := map[string]interface{}{
                "type":  "base64",
                "value": getVideoFrameBase64(),
            }
            params := &dashscope.RequestToRespondParameters{
                Images: []interface{}{image},
            }
            dialog.RequestToRespond("prompt", "", params, false)
        }
    }
}