Go SDK

// Configure upstream parameters.
upstream := &dashscope.Upstream{
    Type:        "AudioOnly",
    Mode:        "push2talk",
    AudioFormat: "pcm",
}

// Configure downstream parameters.
downstream := &dashscope.Downstream{
    Voice:            "longanhuan",
    SampleRate:       16000,
    IntermediateText: "transcript,dialog",
    AudioFormat:      "pcm",
}

// Configure client information.
clientInfo := &dashscope.ClientInfo{
    UserID: "user123",
    Device: &dashscope.Device{
        UUID: "1234567890",
    },
}

requestParams := &dashscope.RequestParameters{
    Upstream:   upstream,
    Downstream: downstream,
    ClientInfo: clientInfo,
}

type MultiModalCallback interface {
    // OnStarted is called when the dialog has started.
    //   - dialogID: The dialog ID returned by the callback.
    OnStarted(dialogID string)

    // OnStopped is called when the dialog has stopped.
    OnStopped()

    // OnStateChanged is called when the dialog state changes.
    //   - state: The new dialog state.
    //   - inputType: The input source that triggered the state change.
    OnStateChanged(state DialogState, inputType InputType)

    // OnSpeechAudioData is the callback for synthesized audio data.
    //   - data: The binary audio data.
    OnSpeechAudioData(data []byte)

    // OnError is called when an error occurs.
    //   - err: The error information.
    OnError(err error)

    // OnConnected is called after a successful connection to the server.
    OnConnected()

    // OnRespondingStarted is called when the model starts responding.
    OnRespondingStarted()

    // OnRespondingEnded is called when the model finishes responding.
    //   - payload: Additional information at the end of the response.
    OnRespondingEnded(payload map[string]interface{})

    // OnSpeechStarted is called when the start of speech input is detected.
    OnSpeechStarted()

    // OnSpeechEnded is called when the end of speech input is detected.
    OnSpeechEnded()

    // OnSpeechContent provides the recognized speech text.
    //   - payload: Data containing the recognized text.
    OnSpeechContent(payload map[string]interface{})

    // OnRespondingContent provides the model's response text.
    //   - payload: Data containing the response text.
    OnRespondingContent(payload map[string]interface{})

    // OnRequestAccepted is called when an interrupt request is accepted.
    OnRequestAccepted()

    // OnClose is called when the connection is closed.
    //   - closeStatusCode: The closing status code.
    //   - closeMsg: The closing message.
    OnClose(closeStatusCode int, closeMsg string)
}

// Handle the visual_qa intent in the callback.
  func (c *MyCallback) OnRespondingContent(payload map[string]interface{}) {
      if output, ok := payload["output"].(map[string]interface{}); ok {
          if extraInfo, ok := output["extra_info"].(map[string]interface{}); ok {
              if commands, ok := extraInfo["commands"].(string); ok {
                  if strings.Contains(commands, "visual_qa") {
                      // Trigger VQA.
                      sendImageVQA(dialog)
                  }
              }
          }
      }
  }

  // Send a VQA request.
  func sendImageVQA(dialog *dashscope.MultiModalDialog) {
      image := map[string]interface{}{
          "type":  "base64",
          "value": imageBase64Data,
      }
      // Alternatively, use a URL.
      // image := map[string]interface{}{
      //     "type":  "url",
      //     "value": imageURL,
      // }

      imagesParams := &dashscope.RequestToRespondParameters{
          Images: []interface{}{image},
      }

      // Call VQA with voice; pass an empty string for the text parameter.
      dialog.RequestToRespond("prompt", "", imagesParams, false)
  }

// 1. Set the request mode to AudioAndVideo.
upstream := &dashscope.Upstream{
    Type:        "AudioAndVideo",
    Mode:        "duplex",
    AudioFormat: "pcm",
}

// 2. Send a request to connect to the video dialog agent.
func sendConnectVideoCommand(dialog *dashscope.MultiModalDialog) {
    videoConnectCommand := []interface{}{
        map[string]interface{}{
            "action": "connect",
            "type":   "voicechat_video_channel",
        },
    }

    params := &dashscope.RequestToRespondParameters{
        BizParams: &dashscope.BizParams{
            Videos: videoConnectCommand,
        },
    }

    dialog.RequestToRespond("prompt", "", params, false)
}

// 3. Send a video frame (under 180 KB) every 500 ms.
func sendVideoFrameLoop(dialog *dashscope.MultiModalDialog, stopChan chan bool) {
    ticker := time.NewTicker(500 * time.Millisecond)
    defer ticker.Stop()

    for {
        select {
        case <-stopChan:
            return
        case <-ticker.C:
            image := map[string]interface{}{
                "type":  "base64",
                "value": getVideoFrameBase64(),
            }
            params := &dashscope.RequestToRespondParameters{
                Images: []interface{}{image},
            }
            dialog.RequestToRespond("prompt", "", params, false)
        }
    }
}