HarmonyOS SDK for real-time multimodal interaction

// Release project: entry/oh-package.json5
"dependencies": {
  "@ali/convsdk": "file:./libs/convsdk.har",             // Voice conversation SDK
  "@ali/mmsdk": "file:./libs/multimodal_sdk.har"         // Multimodal conversation SDK
}

/**
  * Initialize the dialog class
  * @param url: Server URL
  * @param chainMode: Link mode
  * @param workspaceId: Workspace ID
  * @param appId: Application ID
  * @param dialogMode: Interaction mode
  */
constructor(url: string, chainMode: ChainMode, workspaceId: string, appId: string, dialogMode: DialogMode)

/**
 * Set the application context for NativeConversation resource copying and other operations.
 * Call this method before createConversation.
 * @param context Application context
 */
setContext(context: common.UIAbilityContext): void

/**
  * Create a conversation
  * @param requestParam Request parameters
  * @param chatCallback Conversation callback
  */
async createConversation(params: MultiModalRequestParam, chatCallback: IDialogCallback): Promise<void>

/**
 * Connect and start the conversation
 * @param apiKey   API Key
 * @param dialogId Dialog ID; pass an empty string for a new conversation
 * @param taskId   Task ID
 */
start(apiKey: string, dialogId: string, taskId: string): void

// Convenience method without taskId (equivalent to start(apiKey, dialogId, ""))
startTwo(apiKey: string, dialogId: string): void

/**
  * Disconnect and end the conversation
  */
stop(): void

/**
  * Destroy the instance
  */
destroy(): void

/**
  * Set the timeout; a timeout event is reported if no speech is detected within this period
  * @param timeout Timeout in milliseconds
  */
setDialogTimeout(timeout: number): void

/**
  * Interrupt the AI's speech
  */
interrupt(): void

/**
 * Notify the server to start audio uploads. Call only in the Listening state.
 * Call only in Push2Talk mode.
 */
startSpeech(): void

/**
 * Notify the server to stop audio uploads.Call only in Push2Talk mode.
 * The user finishes speaking in Push2Talk mode.
 */
stopSpeech(): void

/**
 * The user cancels speech in Push2Talk mode
 */
cancelSpeech(): void

/**
  * Upload audio.
  * @param audioData Audio frame data
  */
sendAudioData(audioData: Uint8Array | null): void

/**
 * Send reference audio for acoustic echo cancellation (AEC)
 * @param audioData Reference audio data (must match playback data)
 */
sendRefData(audioData: Uint8Array | null): void

/**
 * Ask the server to answer a question or synthesize speech
 * @param type   transcript converts text to speech; prompt sends text to the model for an answer;
 *               visual_qa recognizes an image
 * @param text   Text; for visual_qa, an image URL or base64 data (image <= 180 KB)
 * @param params Additional parameters
 */
requestToRespond(type: string, text: string, params: Record<string, Object>): void

/**
 * Push a video frame
 * @param videoFrame Video frame data
 * @param callback   Frame push result callback
 */
sendVideoFrame(videoFrame: TYVideoFrame, callback: IVideoChatPushFrameCallback): void

/**
 * Notify the start of a spoken response
 */
sendResponseStarted(): void

/**
 * Notify the end of a spoken response
 */
sendResponseEnded(): void

/**
 * Whether the current configuration uses duplex mode
 */
isDuplexMode(): boolean

/**
 * Whether the current mode is Push2Talk (true only in Push2Talk)
 */
isPush2TalkMode(): boolean

/**
 * Whether the current mode is Tap2Talk (true only in Tap2Talk)
 */
isTap2TalkMode(): boolean

/**
 * Get the current conversation state
 */
getDialogState(): DialogState

/**
 * Get the current network quality
 */
getNetworkQuality(): number

/**
 * Resume interaction
 */
beginInteraction(): void

/**
 * Update conversation information
 * @param paramValue Updated parameters
 */
updateInfo(paramValue: Record<string, Object>): void

/**
 * Send a heartbeat
 */
sendHearBeat(): void

/**
 * Whether to use RTC internal audio capture (default: true)
 */
static isRtcUseInternalAudio(): boolean
static setRtcUseInternalAudio(useInternal: boolean): void

/**
 * Whether to use RTC internal video capture (default: true)
 */
static isRtcUseInternalVideo(): boolean
static setRtcUseInternalVideo(useInternal: boolean): void

/**
 * Conversation callback interface
 */
export interface IDialogCallback {
  /**
   * Conversation start result
   */
  onStartResult(isSuccess: boolean, errorInfo: TYError | null): void;

  /**
   * Interruption result
   */
  onInterruptResult(isSuccess: boolean, errorInfo: TYError | null): void;

  /**
   * Conversation ready after clients join the channel; business APIs may then be called
   * @param dialogId Dialog ID
   */
  onReadyToSpeech(dialogId: string): void;

  /**
   * State change callback: DIALOG_IDLE / DIALOG_LISTENING / DIALOG_THINKING / DIALOG_RESPONDING
   */
  onConvStateChangedCallback(state: DialogState): void;

  /**
   * Volume level callback, 0-100
   * @param audioLevel Volume level, 0-100
   * @param audioType  Volume source: MIC for local capture / PLAYER for remote TTS
   */
  onConvSoundLevelCallback(audioLevel: number, audioType: TYVolumeSourceType): void;

  /**
   * Conversation timeout callback
   * @param timeout Timeout in milliseconds
   */
  onSpeechTimeout(timeout: number): void;

  /**
   * Playback audio data callback
   */
  onPlaybackAudioData(bytes: Uint8Array): void;

  /**
   * Errors during the conversation
   */
  onErrorReceived(errorInfo: TYError): void;

  /**
   * Conversation event callback
   */
  onConvEventCallback(var1: ConvEvent | null): void;

  /**
   * Debug information callback
   */
  onDebugInfoTrack(level: number, type: TYDebugInfoType, debugInfo: string): void;

  /**
   * Downstream synthesized speech callback
   */
  onSynthesizedSpeech(bytes: Uint8Array): void;

  /**
   * Wake word detection callback
   */
  onKeyWordSpotted(word: string, type: KeyWordsType): void;
}

{
  "header": { "request_id": "9B32878****************3D053", "status_code": 200 },
  "payload": {
    "output": {
      "event": "SpeechContent",
      "dialog_id": "b39398c9dd8147***35cdea81f7",
      "text": "一二三",
      "finished": false
    }
  }
}

{
  "header": { "request_id": "9B32878****************3D053", "status_code": 200 },
  "payload": {
    "output": {
      "event": "AIResponse",
      "dialog_id": "b39398c9dd8147***35cdea81f7",
      "text": "好的，这是一个测试。",
      "finished": true
    }
  }
}

/**
  * Conversation error
  * @param errorInfo: Error information
  */
onErrorReceived(errorInfo: TYError): void;

// Enable duplex mode
let dialog = new MultiModalDialog(url, ChainMode.WEBSOCKET, workspaceId, appId, DialogMode.DUPLEX)
dialog.setContext(context)
MultiModalDialog.wsUseInternalAEC = true
let requestParam = MultiModalRequestParam.builder().build()
await dialog.createConversation(requestParam, this)

// Send microphone audio (16 kHz PCM for server recognition)
dialog.sendAudioData(audioData)

// In onSynthesizedSpeech, send downstream TTS data as the AEC reference
// bytes is the callback data; also pass it to the application player
dialog.sendRefData(bytes)

// The application player must use the downstream TTS sample rate

// On receiving the "visual_qa" command
private handleVisualQaCommand() {
  // 1. Take a photo or select an image from the gallery
  let imageUrl = uploadImageToOSS()      // Generate a public image URL
  // 2. Send the image URL for image recognition
  dialog.requestToRespond("visual_qa", imageUrl, {})

  // 3. Alternatively, send base64 image data (image <= 180 KB)
  let imageBase64 = getLocalImageBase64()
  dialog.requestToRespond("visual_qa", imageBase64, {})
}

// 1. Set the interaction type to AudioAndVideo
let dialog = new MultiModalDialog(url, ChainMode.WEBSOCKET, workspaceId, appId, DialogMode.DUPLEX);
// Set UpStream.type to AudioAndVideo
// UpStream defaults to AudioOnly; explicitly set AudioAndVideo for LiveAI video calls
let requestParam = MultiModalRequestParam.builder()
  .upStream(UpStream.builder().type('AudioAndVideo').build())
  .build()

// 2. After the conversation starts, send a video frame every 500 ms
// sendVideoFrame requires a frame push result callback (IVideoChatPushFrameCallback)
class VideoFramePushCb implements IVideoChatPushFrameCallback {
  onFinishPushFrame(isSuccess: boolean, errorMsg: string): void {
    //Handle the result, for example by logging and retrying on failure
  }
}
setInterval(() => {
  dialog.sendVideoFrame(getNextVideoFrame(), new VideoFramePushCb())   // LiveAI video frame
}, 500)

// Request TTS synthesis in the Listening state
dialog.requestToRespond("transcript", "幸福是一种技能，是你摒弃了外在多余欲望后的内心平和。", {})

// Set custom prompt variables
let requestParam = MultiModalRequestParam.builder()
  .bizParams(BizParams.builder()
    .userPromptParams({ user_name: "大米" })
    .build())
  .build()

// Set the ASR correction vocabulary
let requestParam = MultiModalRequestParam.builder()
  .upStream(UpStream.builder()
    .asrPostProcessing([
      ReplaceWord.builder()
        .source("1加1")
        .target("一加一")
        .matchMode("partial")
        .build()
    ])
    .build())
  .build()