移动端iOS SDK

    /// 服务端建联成功回调
    public var onConnected: (() -> Void)?

    /// 对话服务协议完成，可以开始说话的回调
    public var onConversationStarted: (() -> Void)?

    /// 音频事件回调。SpeechStarted/SpeechEnded/RespondingStarted/RespondingEnded
    public var onConversationEvent: ((DialogEvent) -> Void)?

    /// Idle/Listening/Thinking/Responding状态转换回调
    public var onConversationStatechanged: ((DialogState) -> Void)?

    /// 收到对话文本回调,包含用户识别内容和大模型回复内容
    public var onMessageReceived: (([String: Any]?, ResponsetMessageType) -> Void)?

    /// 错误回调
    public var onErrorReceived: ((TYError) -> Void)?

    /// 音量变化回调,RTC 链路支持
    public var onVolumeChanged: ((Float, TYVolumeSourceType) -> Void)?

    /// 发送第一帧视频，视频模式回调，RTC 链路支持
    public var onFirstVideoPacketSent: (() -> Void)?

    /// 性能数据回调
    public var onPerformaceDataReceived: ((TYChatPerformace) -> Void)?

    /// 连接状态变化回调，RTC链路支持
    public var onConnectionStatusChanged: ((RTCConnectionStatus) -> Void)?

    /// 录音数据回调，RTC链路支持
    public var onRecorderData:  ((UnsafeMutablePointer<UInt8>, Int32) -> Void)?

    /// 合成音频数据回调
    public var onSynthesizedData:  ((UnsafeMutablePointer<UInt8>, Int32) -> Void)?

//
//  ChatViewController.swift
//  ChatViewController
//
//  Created by songsong.sss on 2025/3/24.
//

import UIKit
import multimodal_dialog
import SnapKit
import AVFoundation

class ChatViewController: UIViewController {
    let videoView = UIView() //视频预览流窗口
    var HOST = ""
    var API_KEY = ""
    var WORKSPACE_ID = ""
    var APP_ID = ""
    var chain = ChainMode.WebSocket
    var isconnected : Bool
    var conversation:MultiModalDialog?
    var isInDialog = false
    var audioRecorder: TYAudioRecorder?
    var audioPlayer: AudioPlayer?
    var image_url: String?

    lazy var titleLabel1: UILabel = createLabel(text: "用户说:")
    lazy var titleLabel2: UILabel = createLabel(text: "AI 说:")
    lazy var titleLabel3: UILabel = createLabel(text: "")

    lazy var textView1: UITextView = createTextField()
    lazy var textView2: UITextView = createTextField()
    lazy var textView3: UITextView = createTextField()

    private let buttonStack: UIStackView = {
            let stack = UIStackView()
            stack.axis = .horizontal
            stack.spacing = 20
            stack.distribution = .fillEqually
            return stack
        }()

    private let refreshButton: UIButton = {
            let btn = UIButton(type: .system)
            btn.setTitle("刷新内容", for: .normal)
            btn.backgroundColor = .systemBlue
            btn.tintColor = .white
            btn.layer.cornerRadius = 8
            return btn
        }()

    private let submitButton: UIButton = {
            let btn = UIButton(type: .system)
            btn.setTitle("提交内容", for: .normal)
            btn.backgroundColor = .systemGreen
            btn.tintColor = .white
            btn.layer.cornerRadius = 8
            return btn
        }()

    init(){
        self.isconnected = false
        super.init(nibName: nil, bundle: nil)
    }

    public func updateParam(url: String, apiKey: String , workSpaceId: String, appId: String, chain: String ,image_url:String){
        self.HOST = url
        self.API_KEY = apiKey
        self.image_url = image_url
        self.WORKSPACE_ID = workSpaceId
        self.APP_ID = appId
        if chain.lowercased() == "websocket" {
            self.chain = ChainMode.WebSocket
            self.audioRecorder = TYAudioRecorder()
            self.audioRecorder?.setup(sampleRate: 16000, numOfChannels: 1, bitsPerChannel: 16)
            self.audioRecorder?.delegate = self

            self.audioPlayer = AudioPlayer()
            self.audioPlayer?.delegate = self
        }else{
            self.chain = ChainMode.RTC
        }
        self.conversation = MultiModalDialog(url: self.HOST, chainMode: self.chain,  workSpaceId:self.WORKSPACE_ID ,
                                             appId: self.APP_ID, mode: DialogMode.duplex)
    }
    override func viewDidLoad() {
        super.viewDidLoad()
        print("==================viewDidLoad")
        setupDialogCallbacks()

        //视频模式页面配置
        if self.chain == ChainMode.RTC {
            view.addSubview(videoView)
            videoView.backgroundColor = .white

            videoView.snp.makeConstraints { make in
                make.edges.equalToSuperview()
            }

            videoView.addGestureRecognizer(UITapGestureRecognizer(target: self, action: #selector(videoViewTapped)))
        }else {
        //音频模式页面配置
            setupUI()
        }
        //启动连接
        connect()
    }

    override func viewWillDisappear(_ animated: Bool) {
        TYLogger.shared.debug("==========viewWillDisappear")
        self.isInDialog = false
        self.audioPlayer?.stop()
        self.audioRecorder?.stopRecorder(shouldNotify: false)
        self.conversation?.stop()
    }

    override func viewDidDisappear(_ animated: Bool) {
        TYLogger.shared.debug("==========viewDidDisappear")
    }

    func connect(){
        self.conversation?.stop()
        isInDialog = false

        var params = MultiModalRequestParam{ multiBuilder in
            multiBuilder.upStream = MultiModalRequestParam.UpStream(builder: { upstreamBuilder in
                upstreamBuilder.mode = DialogMode.duplex.rawValue
                upstreamBuilder.type = "AudioAndVideo"
            })
            multiBuilder.clientInfo = MultiModalRequestParam.ClientInfo(builder: {
                clientInfoBuilder in
                clientInfoBuilder.userId = "test-ios-user"
                clientInfoBuilder.device = MultiModalRequestParam.ClientInfo.Device(uuid: "12345")
            })
            multiBuilder.downStream = MultiModalRequestParam.DownStream(builder: {
                downStreamBuilder in
                downStreamBuilder.sampleRate = 48000
            })
        }

        self.conversation?.start(apiKey:self.API_KEY, params: params, completion: { success, error in
            if success {
                print("success")
                self.isconnected = true
            }else {
                self.isconnected = false
                if let e = error {
                    print("连接失败，错误：\(String(describing: error))")
                } else {
                    print("连接失败，无错误信息")
                }
            }})
    }

    required init?(coder: NSCoder) {
        fatalError("init(coder:) has not been implemented")
    }

    // 必须使用 @objc 标记，因为这个方法需要暴露给 Objective-C 运行时
    @objc func videoViewTapped() {
        TYLogger.shared.debug("videoViewTapped")
    }

    deinit{
        self.conversation?.stop()
    }

    private func setupDialogCallbacks() {
        self.conversation?.onConnected = {
            print("callback: onConnectionReady ")
        }

        self.conversation?.onVolumeChanged = { volume , type in
        }

        self.conversation?.onConversationStarted = {
            TYLogger.shared.debug("callback: onConversationStarted")
            //RTC 进入视频模式
            if self.chain == ChainMode.RTC {
                self.conversation?.requestToRespond(type: "prompt", text: "", params: self.createVideoChatParams())
            }
        }

        self.conversation?.onConversationEvent = { event in
            switch event{
            case .RespondingStarted:
                self.conversation?.sendLocalRespondingStarted()
                break
            case .RespondingEnded:
                //结束发送tts数据
                self.audioPlayer?.finishFeed(true)
                break
            case .SpeechStarted:
                break
            case .SpeechEnded:
                if self.chain != ChainMode.RTC {
                    self.audioRecorder?.stopRecorder(shouldNotify: false)
                }
                break
            default:
                break
            }
            print("callback: onConversationEvent ",event)
        }

        self.conversation?.onMessageReceived = { message, type in
            let payload = message?["payload"] as? [String: Any]
            let output = payload?["output"] as? [String: Any]

            let dialogId = output?["dialog_id"] as? String
            var debug_info = ("dialog_id:").appending(dialogId ?? "")
            if type == ResponsetMessageType.speaking {
                let text = output?["text"] as? String
                DispatchQueue.main.async {
                    self.textView1.text = text
                }

            }else{
                let spoken = output?["spoken"] as? String
                let llm_request_id = output?["llm_request_id"] as? String
                let round_id = output?["round_id"] as? String

                DispatchQueue.main.async {
                    self.textView2.text = spoken
                }
                self.handleCommand(output: output)
                debug_info = debug_info.appending("\n llm_request_id:").appending(llm_request_id ?? "")
                debug_info = debug_info.appending("\n round_id:").appending(round_id ?? "")
            }
            DispatchQueue.main.async {
                self.textView3.text = debug_info
            }
        }

        self.conversation?.onConversationStatechanged = {state in
            DispatchQueue.main.async {
                self.titleLabel3.text = state.rawValue
            }

            switch state {
            case .idle:
                break
            case .listening:
                print("camera::::", self.conversation?.getCurrentCameraDirection())
                if self.chain != ChainMode.RTC {
                    self.audioRecorder?.startRecorder()
                }
                break
            case .responding:
                break
            case .thinking:
                break
            }
            print("callback: onConversationStatechanged ",state)
        }
        self.conversation?.onErrorReceived = { err in
            print("callback: onErrorReceived: ",err.key, err.message )
            if(err.key.hasPrefix("RTCException.")) {
            }
        }

        self.conversation?.onFirstVideoPacketSent = {
            print("callback: onFirstVideoPacketSent")
        }

        self.conversation?.onConnectionStatusChanged = {
            state in
            print("onConnectionStatusChanged ::::::" ,state)
            switch state{
            case .failed :
                self.conversation?.stop()
                break
            case .inited:
                break
            case .disconnected:
                break
            case .connected:
                print("==================connected")
                self.isInDialog = true
                //设置视频显示
                if self.chain == ChainMode.RTC {
                    DispatchQueue.main.async {
                        self.conversation?.setupLocalView(self.videoView, config: TYVideoConfig(fps: 24, width: 480, height: 640, bitrate: 0))
                        self.conversation?.publishLocalVideo()
                    }
                }

                break
            case .connecting:
                break
            case .reconnecting:
                break
            @unknown default:
                break
            }

        }

        self.conversation?.onSynthesizedData = { data, len in
            let audio = Data(bytes: UnsafeRawPointer(data), count: Int(len))
            TYLogger.shared.info("process \(len)")
            self.audioPlayer?.process(audioByte:data, length:Int(len))
        }

    }

    //RTC only
    private func createVideoChatParams() -> [String: Any]{
        var video:[String: Any] = [
            "action":"connect",
            "type" : "voicechat_video_channel"
        ]
        var videos = [video]

        var updateParam = MultiModalRequestParam{ multiBuilder in
            multiBuilder.bizParams = MultiModalRequestParam.BizParams(builder: {
                bizBuilder in
                bizBuilder.videos = videos
            })
        }
        return updateParam.parameters

    }

    //for VQA
    private func createImageParams() -> [String: Any]{
        var imageObject:[String: Any] = [
            "type":"url",
            "value":self.image_url ?? ""
        ]
        var images = [imageObject]

        var updateParam = MultiModalRequestParam{ multiBuilder in
            multiBuilder.clientInfo = MultiModalRequestParam.ClientInfo(builder: {
                clientInfoBuilder in
                clientInfoBuilder.userId = "test-ios-user"
                clientInfoBuilder.device = MultiModalRequestParam.ClientInfo.Device(uuid: "12345")
            })
            multiBuilder.images = images
        }
        return updateParam.parameters

    }

    //handle command
    private func handleCommand(output: [String: Any]?)-> Void {

    }

    // 创建标签的辅助方法
    private func createLabel(text: String) -> UILabel {
        let label = UILabel()
        label.text = text
        label.font = UIFont.systemFont(ofSize: 16, weight: .medium)
        label.textAlignment = .right
        label.setContentHuggingPriority(.defaultHigh, for: .horizontal)
        label.translatesAutoresizingMaskIntoConstraints = false
        return label
    }

    // 创建文本框的辅助方法
    private func createTextField() -> UITextView {
        let tv = UITextView()
        tv.layer.cornerRadius = 8
        tv.layer.borderColor = UIColor.systemGray4.cgColor
        tv.layer.borderWidth = 1
        tv.font = UIFont.systemFont(ofSize: 14)
        tv.isScrollEnabled = false
        tv.autocorrectionType = .no
        return tv
    }

    private func setupUI() {
            view.backgroundColor = .white
            view.addSubview(titleLabel1)
            view.addSubview(textView1)
            view.addSubview(titleLabel2)
            view.addSubview(textView2)
            view.addSubview(textView3)
            view.addSubview(titleLabel3)
            view.addSubview(buttonStack)

            buttonStack.addArrangedSubview(refreshButton)
            buttonStack.addArrangedSubview(submitButton)

            // AutoLayout 约束
            let safeArea = view.safeAreaLayoutGuide
            let padding: CGFloat = 20

            titleLabel1.translatesAutoresizingMaskIntoConstraints = false
            textView1.translatesAutoresizingMaskIntoConstraints = false
            titleLabel2.translatesAutoresizingMaskIntoConstraints = false
            textView2.translatesAutoresizingMaskIntoConstraints = false
            textView3.translatesAutoresizingMaskIntoConstraints = false
            titleLabel3.translatesAutoresizingMaskIntoConstraints = false
            buttonStack.translatesAutoresizingMaskIntoConstraints = false

            NSLayoutConstraint.activate([
                titleLabel1.topAnchor.constraint(equalTo: safeArea.topAnchor, constant: padding),
                titleLabel1.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                titleLabel1.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),

                textView1.topAnchor.constraint(equalTo: titleLabel1.bottomAnchor, constant: 8),
                textView1.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                textView1.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),
                textView1.heightAnchor.constraint(greaterThanOrEqualToConstant: 100),

                titleLabel2.topAnchor.constraint(equalTo: textView1.bottomAnchor, constant: padding),
                titleLabel2.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                titleLabel2.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),

                textView2.topAnchor.constraint(equalTo: titleLabel2.bottomAnchor, constant: 8),
                textView2.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                textView2.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),
                textView2.heightAnchor.constraint(greaterThanOrEqualToConstant: 100),

                textView3.topAnchor.constraint(equalTo: textView2.bottomAnchor, constant: 8),
                textView3.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                textView3.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),
                textView3.heightAnchor.constraint(greaterThanOrEqualToConstant: 100),

                titleLabel3.topAnchor.constraint(equalTo: textView3.bottomAnchor, constant: padding),
                titleLabel3.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                titleLabel3.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),

                buttonStack.bottomAnchor.constraint(equalTo: safeArea.bottomAnchor, constant: -padding),
                buttonStack.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                buttonStack.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),
                buttonStack.heightAnchor.constraint(equalToConstant: 50)
            ])
        }

}

extension [String: Any] {
    func toUTF8String() -> String? {
        do {
            let data = try JSONSerialization.data(withJSONObject: self, options: .prettyPrinted)
            return String(data: data, encoding: .utf8)
        } catch {
            return nil
        }
    }
}

extension ChatViewController: TYAudioRecorderDelegate {
    /// Recorder启动回调，在主线程中调用
    func recorderDidStart(){
        TYLogger.shared.info("recorderDidStart")
    }

    /// Recorder停止回调，在主线程中调用
    func recorderDidStop(){
        TYLogger.shared.info("recorderDidStop")
    }

    /// Recorder收录到数据，通常涉及VAD及压缩等操作，为了避免阻塞主线，
    /// 因此将在AudioQueue的线程中调用，注意线程安全！！！
    func voiceRecorded(_ buffer: UnsafeMutablePointer<UInt8>, length: Int32){
//        TYLogger.shared.debug("voiceRecorded \(length)")
        self.conversation?.sendAudioData(data: buffer, length: length)

    }

    /// 录音机无法打开或其他错误的时候会回调
    func recorderDidFail(_ error: Error?){
        TYLogger.shared.error("recorderDidFail")
    }
}

//播放器回调
extension ChatViewController:AudioPlayerDelegate {
    func playDone() {
        self.conversation?.sendLocalRespondingEnded()

    }
}

//1. 设置交互类型为AudioAndVideo
multiBuilder.upStream = MultiModalRequestParam.UpStream(builder: { upstreamBuilder in
                upstreamBuilder.mode = mode.rawValue
                upstreamBuilder.type = "AudioAndVideo"
            })
//2. 初始化 MultiModalDialog 实例，选择 RTC 链路 chainMode: ChainMode.RTC
self.conversation = MultiModalDialog(url: self.HOST, chainMode: self.chain,  workSpaceId:self.WORKSPACE_ID ,
                                             appId: self.APP_ID, mode: mode)

//3. 在建联后发送voicechat_video_channel 请求
private func createVideoChatParams() -> [String: Any]{
        var video:[String: Any] = [
            "action":"connect",
            "type" : "voicechat_video_channel"
        ]
        var videos = [video]

        var updateParam = MultiModalRequestParam{ multiBuilder in
            multiBuilder.bizParams = MultiModalRequestParam.BizParams(builder: {
                bizBuilder in
                bizBuilder.videos = videos
            })
        }
        return updateParam.parameters
}

//4. 开始音视频交互。处理交互流程。注意 demo 中有关ChainMode.RTC 类型的特殊处理。
//完整代码请参考示例代码。

//1. 设置交互类型为AudioAndVideo
multiBuilder.upStream = MultiModalRequestParam.UpStream(builder: { upstreamBuilder in
                upstreamBuilder.mode = mode.rawValue
                upstreamBuilder.type = "AudioAndVideo"
            })
//2. 初始化 MultiModalDialog 实例。选择WebSocket 链路 chainMode: ChainMode.WebSocket
self.conversation = MultiModalDialog(url: self.HOST, chainMode: self.chain,  workSpaceId:self.WORKSPACE_ID ,
                                             appId: self.APP_ID, mode: mode)

//3. 在建联后主动发起voicechat_video_channel 请求.
//或者通过语音指令”打开视频通话“返回的”open_videochat“指令触发createVideoChatParams
private func createVideoChatParams() -> [String: Any]{
        var video:[String: Any] = [
            "action":"connect",
            "type" : "voicechat_video_channel"
        ]
        var videos = [video]

        var updateParam = MultiModalRequestParam{ multiBuilder in
            multiBuilder.bizParams = MultiModalRequestParam.BizParams(builder: {
                bizBuilder in
                bizBuilder.videos = videos
            })
        }
        return updateParam.parameters
}

//4. 每 500ms 提交一张视频帧照片数据，注意图片尺寸小于 180KB。
/**
 * DEMO 中通过调用本地图片演示在 websocket 链路中实现 liveAI
 * 您集成在应用中需要自行采集视频或者图片，实现每500ms发送一帧数据。
 */
//开始后台执行发送图片的操作
func startPeriodicExecution() {
    timer = DispatchSource.makeTimerSource(queue: queue)
    timer?.schedule(deadline: .now(), repeating: .milliseconds(500))
    timer?.setEventHandler { [weak self] in
        self?.sendLocalImageBase64()
    }
    timer?.resume()
}

//update image base64
private func sendLocalImageBase64() {
    print("send_local_image_base64() called : \(Thread.current)")
    var imageObjectBase64:[String: Any] = [
                "type":"base64",
                "value":getImageAsBase64FromAssets(imageName:"bridge") //请实现图片 base64 编码
    ]
    var images = [imageObjectBase64]

    var updateParam = MultiModalRequestParam{ multiBuilder in
        multiBuilder.images = images
    }

    self.conversation?.updateInfo(params: updateParam.parameters)

}