iOS SDK integration guide

    /// Occurs when a connection to the server is established.
    public var onConnected: (() -> Void)?

    /// Occurs when the conversation is ready for you to start speaking.
    public var onConversationStarted: (() -> Void)?

    /// Occurs when a dialog event is triggered. Events include SpeechStarted, SpeechEnded, RespondingStarted, and RespondingEnded.
    public var onConversationEvent: ((DialogEvent) -> Void)?

    /// Occurs when the conversation state changes. States include Idle, Listening, Thinking, and Responding.
    public var onConversationStateChanged: ((DialogState) -> Void)?

    /// Occurs when a conversation message is received, containing the user's transcribed speech or the large model's response.
    public var onMessageReceived: (([String: Any]?, ResponsetMessageType) -> Void)?

    /// Occurs when an error is received.
    public var onErrorReceived: ((TYError) -> Void)?

    /// Occurs when the audio volume changes. Supported over the RTC link.
    public var onVolumeChanged: ((Float, TYVolumeSourceType) -> Void)?

    /// Occurs when the first video frame is sent in video mode. Supported over the RTC link.
    public var onFirstVideoPacketSent: (() -> Void)?

    /// Occurs when performance data is received.
    public var onPerformanceDataReceived: ((TYChatPerformace) -> Void)?

    /// Occurs when the connection status changes. Supported over the RTC link.
    public var onConnectionStatusChanged: ((RTCConnectionStatus) -> Void)?

    /// Occurs when recorded audio data is available. Supported over the RTC link.
    public var onRecorderData:  ((UnsafeMutablePointer<UInt8>, Int32) -> Void)?

    /// Occurs when synthesized audio data (Text-to-Speech) is available.
    public var onSynthesizedData:  ((UnsafeMutablePointer<UInt8>, Int32) -> Void)?

//
//  ChatViewController.swift
//  ChatViewController
//
//  Created by songsong.sss on 2025/3/24.
//

import UIKit
import multimodal_dialog
import SnapKit
import AVFoundation

class ChatViewController: UIViewController {
    let videoView = UIView() // Video preview stream window
    var HOST = ""
    var API_KEY = ""
    var WORKSPACE_ID = ""
    var APP_ID = ""
    var chain = ChainMode.WebSocket
    var isconnected : Bool
    var conversation:MultiModalDialog?
    var isInDialog = false
    var audioRecorder: TYAudioRecorder?
    var audioPlayer: AudioPlayer?
    var image_url: String?

    lazy var titleLabel1: UILabel = createLabel(text: "User:")
    lazy var titleLabel2: UILabel = createLabel(text: "AI:")
    lazy var titleLabel3: UILabel = createLabel(text: "")

    lazy var textView1: UITextView = createTextField()
    lazy var textView2: UITextView = createTextField()
    lazy var textView3: UITextView = createTextField()

    private let buttonStack: UIStackView = {
            let stack = UIStackView()
            stack.axis = .horizontal
            stack.spacing = 20
            stack.distribution = .fillEqually
            return stack
        }()

    private let refreshButton: UIButton = {
            let btn = UIButton(type: .system)
            btn.setTitle("Refresh Content", for: .normal)
            btn.backgroundColor = .systemBlue
            btn.tintColor = .white
            btn.layer.cornerRadius = 8
            return btn
        }()

    private let submitButton: UIButton = {
            let btn = UIButton(type: .system)
            btn.setTitle("Submit Content", for: .normal)
            btn.backgroundColor = .systemGreen
            btn.tintColor = .white
            btn.layer.cornerRadius = 8
            return btn
        }()

    init(){
        self.isconnected = false
        super.init(nibName: nil, bundle: nil)
    }

    public func updateParam(url: String, apiKey: String , workSpaceId: String, appId: String, chain: String ,image_url:String){
        self.HOST = url
        self.API_KEY = apiKey
        self.image_url = image_url
        self.WORKSPACE_ID = workSpaceId
        self.APP_ID = appId
        if chain.lowercased() == "websocket" {
            self.chain = ChainMode.WebSocket
            self.audioRecorder = TYAudioRecorder()
            self.audioRecorder?.setup(sampleRate: 16000, numOfChannels: 1, bitsPerChannel: 16)
            self.audioRecorder?.delegate = self

            self.audioPlayer = AudioPlayer()
            self.audioPlayer?.delegate = self
        }else{
            self.chain = ChainMode.RTC
        }
        self.conversation = MultiModalDialog(url: self.HOST, chainMode: self.chain,  workSpaceId:self.WORKSPACE_ID ,
                                             appId: self.APP_ID, mode: DialogMode.duplex)
    }
    override func viewDidLoad() {
        super.viewDidLoad()
        print("==================viewDidLoad")
        setupDialogCallbacks()

        // Configure the UI for video mode
        if self.chain == ChainMode.RTC {
            view.addSubview(videoView)
            videoView.backgroundColor = .white

            videoView.snp.makeConstraints { make in
                make.edges.equalToSuperview()
            }

            videoView.addGestureRecognizer(UITapGestureRecognizer(target: self, action: #selector(videoViewTapped)))
        }else {
        // Configure the UI for audio mode
            setupUI()
        }
        // Start the connection
        connect()
    }

    override func viewWillDisappear(_ animated: Bool) {
        TYLogger.shared.debug("==========viewWillDisappear")
        self.isInDialog = false
        self.audioPlayer?.stop()
        self.audioRecorder?.stopRecorder(shouldNotify: false)
        self.conversation?.stop()
    }

    override func viewDidDisappear(_ animated: Bool) {
        TYLogger.shared.debug("==========viewDidDisappear")
    }

    func connect(){
        self.conversation?.stop()
        isInDialog = false

        var params = MultiModalRequestParam{ multiBuilder in
            multiBuilder.upStream = MultiModalRequestParam.UpStream(builder: { upstreamBuilder in
                upstreamBuilder.mode = DialogMode.duplex.rawValue
                upstreamBuilder.type = "AudioAndVideo"
            })
            multiBuilder.clientInfo = MultiModalRequestParam.ClientInfo(builder: {
                clientInfoBuilder in
                clientInfoBuilder.userId = "test-ios-user"
                clientInfoBuilder.device = MultiModalRequestParam.ClientInfo.Device(uuid: "12345")
            })
            multiBuilder.downStream = MultiModalRequestParam.DownStream(builder: {
                downStreamBuilder in
                downStreamBuilder.sampleRate = 48000
            })
        }

        self.conversation?.start(apiKey:self.API_KEY, params: params, completion: { success, error in
            if success {
                print("success")
                self.isconnected = true
            }else {
                self.isconnected = false
                if let e = error {
                    print("Connection failed. Error: \(String(describing: error))")
                } else {
                    print("Connection failed. No error message.")
                }
            }})
    }

    required init?(coder: NSCoder) {
        fatalError("init(coder:) has not been implemented")
    }

    // The @objc attribute is required because this method must be exposed to the Objective-C runtime.
    @objc func videoViewTapped() {
        TYLogger.shared.debug("videoViewTapped")
    }

    deinit{
        self.conversation?.stop()
    }

    private func setupDialogCallbacks() {
        self.conversation?.onConnected = {
            print("callback: onConnectionReady ")
        }

        self.conversation?.onVolumeChanged = { volume , type in
        }

        self.conversation?.onConversationStarted = {
            TYLogger.shared.debug("callback: onConversationStarted")
            // Enter video mode for RTC.
            if self.chain == ChainMode.RTC {
                self.conversation?.requestToRespond(type: "prompt", text: "", params: self.createVideoChatParams())
            }
        }

        self.conversation?.onConversationEvent = { event in
            switch event{
            case .RespondingStarted:
                self.conversation?.sendLocalRespondingStarted()
                break
            case .RespondingEnded:
                // Stop sending TTS data.
                self.audioPlayer?.finishFeed(true)
                break
            case .SpeechStarted:
                break
            case .SpeechEnded:
                if self.chain != ChainMode.RTC {
                    self.audioRecorder?.stopRecorder(shouldNotify: false)
                }
                break
            default:
                break
            }
            print("callback: onConversationEvent ",event)
        }

        self.conversation?.onMessageReceived = { message, type in
            let payload = message?["payload"] as? [String: Any]
            let output = payload?["output"] as? [String: Any]

            let dialogId = output?["dialog_id"] as? String
            var debug_info = ("dialog_id:").appending(dialogId ?? "")
            if type == ResponsetMessageType.speaking {
                let text = output?["text"] as? String
                DispatchQueue.main.async {
                    self.textView1.text = text
                }

            }else{
                let spoken = output?["spoken"] as? String
                let llm_request_id = output?["llm_request_id"] as? String
                let round_id = output?["round_id"] as? String

                DispatchQueue.main.async {
                    self.textView2.text = spoken
                }
                self.handleCommand(output: output)
                debug_info = debug_info.appending("\n llm_request_id:").appending(llm_request_id ?? "")
                debug_info = debug_info.appending("\n round_id:").appending(round_id ?? "")
            }
            DispatchQueue.main.async {
                self.textView3.text = debug_info
            }
        }

        self.conversation?.onConversationStatechanged = {state in
            DispatchQueue.main.async {
                self.titleLabel3.text = state.rawValue
            }

            switch state {
            case .idle:
                break
            case .listening:
                print("camera::::", self.conversation?.getCurrentCameraDirection())
                if self.chain != ChainMode.RTC {
                    self.audioRecorder?.startRecorder()
                }
                break
            case .responding:
                break
            case .thinking:
                break
            }
            print("callback: onConversationStatechanged ",state)
        }
        self.conversation?.onErrorReceived = { err in
            print("callback: onErrorReceived: ",err.key, err.message )
            if(err.key.hasPrefix("RTCException.")) {
            }
        }

        self.conversation?.onFirstVideoPacketSent = {
            print("callback: onFirstVideoPacketSent")
        }

        self.conversation?.onConnectionStatusChanged = {
            state in
            print("onConnectionStatusChanged ::::::" ,state)
            switch state{
            case .failed :
                self.conversation?.stop()
                break
            case .inited:
                break
            case .disconnected:
                break
            case .connected:
                print("==================connected")
                self.isInDialog = true
                // Set up the video display.
                if self.chain == ChainMode.RTC {
                    DispatchQueue.main.async {
                        self.conversation?.setupLocalView(self.videoView, config: TYVideoConfig(fps: 24, width: 480, height: 640, bitrate: 0))
                        self.conversation?.publishLocalVideo()
                    }
                }

                break
            case .connecting:
                break
            case .reconnecting:
                break
            @unknown default:
                break
            }

        }

        self.conversation?.onSynthesizedData = { data, len in
            let audio = Data(bytes: UnsafeRawPointer(data), count: Int(len))
            TYLogger.shared.info("process \(len)")
            self.audioPlayer?.process(audioByte:data, length:Int(len))
        }

    }

    //RTC only
    private func createVideoChatParams() -> [String: Any]{
        var video:[String: Any] = [
            "action":"connect",
            "type" : "voicechat_video_channel"
        ]
        var videos = [video]

        var updateParam = MultiModalRequestParam{ multiBuilder in
            multiBuilder.bizParams = MultiModalRequestParam.BizParams(builder: {
                bizBuilder in
                bizBuilder.videos = videos
            })
        }
        return updateParam.parameters

    }

    // For VQA
    private func createImageParams() -> [String: Any]{
        var imageObject:[String: Any] = [
            "type":"url",
            "value":self.image_url ?? ""
        ]
        var images = [imageObject]

        var updateParam = MultiModalRequestParam{ multiBuilder in
            multiBuilder.clientInfo = MultiModalRequestParam.ClientInfo(builder: {
                clientInfoBuilder in
                clientInfoBuilder.userId = "test-ios-user"
                clientInfoBuilder.device = MultiModalRequestParam.ClientInfo.Device(uuid: "12345")
            })
            multiBuilder.images = images
        }
        return updateParam.parameters

    }

    // Handle command
    private func handleCommand(output: [String: Any]?)-> Void {

    }

    // Helper method for creating a label
    private func createLabel(text: String) -> UILabel {
        let label = UILabel()
        label.text = text
        label.font = UIFont.systemFont(ofSize: 16, weight: .medium)
        label.textAlignment = .right
        label.setContentHuggingPriority(.defaultHigh, for: .horizontal)
        label.translatesAutoresizingMaskIntoConstraints = false
        return label
    }

    // Helper method for creating a text view
    private func createTextField() -> UITextView {
        let tv = UITextView()
        tv.layer.cornerRadius = 8
        tv.layer.borderColor = UIColor.systemGray4.cgColor
        tv.layer.borderWidth = 1
        tv.font = UIFont.systemFont(ofSize: 14)
        tv.isScrollEnabled = false
        tv.autocorrectionType = .no
        return tv
    }

    private func setupUI() {
            view.backgroundColor = .white
            view.addSubview(titleLabel1)
            view.addSubview(textView1)
            view.addSubview(titleLabel2)
            view.addSubview(textView2)
            view.addSubview(textView3)
            view.addSubview(titleLabel3)
            view.addSubview(buttonStack)

            buttonStack.addArrangedSubview(refreshButton)
            buttonStack.addArrangedSubview(submitButton)

            // AutoLayout constraints
            let safeArea = view.safeAreaLayoutGuide
            let padding: CGFloat = 20

            titleLabel1.translatesAutoresizingMaskIntoConstraints = false
            textView1.translatesAutoresizingMaskIntoConstraints = false
            titleLabel2.translatesAutoresizingMaskIntoConstraints = false
            textView2.translatesAutoresizingMaskIntoConstraints = false
            textView3.translatesAutoresizingMaskIntoConstraints = false
            titleLabel3.translatesAutoresizingMaskIntoConstraints = false
            buttonStack.translatesAutoresizingMaskIntoConstraints = false

            NSLayoutConstraint.activate([
                titleLabel1.topAnchor.constraint(equalTo: safeArea.topAnchor, constant: padding),
                titleLabel1.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                titleLabel1.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),

                textView1.topAnchor.constraint(equalTo: titleLabel1.bottomAnchor, constant: 8),
                textView1.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                textView1.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),
                textView1.heightAnchor.constraint(greaterThanOrEqualToConstant: 100),

                titleLabel2.topAnchor.constraint(equalTo: textView1.bottomAnchor, constant: padding),
                titleLabel2.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                titleLabel2.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),

                textView2.topAnchor.constraint(equalTo: titleLabel2.bottomAnchor, constant: 8),
                textView2.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                textView2.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),
                textView2.heightAnchor.constraint(greaterThanOrEqualToConstant: 100),

                textView3.topAnchor.constraint(equalTo: textView2.bottomAnchor, constant: 8),
                textView3.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                textView3.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),
                textView3.heightAnchor.constraint(greaterThanOrEqualToConstant: 100),

                titleLabel3.topAnchor.constraint(equalTo: textView3.bottomAnchor, constant: padding),
                titleLabel3.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                titleLabel3.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),

                buttonStack.bottomAnchor.constraint(equalTo: safeArea.bottomAnchor, constant: -padding),
                buttonStack.leadingAnchor.constraint(equalTo: safeArea.leadingAnchor, constant: padding),
                buttonStack.trailingAnchor.constraint(equalTo: safeArea.trailingAnchor, constant: -padding),
                buttonStack.heightAnchor.constraint(equalToConstant: 50)
            ])
        }

}

extension [String: Any] {
    func toUTF8String() -> String? {
        do {
            let data = try JSONSerialization.data(withJSONObject: self, options: .prettyPrinted)
            return String(data: data, encoding: .utf8)
        } catch {
            return nil
        }
    }
}

extension ChatViewController: TYAudioRecorderDelegate {
    /// Callback for when the recorder starts. This is called on the main thread.
    func recorderDidStart(){
        TYLogger.shared.info("recorderDidStart")
    }

    /// Callback for when the recorder stops. This is called on the main thread.
    func recorderDidStop(){
        TYLogger.shared.info("recorderDidStop")
    }

    /// Called when the recorder captures audio data. This method is called on an AudioQueue thread to avoid blocking the main thread
    /// because it typically involves operations such as VAD and compression. Ensure thread safety.
    func voiceRecorded(_ buffer: UnsafeMutablePointer<UInt8>, length: Int32){
//        TYLogger.shared.debug("voiceRecorded \(length)")
        self.conversation?.sendAudioData(data: buffer, length: length)

    }

    /// This callback is triggered when the recorder fails to open or another error occurs.
    func recorderDidFail(_ error: Error?){
        TYLogger.shared.error("recorderDidFail")
    }
}

// Player callbacks
extension ChatViewController:AudioPlayerDelegate {
    func playDone() {
        self.conversation?.sendLocalRespondingEnded()

    }
}

// 1. Set the interaction type to AudioAndVideo.
multiBuilder.upStream = MultiModalRequestParam.UpStream(builder: { upstreamBuilder in
                upstreamBuilder.mode = mode.rawValue
                upstreamBuilder.type = "AudioAndVideo"
            })
// 2. Initialize a MultiModalDialog instance and select the RTC link by setting chainMode to ChainMode.RTC.
self.conversation = MultiModalDialog(url: self.HOST, chainMode: self.chain,  workSpaceId:self.WORKSPACE_ID ,
                                             appId: self.APP_ID, mode: mode)

// 3. After the connection is established, send a voicechat_video_channel request.
private func createVideoChatParams() -> [String: Any]{
        var video:[String: Any] = [
            "action":"connect",
            "type" : "voicechat_video_channel"
        ]
        var videos = [video]

        var updateParam = MultiModalRequestParam{ multiBuilder in
            multiBuilder.bizParams = MultiModalRequestParam.BizParams(builder: {
                bizBuilder in
                bizBuilder.videos = videos
            })
        }
        return updateParam.parameters
}

// 4. Start the audio and video interaction. See the sample code for the complete flow, including special handling for ChainMode.RTC.

// 1. Set the interaction type to "AudioAndVideo".
multiBuilder.upStream = MultiModalRequestParam.UpStream(builder: { upstreamBuilder in
                upstreamBuilder.mode = mode.rawValue
                upstreamBuilder.type = "AudioAndVideo"
            })
// 2. Initialize a MultiModalDialog instance. Select the WebSocket link by setting chainMode to ChainMode.WebSocket.
self.conversation = MultiModalDialog(url: self.HOST, chainMode: self.chain,  workSpaceId:self.WORKSPACE_ID ,
                                             appId: self.APP_ID, mode: mode)

// 3. After the connection is established, send a 'voicechat_video_channel' request.
// Alternatively, use the 'open_videochat' command to trigger createVideoChatParams. The service returns this command in response to the "Open video call" voice command.
private func createVideoChatParams() -> [String: Any]{
        var video:[String: Any] = [
            "action":"connect",
            "type" : "voicechat_video_channel"
        ]
        var videos = [video]

        var updateParam = MultiModalRequestParam{ multiBuilder in
            multiBuilder.bizParams = MultiModalRequestParam.BizParams(builder: {
                bizBuilder in
                bizBuilder.videos = videos
            })
        }
        return updateParam.parameters
}

// 4. Submit a video frame every 500 ms. The image size must be less than 180 KB.
/**
 * This demo shows how to use LiveAI over a WebSocket link by sending a local image.
 * In your application, you must implement the logic to capture and send a video frame every 500 ms.
 */
// Start the background execution to send images.
func startPeriodicExecution() {
    timer = DispatchSource.makeTimerSource(queue: queue)
    timer?.schedule(deadline: .now(), repeating: .milliseconds(500))
    timer?.setEventHandler { [weak self] in
        self?.sendLocalImageBase64()
    }
    timer?.resume()
}

// Send a base64-encoded image to update the conversation.
private func sendLocalImageBase64() {
    print("send_local_image_base64() called : \(Thread.current)")
    var imageObjectBase64:[String: Any] = [
                "type":"base64",
                "value":getImageAsBase64FromAssets(imageName:"bridge") // Implement the base64 encoding for the image.
    ]
    var images = [imageObjectBase64]

    var updateParam = MultiModalRequestParam{ multiBuilder in
        multiBuilder.images = images
    }

    self.conversation?.updateInfo(params: updateParam.parameters)

}