讯飞虚拟人简单交互实现

本文以科大讯飞虚拟人为例前端使用简单html接入结合后端SpringBoot做演示。1.前期准备登录讯飞虚拟人主页https://virtual-man.xfyun.cn/注册并登录应用控制台。选择接口服务点击免费开通后申请订阅选择接口能力。目前免费如果需要AI交互勾选大模型对话如果涉及的业务数据保密或者是已有对应的AI智能体服务只是虚拟人描述或者播报选择在线虚拟人驱动。填写下面信息单位信息自定义填写。授权成功后可以在我的订阅查看审批状态已授权后可以创建接口服务。接口创建好后可以拿到连接信息已对话为例。下面在线虚拟人驱动和大模型对话可以进行配置。配置后重新发布即可。2.前端开发根据官方文档可以多种方式接入可以直接调用sdk也可以使用api。vue项目可以直接使用。为了方便操作我使用简单html方式。!DOCTYPE html html langzh-CN head meta charsetUTF-8 title讯飞虚拟人 - 智能对话/title style * { margin: 0; padding: 0; box-sizing: border-box; } body { font-family: Segoe UI, sans-serif; background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); min-height: 100vh; padding: 24px; } .container { max-width: 1400px; margin: 0 auto; } .header { text-align: center; margin-bottom: 30px; } .header h1 { color: white; } .dashboard { display: flex; gap: 24px; flex-wrap: wrap; } .config-panel { flex: 1; min-width: 320px; background: rgba(255,255,255,0.95); border-radius: 24px; padding: 24px; } .display-panel { flex: 2; min-width: 500px; background: rgba(0,0,0,0.3); border-radius: 24px; padding: 20px; } .input-group { margin-bottom: 18px; } .input-group label { display: block; font-size: 0.85rem; font-weight: 600; color: #334e68; margin-bottom: 6px; } .input-group input { width: 100%; padding: 12px; border-radius: 12px; border: 1px solid #ddd; background: #f8fafc; } .btn { background: #00aaff; color: white; padding: 12px; border-radius: 40px; border: none; cursor: pointer; width: 100%; } .video-container { background: #0f172a; border-radius: 20px; aspect-ratio: 16 / 9; margin-bottom: 20px; overflow: hidden; } #videoPlayer { width: 100%; height: 100%; object-fit: contain; background: #000; } .chat-area { background: rgba(255,255,255,0.1); border-radius: 20px; padding: 16px; } .message-history { background: #1e293b; border-radius: 16px; padding: 12px; height: 280px; overflow-y: auto; margin-bottom: 12px; } .message { margin-bottom: 12px; display: flex; } .message.user { justify-content: flex-end; } .message.bot .bubble { background: #334155; color: white; } .message.user .bubble { background: #00aaff; color: white; } .bubble { padding: 8px 14px; border-radius: 18px; max-width: 85%; word-break: break-word; } .input-row { display: flex; gap: 10px; } .input-row input { flex: 1; padding: 12px; border-radius: 40px; border: none; } .send-btn { background: #00aaff; border: none; padding: 0 24px; border-radius: 40px; color: white; cursor: pointer; } .status-badge { display: inline-block; background: #e2e8f0; padding: 4px 12px; border-radius: 40px; font-size: 12px; } .status-badge.connected { background: #22c55e; color: white; } .log-area { background: #0f172a; border-radius: 12px; padding: 10px; margin-top: 12px; font-family: monospace; font-size: 12px; color: #86efac; max-height: 100px; overflow-y: auto; } .thinking { color: #94a3b8; font-style: italic; padding: 8px 14px; } media (max-width: 900px) { .dashboard { flex-direction: column; } } /style script srchttps://cdn.bootcdn.net/ajax/libs/flv.js/1.6.2/flv.min.js/script /head body div classcontainer div classheader h1 讯飞虚拟人 · 智能对话/h1 p虚拟人理解您的意图并智能回复 | 需开启大模型对话能力/p /div div classdashboard div classconfig-panel button classbtn idstartBtn启动虚拟人/button button classbtn idstopBtn stylemargin-top: 10px; background:gray; disabled停止/button /div div classdisplay-panel div classvideo-container video idvideoPlayer autoplay playsinline/video /div div classchat-area div styledisplay: flex; justify-content: space-between; margin-bottom: 12px; span stylecolor:white; 智能对话/span span idconnStatus classstatus-badge未连接/span /div div classmessage-history idmessageHistory div classmessage botdiv classbubble✨ 你好我是智能虚拟人助手请问有什么可以帮您/div/div /div div classinput-row input typetext iduserInput placeholder输入您的问题虚拟人会智能回答... button classsend-btn idsendBtn发送/button /div div classlog-area idlogArea/div /div /div /div /div script let sessionId null; let flvPlayer null; let isWaitingReply false; const API_BASE /quality-analysis/api/virtual; function log(msg, err false) { const el document.getElementById(logArea); el.innerHTML \n[${new Date().toLocaleTimeString()}] ${err ? ❌ : ✓} ${msg}; el.scrollTop el.scrollHeight; } function addMessage(text, isUser false) { const div document.createElement(div); div.className message ${isUser ? user : bot}; div.innerHTML div classbubble${escapeHtml(text)}/div; document.getElementById(messageHistory).appendChild(div); div.scrollIntoView({ behavior: smooth }); } function addThinking() { const div document.createElement(div); div.className message bot; div.id thinkingMsg; div.innerHTML div classthinking 虚拟人正在思考.../div; document.getElementById(messageHistory).appendChild(div); div.scrollIntoView({ behavior: smooth }); } function removeThinking() { const thinking document.getElementById(thinkingMsg); if (thinking) thinking.remove(); } function escapeHtml(str) { if (!str) return ; return str.replace(/[]/g, function(m) { if (m ) return amp;; if (m ) return lt;; if (m ) return gt;; return m; }); } function playFlv(flvUrl) { const videoPlayer document.getElementById(videoPlayer); if (flvPlayer) { flvPlayer.destroy(); flvPlayer null; } if (typeof flvjs undefined) { log(flv.js 未加载, true); return; } if (!flvjs.isSupported()) { log(当前浏览器不支持 flv.js, true); return; } try { flvPlayer flvjs.createPlayer({ type: flv, url: flvUrl, isLive: true, enableWorker: false, enableStashBuffer: false }); flvPlayer.attachMediaElement(videoPlayer); flvPlayer.load(); flvPlayer.play().catch(e log(播放失败: ${e.message}, true)); log(flv.js 播放器已启动); } catch (e) { log(创建播放器失败: ${e.message}, true); } } async function start() { log(正在启动虚拟人...); try { const res await fetch(${API_BASE}/start, { method: POST, headers: {Content-Type: application/json}, body: JSON.stringify({}) }); const data await res.json(); if (data.success) { sessionId data.sessionId; const flvUrl data.streamUrl; log(启动成功Session: ${sessionId}); document.getElementById(connStatus).innerText 已连接; document.getElementById(connStatus).classList.add(connected); document.getElementById(startBtn).disabled true; document.getElementById(stopBtn).disabled false; if (flvUrl) playFlv(flvUrl); // 发送欢迎语 setTimeout(() { sendAndGetReply(简单说一下你是谁30个字以内。); }, 2000); } else { log(启动失败: data.error, true); } } catch (e) { log(请求失败: e.message, true); } } // ✅ 核心发送文本并获取虚拟人的智能回复 async function sendAndGetReply(text) { if (!sessionId) { log(未连接, true); return; } if (isWaitingReply) { log(等待回复中请稍后再试, true); return; } isWaitingReply true; addThinking(); try { log(发送: ${text}); const response await fetch(${API_BASE}/talk, { method: POST, headers: {Content-Type: application/json}, body: JSON.stringify({ sessionId, text }) }); const data await response.json(); removeThinking(); if (data.success) { log(发送成功); if (data.reply data.reply.trim()) { addMessage(data.reply, false); log(虚拟人回复: ${data.reply}); } else { addMessage(抱歉我没有理解您的问题, false); log(虚拟人没有返回回复内容, true); } } else { log(发送失败: ${data.error}, true); addMessage(发送失败: ${data.error}, false); } } catch (error) { removeThinking(); log(请求失败: ${error.message}, true); addMessage(请求失败: ${error.message}, false); } finally { isWaitingReply false; } } async function stop() { if (!sessionId) return; if (flvPlayer) { flvPlayer.destroy(); flvPlayer null; } await fetch(${API_BASE}/stop, { method: POST, headers: {Content-Type: application/json}, body: JSON.stringify({sessionId}) }); sessionId null; document.getElementById(connStatus).innerText 未连接; document.getElementById(connStatus).classList.remove(connected); document.getElementById(startBtn).disabled false; document.getElementById(stopBtn).disabled true; log(已停止); } function handleUserMessage() { const input document.getElementById(userInput); const msg input.value.trim(); if (!msg) return; if (!sessionId) { log(请先启动虚拟人, true); return; } addMessage(msg, true); input.value ; sendAndGetReply(msg); } document.getElementById(startBtn).onclick start; document.getElementById(stopBtn).onclick stop; document.getElementById(sendBtn).onclick handleUserMessage; document.getElementById(userInput).addEventListener(keydown, e { if (e.key Enter) handleUserMessage(); }); log(页面加载完成智能对话模式已启用); log( 提示虚拟人会根据您的问题智能回答需要在讯飞平台开启大模型对话能力); /script /body /html3.后端开发官方文档 https://www.yuque.com/xnrpt/bbc1du/xamwb751mbpgeg2o简单以三个接口为例分别是启动虚拟人与虚拟人交互和停止虚拟人。对应后端的控制器层代码。RestController RequestMapping(/api/virtual) public class AvatarController { Autowired private AvatarConfigMapper avatarConfigMapper; // 会话管理 private final ConcurrentHashMapString, AvatarSession sessions new ConcurrentHashMap(); /** * 启动虚拟人会话返回流地址 * POST /avatar/start */ PostMapping(/start) public MapString, Object start() { MapString, Object result new HashMap(); try { //连接信息配置在数据库表中使用的时候直接查询即可。 AvatarConfig avatarConfig avatarConfigMapper.selectOne(null); AvatarSession session new AvatarSession(avatarConfig); String streamUrl session.start(); sessions.put(session.getSessionId(), session); result.put(streamUrl, streamUrl); result.put(sessionId, session.getSessionId()); result.put(success, true); } catch (Exception e) { result.put(success, false); result.put(message, 启动失败: e.getMessage()); } return result; } /** * 文本交互走语义理解 * POST /avatar/interact */ PostMapping(/talk) public MapString, Object talk(RequestBody MapString, String body) { String sessionId body.get(sessionId); String text body.get(text); AvatarSession session sessions.get(sessionId); if (session null || !session.isAlive()) { return Map.of(success, false, error, 会话不存在或已过期); } try{ String reply session.interact(text); return Map.of(success, true, reply, reply); } catch (Exception e) { return Map.of(success, false, error, e.getMessage()); } } /** * 停止虚拟人会话 * POST /avatar/stop */ PostMapping(/stop) public MapString, Object stop(RequestBody MapString, String body) { MapString, Object result new HashMap(); try{ String sessionId body.get(sessionId); AvatarSession session sessions.remove(sessionId); if (session ! null) { session.close(); } return Map.of(success, true); }catch (Exception e) { return Map.of(success, false, error, e.getMessage()); } } }AvatarSession代码Slf4j public class AvatarSession { private WebSocketClient ws; private AvatarConfig config; private String sessionId; private String streamUrl; private Thread pingThread; private volatile boolean alive false; // 用于接收交互回复 private volatile String lastReply null; private CountDownLatch replyLatch; public AvatarSession(AvatarConfig config) { this.config config; this.sessionId UUID.randomUUID().toString().replace(-, ); } public String getSessionId() { return sessionId; } public String getStreamUrl() { return streamUrl; } public boolean isAlive() { return alive; } /** * 启动虚拟人返回流地址 */ public String start() throws Exception { String authUrl AvatarAuthUtil.buildAuthUrl(config.getApiKey(), config.getApiSecret()); CountDownLatch streamReady new CountDownLatch(1); String[] errorMsg {null}; ws new WebSocketClient(new URI(authUrl)) { Override public void onOpen(ServerHandshake handshake) { log.info([Session-{}] 已连接, sessionId); send(buildStart()); } Override public void onMessage(String message) { JSONObject resp JSON.parseObject(message); JSONObject header resp.getJSONObject(header); int code header.getIntValue(code); if (code ! 0) { errorMsg[0] code code , header.getString(message); log.error([Session-{}] 错误: {}, sessionId, errorMsg[0]); streamReady.countDown(); if (replyLatch ! null) replyLatch.countDown(); return; } JSONObject payload resp.getJSONObject(payload); if (payload null) return; // 流地址 JSONObject avatar payload.getJSONObject(avatar); if (avatar ! null stream_info.equals(avatar.getString(event_type))) { streamUrl avatar.getString(stream_url); log.info([Session-{}] 流地址: {}, sessionId, streamUrl); streamReady.countDown(); } // 文本交互回复 JSONObject nlp payload.getJSONObject(nlp); if (nlp ! null) { String answerText null; JSONObject answer nlp.getJSONObject(answer); if (answer ! null) { answerText answer.getString(text); } // tts_answer 是虚拟人实际播报的文本 JSONObject ttsAnswer nlp.getJSONObject(tts_answer); if (ttsAnswer ! null ttsAnswer.getString(text) ! null) { answerText ttsAnswer.getString(text); } int status nlp.getIntValue(status); if (answerText ! null !answerText.isEmpty()) { if (lastReply null) lastReply ; lastReply answerText; } // status2 表示回复结束 if (status 2 replyLatch ! null) { replyLatch.countDown(); } } } Override public void onClose(int code, String reason, boolean remote) { log.info([Session-{}] 关闭: {}, sessionId, reason); alive false; streamReady.countDown(); if (replyLatch ! null) replyLatch.countDown(); } Override public void onError(Exception ex) { log.error([Session-{}] 异常, sessionId, ex); errorMsg[0] ex.getMessage(); alive false; streamReady.countDown(); if (replyLatch ! null) replyLatch.countDown(); } }; ws.connectBlocking(15, TimeUnit.SECONDS); if (!streamReady.await(30, TimeUnit.SECONDS) || streamUrl null) { close(); throw new RuntimeException(启动失败: (errorMsg[0] ! null ? errorMsg[0] : 超时)); } alive true; // 心跳 pingThread new Thread(() - { while (alive ws.isOpen()) { try { ws.send(buildPing()); Thread.sleep(4000); } catch (Exception e) { break; } } }); pingThread.setDaemon(true); pingThread.start(); return streamUrl; } /** * 文本驱动不走语义理解直接播报 */ public void drive(String text) { if (!alive || ws null) return; JSONObject msg new JSONObject(); JSONObject header new JSONObject(); header.put(app_id, config.getAppId()); header.put(ctrl, text_driver); header.put(request_id, uid()); msg.put(header, header); JSONObject parameter new JSONObject(); parameter.put(avatar_dispatch, new JSONObject().fluentPut(interactive_mode, 1)); parameter.put(tts, new JSONObject() .fluentPut(vcn, config.getVcn()) .fluentPut(speed, 50).fluentPut(volume, 50)); parameter.put(air, new JSONObject() .fluentPut(air, 1).fluentPut(add_nonsemantic, 1)); msg.put(parameter, parameter); JSONObject payload new JSONObject(); payload.put(text, new JSONObject().fluentPut(content, text)); msg.put(payload, payload); ws.send(msg.toJSONString()); log.info([Session-{}] 文本驱动: {}, sessionId, text); } /** * 文本交互走语义理解虚拟人会思考后回复 */ public String interact(String text) { if (!alive || ws null) return null; lastReply null; replyLatch new CountDownLatch(1); JSONObject msg new JSONObject(); JSONObject header new JSONObject(); header.put(app_id, config.getAppId()); header.put(ctrl, text_interact); header.put(request_id, uid()); msg.put(header, header); JSONObject parameter new JSONObject(); parameter.put(tts, new JSONObject() .fluentPut(vcn, config.getVcn()) .fluentPut(speed, 50).fluentPut(volume, 50)); parameter.put(air, new JSONObject() .fluentPut(air, 1).fluentPut(add_nonsemantic, 1)); msg.put(parameter, parameter); JSONObject payload new JSONObject(); payload.put(text, new JSONObject().fluentPut(content, text)); msg.put(payload, payload); ws.send(msg.toJSONString()); log.info([Session-{}] 文本交互: {}, sessionId, text); // 等待回复最长30秒 try { replyLatch.await(30, TimeUnit.SECONDS); } catch (InterruptedException ignored) {} return lastReply; } /** * 关闭会话 */ public void close() { alive false; if (ws ! null ws.isOpen()) { try { JSONObject msg new JSONObject(); JSONObject header new JSONObject(); header.put(app_id, config.getAppId()); header.put(ctrl, Stop); header.put(request_id, uid()); msg.put(header, header); ws.send(msg.toJSONString()); Thread.sleep(500); ws.close(); } catch (Exception ignored) {} } log.info([Session-{}] 已关闭, sessionId); } // 协议构建 private String buildStart() { JSONObject msg new JSONObject(); JSONObject header new JSONObject(); header.put(app_id, config.getAppId()); header.put(ctrl, start); header.put(request_id, uid()); header.put(scene_id, config.getSceneId()); msg.put(header, header); JSONObject parameter new JSONObject(); JSONObject avatar new JSONObject(); avatar.put(stream, new JSONObject() .fluentPut(protocol, flv) .fluentPut(fps, 25) .fluentPut(bitrate, 2000)); avatar.put(avatar_id, config.getAvatarId()); avatar.put(width, 1280); avatar.put(height, 720); parameter.put(avatar, avatar); parameter.put(tts, new JSONObject() .fluentPut(vcn, config.getVcn()) .fluentPut(speed, 50).fluentPut(volume, 50)); msg.put(parameter, parameter); msg.put(payload, new JSONObject()); return msg.toJSONString(); } private String buildPing() { JSONObject msg new JSONObject(); JSONObject header new JSONObject(); header.put(app_id, config.getAppId()); header.put(ctrl, ping); header.put(request_id, uid()); msg.put(header, header); return msg.toJSONString(); } private String uid() { return UUID.randomUUID().toString().replace(-, ); } }工具类AvatarAuthUtilpublic class AvatarAuthUtil { public static String buildAuthUrl(String apiKey, String apiSecret) throws Exception { String host avatar.cn-huadong-1.xf-yun.com; String path /v1/interact; SimpleDateFormat sdf new SimpleDateFormat(EEE, dd MMM yyyy HH:mm:ss z, Locale.US); sdf.setTimeZone(TimeZone.getTimeZone(GMT)); String date sdf.format(new Date()); String signatureOrigin host: host \n date: date \n GET path HTTP/1.1; Mac mac Mac.getInstance(HmacSHA256); mac.init(new SecretKeySpec(apiSecret.getBytes(StandardCharsets.UTF_8), HmacSHA256)); String signature Base64.getEncoder().encodeToString( mac.doFinal(signatureOrigin.getBytes(StandardCharsets.UTF_8))); String authorizationOrigin String.format( api_key\%s\, algorithm\hmac-sha256\, headers\host date request-line\, signature\%s\, apiKey, signature); String authorization Base64.getEncoder().encodeToString( authorizationOrigin.getBytes(StandardCharsets.UTF_8)); return wss:// host path ? authorization URLEncoder.encode(authorization, UTF-8) date URLEncoder.encode(date, UTF-8) host URLEncoder.encode(host, UTF-8); } }重要参数header.put(ctrl, start); //启动虚拟人header.put(ctrl, text_driver); //普通文本驱动-播报header.put(ctrl, text_interact); //语音交互需要配合ai服务header.put(ctrl, Stop); //停止释放资源要不然始终占用fluentPut(protocol, flv) //必传视频协议支持rtmpxrtc、webrtc、flv目前只有xrtc支持透明背景需配合alpha参数传1。浏览器预览使用flv.其他参数可以参考官方样例4.讲解1.启动利用AvatarAuthUtil.buildAuthUrl(apiKey, apiSecret)生成带鉴权参数的 WebSocket URL创建streamReady用于等待“流地址准备好”的异步通知创建 WebSocketClient 实例重写onOpen/onMessage/onClose/onError1.onOpen当 WebSocket 连接建立成功时打一条“已连接”的日志发送“start”协议消息给服务端buildStart()构造buildStart()内包含app_id、scene_id、avatar_id、流配置、tts 配置等。这个“start”消息会触发服务端创建虚拟人并下发流信息2.onMessage 1.使用fastjson把字符串解析成 JSON2.从header中取出code判断是否错误3.从payload中分别解析avatar获取流地址stream_urlnlp获取语义理解和 TTS 的回复文本3.onClose / onError不论是正常关闭onClose还是异常onError标记alive false心跳线程会自动结束解除所有正在等待结果的CountDownLatch防止死等记录日志便于排查问题connectBlocking阻塞等待最多 15 秒连接成功再通过streamReady.await等待服务端下发streamUrl最多 30 秒连接或获取流地址失败调用close()并抛出异常成功后设置alive true启动心跳线程返回streamUrl2.交互2.1drive() – 文本驱动不走语义ctrl text_driver这是一种“直接播报”的模式不做复杂语义理解服务端基本只是把 text 转成语音并驱动虚拟人口型表情适合播报固定文案、公告、脚本等场景参数中配置了vcn声音角色speed、volume语速、音量air一些语气词或非语义增强配置2.2interact() – 文本交互走语义ctrl text_interact表示要进行语义交互发送用户输入text给服务端创建新的replyLatch并在onMessage中靠status2时countDown()结束等待等待时间上限 30 秒超时直接返回当前已经累积的lastReply可能为 null 或不完整返回值为服务端整合好的回复文本2.3close() – 关闭会话主动关闭时先将alive false让心跳线程自动停止如果 WebSocket 还开着发送一个ctrl Stop的关闭指令给服务端等待 500ms给服务端一点处理时间然后调用ws.close()断开连接最后记录“已关闭”的日志5.演示至此一个简单的使用大模型交互的虚拟人创建完成。不需要时要点击停止否则会始终消耗资源视频略。6.扩展业务系统引入大模型对话支持提示词和知识库在大模型对话配置里面可以针对业务场景对提示词或者知识库进行配置。配置后需要重新发布才可以生效。需要调整一下语义理解顺序这样才能更准确的提供帮助服务。