冯杨 / liveTalking

Go to a project
Toggle navigation Toggle navigation pinning
冯杨
e8e0e9ee 1 parent 77ead9ba

websocket断链重连，不限制连接次数；tts合成语音，角色声音更换一种；socket，csn源路径修改;音频处理控件引入，是搭配Fay控制器传入音频的配置。

Inline Side-by-side
Showing 6 changed files with 192 additions and 19 deletions
... ... @@ -45,8 +45,6 @@ import asyncio
import torch
from typing import Dict
from logger import logger
from pydub import AudioSegment
from io import BytesIO
app = Flask(__name__)
... ... @@ -153,7 +151,8 @@ async def human(request):
{"code": 0, "data":"ok"}
),
)
from pydub import AudioSegment
from io import BytesIO
async def humanaudio(request):
try:
params = await request.json()
... ... @@ -496,7 +495,6 @@ if __name__ == '__main__':
pagename='rtcpushapi.html'
logger.info('start http server; http://<serverip>:'+str(opt.listenport)+'/'+pagename)
logger.info('如果使用webrtc，推荐访问webrtc集成前端: http://<serverip>:'+str(opt.listenport)+'/dashboard.html')
def run_server(runner):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
... ...
... ... @@ -90,7 +90,7 @@ class BaseTTS:
###########################################################################################
class EdgeTTS(BaseTTS):
def txt_to_audio(self,msg):
voicename = "zh-CN-YunxiaNeural"
voicename = "zh-CN-XiaoxiaoNeural"
text,textevent = msg
t = time.time()
asyncio.new_event_loop().run_until_complete(self.__main(voicename,text))
... ...
... ... @@ -284,6 +284,7 @@
<h1 class="text-center mb-4">livetalking数字人交互平台</h1>
</div>
</div>
<input type="hidden" id="username" value="User">
<div class="row">
<!-- 视频区域 -->
... ... @@ -450,6 +451,154 @@
}
}
var ws;
var reconnectInterval = 5000; // 初始重连间隔为5秒
var reconnectAttempts = 0;
var maxReconnectInterval = 60000; // 最大重连间隔为60秒
var isReconnecting = false; // 标记是否正在重连中
function generateUsername() {
var username = 'User';
// + Math.floor(Math.random() * 10000)
return username;
}
function setUsername() {
var storedUsername = localStorage.getItem('username');
if (!storedUsername) {
storedUsername = generateUsername();
localStorage.setItem('username', storedUsername);
}
$('#username').val(storedUsername); // Use the username as the session ID
}
setUsername();
function connectWebSocket() {
var host = window.location.hostname;
ws = new WebSocket("ws://127.0.0.1:10002");
ws.onopen = function() {
console.log('Connected to WebSocket on port 10002');
reconnectAttempts = 0; // 重置重连次数
reconnectInterval = 5000; // 重置重连间隔
updateConnectionStatus('connected');
// 在连接成功后发送 {"Username": "User"}
var loginMessage = JSON.stringify({ "Username": $('#username').val() });
ws.send(loginMessage);
loginMessage = JSON.stringify({ "Output": "1" });
ws.send(loginMessage);
};
function addMessage(text, type = "right") {
const chatOverlay = document.getElementById("chatOverlay");
const messageDiv = document.createElement("div");
messageDiv.classList.add("message", type);
const avatar = document.createElement("img");
avatar.classList.add("avatar");
avatar.src = type === "right" ? "images/avatar-right.png" : "images/avatar-left.png";
const textContainer = document.createElement("div");
textContainer.classList.add("text-container");
const textElement = document.createElement("div");
textElement.classList.add("text");
textElement.textContent = text;
const timeElement = document.createElement("div");
timeElement.classList.add("time");
timeElement.textContent = new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
textContainer.appendChild(textElement);
textContainer.appendChild(timeElement);
messageDiv.appendChild(avatar);
messageDiv.appendChild(textContainer);
chatOverlay.appendChild(messageDiv);
// 自动滚动到底部
chatOverlay.scrollTop = chatOverlay.scrollHeight;
}
ws.onmessage = function(e) {
console.log('WebSocket原始消息(dashboard.html):', e.data);
var messageData = JSON.parse(e.data);
if (messageData.Data && messageData.Data.Key) {
if(messageData.Data.Key == "audio"){
var value = messageData.Data.HttpValue;
console.log('Value:', value);
fetch('/humanaudio', {
body: JSON.stringify({
file_url: value,
sessionid:parseInt(document.getElementById('sessionid').value),
}),
headers: {
'Content-Type': 'application/json'
},
method: 'POST'
});
}else if (messageData.Data.Key == "text") {
var reply = messageData.Data.Value;
addMessage(reply, "left");
}
}
};
ws.onclose = function(e) {
console.log('WebSocket connection closed');
attemptReconnect();
};
ws.onerror = function(e) {
console.error('WebSocket error:', e);
ws.close(); // 关闭连接并尝试重连
};
}
function attemptReconnect() {
if (isReconnecting) return; // 防止多次重连
isReconnecting = true;
reconnectAttempts++;
// 使用指数退避算法计算下一次重连间隔
var currentInterval = Math.min(reconnectInterval * Math.pow(1.5, reconnectAttempts - 1), maxReconnectInterval);
console.log('Attempting to reconnect... (Attempt ' + reconnectAttempts + ', 间隔: ' + currentInterval/1000 + '秒)');
updateConnectionStatus('connecting');
if(document.getElementById('is_open') && parseInt(document.getElementById('is_open').value) == 1){
stop()
}
setTimeout(function() {
isReconnecting = false;
connectWebSocket();
}, currentInterval);
}
// 初始化 WebSocket 连接
connectWebSocket();
// 添加页面可见性变化监听，当页面从隐藏变为可见时尝试重连
document.addEventListener('visibilitychange', function() {
if (document.visibilityState === 'visible') {
// 页面变为可见状态，检查WebSocket连接
if (!ws || ws.readyState === WebSocket.CLOSED || ws.readyState === WebSocket.CLOSING) {
console.log('页面可见，检测到WebSocket未连接，尝试重连...');
updateConnectionStatus('connecting');
// 重置重连计数和间隔，立即尝试重连
reconnectAttempts = 0;
reconnectInterval = 5000;
connectWebSocket();
}
}
});
// 添加聊天消息
function addChatMessage(message, type = 'user') {
const messagesContainer = $('#chat-messages');
... ...
... ... @@ -50,7 +50,7 @@
<input type="text" id="audiotype" value="0">
<script src="client.js"></script>
<script type="text/javascript" src="http://cdn.sockjs.org/sockjs-0.3.4.js"></script>
<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/sockjs-client@1.5.1/dist/sockjs.min.js"></script>
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
</body>
<script type="text/javascript" charset="utf-8">
... ...
... ... @@ -3,6 +3,8 @@
<head>
<meta charset="UTF-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" href="favicon.ico" type="image/x-icon">
<link rel="shortcut icon" href="favicon.ico" type="image/x-icon">
<title>WebRTC 数字人</title>
<style>
body {
... ... @@ -276,7 +278,7 @@
</div>
<script src="client.js"></script>
<script type="text/javascript" src="http://cdn.sockjs.org/sockjs-0.3.4.js"></script>
<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/sockjs-client@1.5.1/dist/sockjs.min.js"></script>
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script type="text/javascript" charset="utf-8">
... ... @@ -387,17 +389,20 @@
// WebSocket connection to Fay digital avatar (port 10002)
var ws;
var reconnectInterval = 5000; // 每5秒尝试重连一次
var maxReconnectAttempts = 10; // 最大重连次数
var reconnectInterval = 5000; // 初始重连间隔为5秒
var reconnectAttempts = 0;
var maxReconnectInterval = 60000; // 最大重连间隔为60秒
var isReconnecting = false; // 标记是否正在重连中
function generateUsername() {
var username = 'User' + Math.floor(Math.random() * 10000);
var username = 'User';
// + Math.floor(Math.random() * 10000)
return username;
}
function setUsername() {
var storedUsername = localStorage.getItem('username');
// console.log("当前存有的username:"+storedUsername);
if (!storedUsername) {
storedUsername = generateUsername();
localStorage.setItem('username', storedUsername);
... ... @@ -411,7 +416,7 @@
ws.onopen = function() {
console.log('Connected to WebSocket on port 10002');
reconnectAttempts = 0; // 重置重连次数
reconnectInterval = 5000; // 重置重连间隔
// 在连接成功后发送 {"Username": "User"}
var loginMessage = JSON.stringify({ "Username": $('#username').val() });
ws.send(loginMessage);
... ... @@ -453,6 +458,7 @@
}
ws.onmessage = function(e) {
console.log('WebSocket原始消息(webrtcapi.html):', e.data);
var messageData = JSON.parse(e.data);
if (messageData.Data && messageData.Data.Key) {
... ... @@ -489,22 +495,42 @@
}
function attemptReconnect() {
if (reconnectAttempts < maxReconnectAttempts) {
if (isReconnecting) return; // 防止多次重连
isReconnecting = true;
reconnectAttempts++;
console.log('Attempting to reconnect... (Attempt ' + reconnectAttempts + ')');
if(parseInt(document.getElementById('is_open').value) == 1){
// 使用指数退避算法计算下一次重连间隔
var currentInterval = Math.min(reconnectInterval * Math.pow(1.5, reconnectAttempts - 1), maxReconnectInterval);
console.log('Attempting to reconnect... (Attempt ' + reconnectAttempts + ', 间隔: ' + currentInterval/1000 + '秒)');
if(document.getElementById('is_open') && parseInt(document.getElementById('is_open').value) == 1){
stop()
}
setTimeout(function() {
isReconnecting = false;
connectWebSocket();
}, reconnectInterval);
} else {
console.error('Maximum reconnection attempts reached. Could not reconnect to WebSocket.');
}
}, currentInterval);
}
// 初始化 WebSocket 连接
connectWebSocket();
// 添加页面可见性变化监听，当页面从隐藏变为可见时尝试重连
document.addEventListener('visibilitychange', function() {
if (document.visibilityState === 'visible') {
// 页面变为可见状态，检查WebSocket连接
if (!ws || ws.readyState === WebSocket.CLOSED || ws.readyState === WebSocket.CLOSING) {
console.log('页面可见，检测到WebSocket未连接，尝试重连...');
// 重置重连计数和间隔，立即尝试重连
reconnectAttempts = 0;
reconnectInterval = 5000;
connectWebSocket();
}
}
});
});
</script>
</body>
... ...
... ... @@ -51,7 +51,7 @@
</div>
<script src="client.js"></script>
<script type="text/javascript" src="http://cdn.sockjs.org/sockjs-0.3.4.js"></script>
<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/sockjs-client@1.5.1/dist/sockjs.min.js"></script>
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
</body>
<script type="text/javascript" charset="utf-8">
... ...