websocket断链重连，不限制连接次数；tts合成语音，角色声音更换一种；socket，csn源路径修改;音频处理控件引入，是搭配Fay控制器传入音频的配置。

冯杨
Commit e8e0e9ee674fa3c5075583a55126fffe12fb5c69 e8e0e9ee 1 parent 77ead9ba
Showing 6 changed files with 222 additions and 49 deletions
app.py
ttsreal.py
web/dashboard.html
web/webrtcapi-custom.html
web/webrtcapi.html
web/webrtcchat.html
--- a/app.py
View file @e8e0e9e
+++ b/app.py
View file @e8e0e9e
@@ -45,8 +45,6 @@ import asyncio
 import torch
 from typing import Dict
 from logger import logger
-from pydub import AudioSegment  
-from io import BytesIO
 app = Flask(__name__)
@@ -55,7 +53,7 @@ nerfreals:Dict[int, BaseReal] = {} #sessionid:BaseReal
 opt = None
 model = None
 avatar = None
-        
+
 #####webrtc###############################
 pcs = set()
@@ -95,7 +93,7 @@ async def offer(request):
     nerfreals[sessionid] = None
     nerfreal = await asyncio.get_event_loop().run_in_executor(None, build_nerfreal,sessionid)
     nerfreals[sessionid] = nerfreal
-    
+
     pc = RTCPeerConnection()
     pcs.add(pc)
@@ -144,7 +142,7 @@ async def human(request):
     if params['type']=='echo':
         nerfreals[sessionid].put_msg_txt(params['text'])
     elif params['type']=='chat':
-        res=await asyncio.get_event_loop().run_in_executor(None, llm_response, params['text'],nerfreals[sessionid])                         
+        res=await asyncio.get_event_loop().run_in_executor(None, llm_response, params['text'],nerfreals[sessionid])
         #nerfreals[sessionid].put_msg_txt(res)
     return web.Response(
@@ -153,13 +151,14 @@ async def human(request):
             {"code": 0, "data":"ok"}
         ),
     )
-
+from pydub import AudioSegment
+from io import BytesIO
 async def humanaudio(request):
     try:
         params = await request.json()
         sessionid = int(params.get('sessionid', 0))
         fileobj = params.get('file_url')
-        
+
         # 获取音频文件数据
         if isinstance(fileobj, str) and fileobj.startswith("http"):
             async with aiohttp.ClientSession() as session:
@@ -177,20 +176,20 @@ async def humanaudio(request):
             filename = fileobj.filename
             filebytes = fileobj.file.read()
             is_mp3 = filename.lower().endswith('.mp3')
-        
+
         if is_mp3:
             audio = AudioSegment.from_file(BytesIO(filebytes), format="mp3")
             out_io = BytesIO()
             audio.export(out_io, format="wav")
             filebytes = out_io.getvalue()
-        
+
         nerfreals[sessionid].put_audio_file(filebytes)
         return web.Response(
             content_type="application/json",
             text=json.dumps({"code": 0, "msg": "ok"})
         )
-        
+
     except Exception as e:
         return web.Response(
             content_type="application/json",
@@ -200,7 +199,7 @@ async def humanaudio(request):
 async def set_audiotype(request):
     params = await request.json()
-    sessionid = params.get('sessionid',0)    
+    sessionid = params.get('sessionid',0)
     nerfreals[sessionid].set_custom_state(params['audiotype'],params['reinit'])
     return web.Response(
@@ -275,7 +274,7 @@ async def run(push_url,sessionid):
     await pc.setRemoteDescription(RTCSessionDescription(sdp=answer,type='answer'))
 ##########################################
 # os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
-# os.environ['MULTIPROCESSING_METHOD'] = 'forkserver'                                                    
+# os.environ['MULTIPROCESSING_METHOD'] = 'forkserver'
 if __name__ == '__main__':
     mp.set_start_method('spawn')
     parser = argparse.ArgumentParser()
@@ -291,7 +290,7 @@ if __name__ == '__main__':
     ### training options
     parser.add_argument('--ckpt', type=str, default='data/pretrained/ngp_kf.pth')
-   
+
     parser.add_argument('--num_rays', type=int, default=4096 * 16, help="num rays sampled per image for each training step")
     parser.add_argument('--cuda_ray', action='store_true', help="use CUDA raymarching instead of pytorch")
     parser.add_argument('--max_steps', type=int, default=16, help="max num steps sampled per ray (only valid when using --cuda_ray)")
@@ -309,7 +308,7 @@ if __name__ == '__main__':
     ### network backbone options
     parser.add_argument('--fp16', action='store_true', help="use amp mixed precision training")
-    
+
     parser.add_argument('--bg_img', type=str, default='white', help="background image")
     parser.add_argument('--fbg', action='store_true', help="frame-wise bg")
     parser.add_argument('--exp_eye', action='store_true', help="explicitly control the eyes")
@@ -423,11 +422,11 @@ if __name__ == '__main__':
         with open(opt.customvideo_config,'r') as file:
             opt.customopt = json.load(file)
-    if opt.model == 'ernerf':       
+    if opt.model == 'ernerf':
         from nerfreal import NeRFReal,load_model,load_avatar
         model = load_model(opt)
-        avatar = load_avatar(opt) 
-        
+        avatar = load_avatar(opt)
+
         # we still need test_loader to provide audio features for testing.
         # for k in range(opt.max_session):
         #     opt.sessionid=k
@@ -437,8 +436,8 @@ if __name__ == '__main__':
         from musereal import MuseReal,load_model,load_avatar,warm_up
         logger.info(opt)
         model = load_model()
-        avatar = load_avatar(opt.avatar_id) 
-        warm_up(opt.batch_size,model)      
+        avatar = load_avatar(opt.avatar_id)
+        warm_up(opt.batch_size,model)
         # for k in range(opt.max_session):
         #     opt.sessionid=k
         #     nerfreal = MuseReal(opt,audio_processor,vae, unet, pe,timesteps)
@@ -496,7 +495,6 @@ if __name__ == '__main__':
         pagename='rtcpushapi.html'
     logger.info('start http server; http://<serverip>:'+str(opt.listenport)+'/'+pagename)
     logger.info('如果使用webrtc，推荐访问webrtc集成前端: http://<serverip>:'+str(opt.listenport)+'/dashboard.html')
-
     def run_server(runner):
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
--- a/ttsreal.py
View file @e8e0e9e
+++ b/ttsreal.py
View file @e8e0e9e
@@ -90,7 +90,7 @@ class BaseTTS:
 ###########################################################################################
 class EdgeTTS(BaseTTS):
     def txt_to_audio(self,msg):
-        voicename = "zh-CN-YunxiaNeural"
+        voicename = "zh-CN-XiaoxiaoNeural"
         text,textevent = msg
         t = time.time()
         asyncio.new_event_loop().run_until_complete(self.__main(voicename,text))
--- a/web/dashboard.html
View file @e8e0e9e
+++ b/web/dashboard.html
View file @e8e0e9e
@@ -284,6 +284,7 @@
                 <h1 class="text-center mb-4">livetalking数字人交互平台</h1>
             </div>
         </div>
+        <input type="hidden" id="username" value="User">
         <div class="row">
             <!-- 视频区域 -->
@@ -450,6 +451,154 @@
                 }
             }
+            var ws;
+            var reconnectInterval = 5000; // 初始重连间隔为5秒
+            var reconnectAttempts = 0;
+            var maxReconnectInterval = 60000; // 最大重连间隔为60秒
+            var isReconnecting = false; // 标记是否正在重连中
+
+            function generateUsername() {
+                var username = 'User';
+                // + Math.floor(Math.random() * 10000)
+                return username;
+            }
+
+            function setUsername() {
+                var storedUsername = localStorage.getItem('username');
+                if (!storedUsername) {
+                    storedUsername = generateUsername();
+                    localStorage.setItem('username', storedUsername);
+                }
+                $('#username').val(storedUsername); // Use the username as the session ID
+            }
+
+            setUsername();
+            function connectWebSocket() {
+                var host = window.location.hostname;
+                ws = new WebSocket("ws://127.0.0.1:10002");
+                ws.onopen = function() {
+                    console.log('Connected to WebSocket on port 10002');
+                    reconnectAttempts = 0; // 重置重连次数
+                    reconnectInterval = 5000; // 重置重连间隔
+                    updateConnectionStatus('connected');
+
+                    // 在连接成功后发送 {"Username": "User"}
+                    var loginMessage = JSON.stringify({ "Username": $('#username').val() });
+                    ws.send(loginMessage);
+                    loginMessage = JSON.stringify({ "Output": "1" });
+                    ws.send(loginMessage);
+                };
+
+
+                function addMessage(text, type = "right") {
+                    const chatOverlay = document.getElementById("chatOverlay");
+                    const messageDiv = document.createElement("div");
+                    messageDiv.classList.add("message", type);
+
+                    const avatar = document.createElement("img");
+                    avatar.classList.add("avatar");
+                    avatar.src = type === "right" ? "images/avatar-right.png" : "images/avatar-left.png";
+
+                    const textContainer = document.createElement("div");
+                    textContainer.classList.add("text-container");
+
+                    const textElement = document.createElement("div");
+                    textElement.classList.add("text");
+                    textElement.textContent = text;
+
+                    const timeElement = document.createElement("div");
+                    timeElement.classList.add("time");
+                    timeElement.textContent = new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
+
+                    textContainer.appendChild(textElement);
+                    textContainer.appendChild(timeElement);
+
+                    messageDiv.appendChild(avatar);
+                    messageDiv.appendChild(textContainer);
+
+                    chatOverlay.appendChild(messageDiv);
+
+                    // 自动滚动到底部
+                    chatOverlay.scrollTop = chatOverlay.scrollHeight;
+                }
+
+                ws.onmessage = function(e) {
+                    console.log('WebSocket原始消息(dashboard.html):', e.data);
+                    var messageData = JSON.parse(e.data);
+
+                    if (messageData.Data && messageData.Data.Key) {
+                        if(messageData.Data.Key == "audio"){
+                            var value = messageData.Data.HttpValue;
+                            console.log('Value:', value);
+                            fetch('/humanaudio', {
+                                body: JSON.stringify({
+                                    file_url: value,
+                                    sessionid:parseInt(document.getElementById('sessionid').value),
+                                }),
+                                headers: {
+                                    'Content-Type': 'application/json'
+                                },
+                                method: 'POST'
+                            });
+                        }else if (messageData.Data.Key == "text") {
+                            var reply = messageData.Data.Value;
+                            addMessage(reply, "left");
+                        }
+                    }
+
+                };
+
+                ws.onclose = function(e) {
+                    console.log('WebSocket connection closed');
+                    attemptReconnect();
+                };
+
+                ws.onerror = function(e) {
+                    console.error('WebSocket error:', e);
+                    ws.close(); // 关闭连接并尝试重连
+                };
+            }
+
+            function attemptReconnect() {
+                if (isReconnecting) return; // 防止多次重连
+                
+                isReconnecting = true;
+                reconnectAttempts++;
+                
+                // 使用指数退避算法计算下一次重连间隔
+                var currentInterval = Math.min(reconnectInterval * Math.pow(1.5, reconnectAttempts - 1), maxReconnectInterval);
+                
+                console.log('Attempting to reconnect... (Attempt ' + reconnectAttempts + ', 间隔: ' + currentInterval/1000 + '秒)');
+                updateConnectionStatus('connecting');
+                
+                if(document.getElementById('is_open') && parseInt(document.getElementById('is_open').value) == 1){
+                    stop()
+                }
+                
+                setTimeout(function() {
+                    isReconnecting = false;
+                    connectWebSocket();
+                }, currentInterval);
+            }
+
+            // 初始化 WebSocket 连接
+            connectWebSocket();
+            
+            // 添加页面可见性变化监听，当页面从隐藏变为可见时尝试重连
+            document.addEventListener('visibilitychange', function() {
+                if (document.visibilityState === 'visible') {
+                    // 页面变为可见状态，检查WebSocket连接
+                    if (!ws || ws.readyState === WebSocket.CLOSED || ws.readyState === WebSocket.CLOSING) {
+                        console.log('页面可见，检测到WebSocket未连接，尝试重连...');
+                        updateConnectionStatus('connecting');
+                        // 重置重连计数和间隔，立即尝试重连
+                        reconnectAttempts = 0;
+                        reconnectInterval = 5000;
+                        connectWebSocket();
+                    }
+                }
+            });
+
             // 添加聊天消息
             function addChatMessage(message, type = 'user') {
                 const messagesContainer = $('#chat-messages');
@@ -578,9 +727,9 @@
                 e.preventDefault();
                 var message = $('#chat-message').val();
                 if (!message.trim()) return;
-                
+
                 console.log('Sending chat message:', message);
-                
+
                 // fetch('/human', {
                 //     body: JSON.stringify({
                 //         text: message,
@@ -593,14 +742,14 @@
                 //     },
                 //     method: 'POST'
                 // });
-                
+
                 var payload = {
                     "model": "fay-streaming",
                     "messages": [
-                    {
-                        "role": $('#username').val(),
-                        "content": message
-                    }
+                        {
+                            "role": $('#username').val(),
+                            "content": message
+                        }
                     ],
                     "observation": ""
                 };
@@ -608,14 +757,14 @@
                 fetch('http://127.0.0.1:5000/v1/chat/completions', {
                     body: JSON.stringify(payload),
                     headers: {
-                    'Content-Type': 'application/json'
+                        'Content-Type': 'application/json'
                     },
                     method: 'POST'
                 }).then(function(response) {
                     if (response.ok) {
-                    console.log('Message sent to API.');
+                        console.log('Message sent to API.');
                     } else {
-                    console.error('Failed to send message to API.');
+                        console.error('Failed to send message to API.');
                     }
                 }).catch(function(error) {
                     console.error('Error:', error);
--- a/web/webrtcapi-custom.html
View file @e8e0e9e
+++ b/web/webrtcapi-custom.html
View file @e8e0e9e
@@ -50,7 +50,7 @@
 <input type="text" id="audiotype" value="0">
 <script src="client.js"></script>
-<script type="text/javascript" src="http://cdn.sockjs.org/sockjs-0.3.4.js"></script>
+<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/sockjs-client@1.5.1/dist/sockjs.min.js"></script>
 <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
 </body>
 <script type="text/javascript" charset="utf-8">
--- a/web/webrtcapi.html
View file @e8e0e9e
+++ b/web/webrtcapi.html
View file @e8e0e9e
@@ -3,6 +3,8 @@
 <head>
     <meta charset="UTF-8"/>
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <link rel="icon" href="favicon.ico" type="image/x-icon">
+    <link rel="shortcut icon" href="favicon.ico" type="image/x-icon">
     <title>WebRTC 数字人</title>
     <style>
     body {
@@ -276,7 +278,7 @@
 </div>
 <script src="client.js"></script>
-<script type="text/javascript" src="http://cdn.sockjs.org/sockjs-0.3.4.js"></script>
+<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/sockjs-client@1.5.1/dist/sockjs.min.js"></script>
 <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
 <script type="text/javascript" charset="utf-8">
@@ -387,17 +389,20 @@
       // WebSocket connection to Fay digital avatar (port 10002)
       var ws;
-      var reconnectInterval = 5000; // 每5秒尝试重连一次
-      var maxReconnectAttempts = 10; // 最大重连次数
+      var reconnectInterval = 5000; // 初始重连间隔为5秒
       var reconnectAttempts = 0;
+      var maxReconnectInterval = 60000; // 最大重连间隔为60秒
+      var isReconnecting = false; // 标记是否正在重连中
       function generateUsername() {
-        var username = 'User' + Math.floor(Math.random() * 10000);
+        var username = 'User';
+        // + Math.floor(Math.random() * 10000)
         return username;
       }
       function setUsername() {
         var storedUsername = localStorage.getItem('username');
+        // console.log("当前存有的username:"+storedUsername);
         if (!storedUsername) {
           storedUsername = generateUsername();
           localStorage.setItem('username', storedUsername);
@@ -411,7 +416,7 @@
           ws.onopen = function() {
               console.log('Connected to WebSocket on port 10002');
               reconnectAttempts = 0; // 重置重连次数
-
+              reconnectInterval = 5000; // 重置重连间隔
               // 在连接成功后发送 {"Username": "User"}
               var loginMessage = JSON.stringify({ "Username": $('#username').val() });
               ws.send(loginMessage);
@@ -453,6 +458,7 @@
           }
           ws.onmessage = function(e) {
+              console.log('WebSocket原始消息(webrtcapi.html):', e.data);
               var messageData = JSON.parse(e.data);
               if (messageData.Data && messageData.Data.Key) {
@@ -489,22 +495,42 @@
       }
       function attemptReconnect() {
-          if (reconnectAttempts < maxReconnectAttempts) {
-              reconnectAttempts++;
-              console.log('Attempting to reconnect... (Attempt ' + reconnectAttempts + ')');
-              if(parseInt(document.getElementById('is_open').value) == 1){
-                stop()
-              }
-              setTimeout(function() {
-                  connectWebSocket();
-              }, reconnectInterval);
-          } else {
-              console.error('Maximum reconnection attempts reached. Could not reconnect to WebSocket.');
+          if (isReconnecting) return; // 防止多次重连
+          
+          isReconnecting = true;
+          reconnectAttempts++;
+          
+          // 使用指数退避算法计算下一次重连间隔
+          var currentInterval = Math.min(reconnectInterval * Math.pow(1.5, reconnectAttempts - 1), maxReconnectInterval);
+          
+          console.log('Attempting to reconnect... (Attempt ' + reconnectAttempts + ', 间隔: ' + currentInterval/1000 + '秒)');
+          
+          if(document.getElementById('is_open') && parseInt(document.getElementById('is_open').value) == 1){
+            stop()
           }
+          
+          setTimeout(function() {
+              isReconnecting = false;
+              connectWebSocket();
+          }, currentInterval);
       }
       // 初始化 WebSocket 连接
       connectWebSocket();
+      
+      // 添加页面可见性变化监听，当页面从隐藏变为可见时尝试重连
+      document.addEventListener('visibilitychange', function() {
+          if (document.visibilityState === 'visible') {
+              // 页面变为可见状态，检查WebSocket连接
+              if (!ws || ws.readyState === WebSocket.CLOSED || ws.readyState === WebSocket.CLOSING) {
+                  console.log('页面可见，检测到WebSocket未连接，尝试重连...');
+                  // 重置重连计数和间隔，立即尝试重连
+                  reconnectAttempts = 0;
+                  reconnectInterval = 5000;
+                  connectWebSocket();
+              }
+          }
+      });
     });
 </script>
 </body>
--- a/web/webrtcchat.html
View file @e8e0e9e
+++ b/web/webrtcchat.html
View file @e8e0e9e
@@ -51,7 +51,7 @@
 </div>
 <script src="client.js"></script>
-<script type="text/javascript" src="http://cdn.sockjs.org/sockjs-0.3.4.js"></script>
+<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/sockjs-client@1.5.1/dist/sockjs.min.js"></script>
 <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
 </body>
 <script type="text/javascript" charset="utf-8">