冯杨 / liveTalking

Go to a project
Toggle navigation Toggle navigation pinning
冯杨
76b1a0d1 1 parent d5b5e867

对接本地Fay控制器的时候，进行修改调试

Inline Side-by-side
Showing 3 changed files with 134 additions and 66 deletions
... ... @@ -45,6 +45,8 @@ import asyncio
import torch
from typing import Dict
from logger import logger
from pydub import AudioSegment
from io import BytesIO
app = Flask(__name__)
... ... @@ -156,8 +158,10 @@ async def humanaudio(request):
try:
params = await request.json()
sessionid = int(params.get('sessionid', 0))
fileobj = params['file_url']
if fileobj.startswith("http"):
fileobj = params.get('file_url')
# 获取音频文件数据
if isinstance(fileobj, str) and fileobj.startswith("http"):
async with aiohttp.ClientSession() as session:
async with session.get(fileobj) as response:
if response.status == 200:
... ... @@ -167,10 +171,19 @@ async def humanaudio(request):
content_type="application/json",
text=json.dumps({"code": -1, "msg": "Error downloading file"})
)
# 根据 URL 后缀判断是否为 MP3 文件
is_mp3 = fileobj.lower().endswith('.mp3')
else:
filename = fileobj.filename
filebytes = fileobj.file.read()
is_mp3 = filename.lower().endswith('.mp3')
if is_mp3:
audio = AudioSegment.from_file(BytesIO(filebytes), format="mp3")
out_io = BytesIO()
audio.export(out_io, format="wav")
filebytes = out_io.getvalue()
nerfreals[sessionid].put_audio_file(filebytes)
return web.Response(
... ...
... ... @@ -8,15 +8,18 @@ def llm_response(message,nerfreal:BaseReal):
from openai import OpenAI
client = OpenAI(
# 如果您没有配置环境变量，请在此处用您的API Key进行替换
api_key=os.getenv("DASHSCOPE_API_KEY"),
# api_key=os.getenv("DASHSCOPE_API_KEY"),
api_key = "localkey",
base_url="http://127.0.0.1:5000/v1"
# 填写DashScope SDK的base_url
base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
# base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
)
end = time.perf_counter()
logger.info(f"llm Time init: {end-start}s")
completion = client.chat.completions.create(
model="qwen-plus",
messages=[{'role': 'system', 'content': 'You are a helpful assistant.'},
model="fay-streaming",
# model="qwen-plus",
messages=[{'role': 'system', 'content': '你是小艺，是由艺云展陈开发的AI语音聊天机器人,回答风格精简。'},
{'role': 'user', 'content': message}],
stream=True,
# 通过以下设置，在流式输出的最后一行展示token使用信息
... ...
... ... @@ -267,6 +267,7 @@
<!-- 主内容区域 -->
<div id="main-content">
<input type="hidden" id="username" value="User">
<div id="media">
<h2>艺云展陈</h2>
<audio id="audio" autoplay="true"></audio>
... ... @@ -278,10 +279,7 @@
<script type="text/javascript" src="http://cdn.sockjs.org/sockjs-0.3.4.js"></script>
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script type="text/javascript" charset="utf-8">
var ws;
var reconnectInterval = 5000;
var maxReconnectAttempts = 10;
var reconnectAttempts = 0;
$(document).ready(function() {
// 侧边栏切换功能
$('#sidebar-toggle').click(function() {
... ... @@ -389,69 +387,123 @@
// WebSocket connection to Fay digital avatar (port 10002)
var ws;
var reconnectInterval = 5000;
var maxReconnectAttempts = 10;
var reconnectInterval = 5000; // 每5秒尝试重连一次
var maxReconnectAttempts = 10; // 最大重连次数
var reconnectAttempts = 0;
function generateUsername() {
var username = 'User' + Math.floor(Math.random() * 10000);
return username;
}
function setUsername() {
var storedUsername = localStorage.getItem('username');
if (!storedUsername) {
storedUsername = generateUsername();
localStorage.setItem('username', storedUsername);
}
$('#username').val(storedUsername); // Use the username as the session ID
}
setUsername();
function connectWebSocket() {
var host = window.location.hostname;
ws = new WebSocket("ws://" + host + ":10002");
ws.onopen = function() {
console.log('Connected to WebSocket on port 10002');
reconnectAttempts = 0;
var loginMessage = JSON.stringify({ "Username": "User" });
ws.send(loginMessage);
loginMessage = JSON.stringify({ "Output": "1" });
ws.send(loginMessage);
};
ws.onmessage = function(e) {
var messageData = JSON.parse(e.data);
if (messageData.Data && messageData.Data.Key) {
if(messageData.Data.Key == "audio"){
var value = messageData.Data.HttpValue;
fetch('/humanaudio', {
body: JSON.stringify({
file_url: value,
sessionid:parseInt(document.getElementById('sessionid').value),
}),
headers: {
'Content-Type': 'application/json'
},
method: 'POST'
});
}
}
};
ws.onclose = function(e) {
console.log('WebSocket connection closed');
attemptReconnect();
};
ws.onerror = function(e) {
console.error('WebSocket error:', e);
ws.close();
};
var host = window.location.hostname;
ws = new WebSocket("ws://127.0.0.1:10002");
ws.onopen = function() {
console.log('Connected to WebSocket on port 10002');
reconnectAttempts = 0; // 重置重连次数
// 在连接成功后发送 {"Username": "User"}
var loginMessage = JSON.stringify({ "Username": $('#username').val() });
ws.send(loginMessage);
loginMessage = JSON.stringify({ "Output": "1" });
ws.send(loginMessage);
};
function addMessage(text, type = "right") {
const chatOverlay = document.getElementById("chatOverlay");
const messageDiv = document.createElement("div");
messageDiv.classList.add("message", type);
const avatar = document.createElement("img");
avatar.classList.add("avatar");
avatar.src = type === "right" ? "images/avatar-right.png" : "images/avatar-left.png";
const textContainer = document.createElement("div");
textContainer.classList.add("text-container");
const textElement = document.createElement("div");
textElement.classList.add("text");
textElement.textContent = text;
const timeElement = document.createElement("div");
timeElement.classList.add("time");
timeElement.textContent = new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
textContainer.appendChild(textElement);
textContainer.appendChild(timeElement);
messageDiv.appendChild(avatar);
messageDiv.appendChild(textContainer);
chatOverlay.appendChild(messageDiv);
// 自动滚动到底部
chatOverlay.scrollTop = chatOverlay.scrollHeight;
}
ws.onmessage = function(e) {
var messageData = JSON.parse(e.data);
if (messageData.Data && messageData.Data.Key) {
if(messageData.Data.Key == "audio"){
var value = messageData.Data.HttpValue;
console.log('Value:', value);
fetch('/humanaudio', {
body: JSON.stringify({
file_url: value,
sessionid:parseInt(document.getElementById('sessionid').value),
}),
headers: {
'Content-Type': 'application/json'
},
method: 'POST'
});
}else if (messageData.Data.Key == "text") {
var reply = messageData.Data.Value;
addMessage(reply, "left");
}
}
};
ws.onclose = function(e) {
console.log('WebSocket connection closed');
attemptReconnect();
};
ws.onerror = function(e) {
console.error('WebSocket error:', e);
ws.close(); // 关闭连接并尝试重连
};
}
function attemptReconnect() {
if (reconnectAttempts < maxReconnectAttempts) {
reconnectAttempts++;
console.log('Attempting to reconnect... (Attempt ' + reconnectAttempts + ')');
setTimeout(function() {
connectWebSocket();
}, reconnectInterval);
} else {
console.error('Maximum reconnection attempts reached. Could not reconnect to WebSocket.');
}
if (reconnectAttempts < maxReconnectAttempts) {
reconnectAttempts++;
console.log('Attempting to reconnect... (Attempt ' + reconnectAttempts + ')');
if(parseInt(document.getElementById('is_open').value) == 1){
stop()
}
setTimeout(function() {
connectWebSocket();
}, reconnectInterval);
} else {
console.error('Maximum reconnection attempts reached. Could not reconnect to WebSocket.');
}
}
// Initialize WebSocket connection when document is ready
// 初始化 WebSocket 连接
connectWebSocket();
});
</script>
... ...