VllmGPT.py
2.69 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
import json
import requests
# from core import content_db
class VllmGPT:
def __init__(self, host="192.168.1.3",
port="8101",
model="THUDM/chatglm3-6b",
max_tokens="1024"):
self.host = host
self.port = port
self.model=model
self.max_tokens=max_tokens
self.__URL = "http://{}:{}/v1/completions".format(self.host, self.port)
self.__URL2 = "http://{}:{}/v1/chat/completions".format(self.host, self.port)
def chat(self,cont):
chat_list = []
# contentdb = content_db.new_instance()
# list = contentdb.get_list('all','desc',11)
# answer_info = dict()
# chat_list = []
# i = len(list)-1
# while i >= 0:
# answer_info = dict()
# if list[i][0] == "member":
# answer_info["role"] = "user"
# answer_info["content"] = list[i][2]
# elif list[i][0] == "fay":
# answer_info["role"] = "bot"
# answer_info["content"] = list[i][2]
# chat_list.append(answer_info)
# i -= 1
content = {
"model": self.model,
"prompt":"请简单回复我。" + cont,
"history":chat_list}
url = self.__URL
req = json.dumps(content)
headers = {'content-type': 'application/json'}
r = requests.post(url, headers=headers, data=req)
res = json.loads(r.text)
return res['choices'][0]['text']
def question2(self,cont):
chat_list = []
# contentdb = content_db.new_instance()
# list = contentdb.get_list('all','desc',11)
# answer_info = dict()
# chat_list = []
# i = len(list)-1
# while i >= 0:
# answer_info = dict()
# if list[i][0] == "member":
# answer_info["role"] = "user"
# answer_info["content"] = list[i][2]
# elif list[i][0] == "fay":
# answer_info["role"] = "bot"
# answer_info["content"] = list[i][2]
# chat_list.append(answer_info)
# i -= 1
content = {
"model": self.model,
"prompt":"请简单回复我。" + cont,
"history":chat_list}
url = self.__URL2
req = json.dumps(content)
headers = {'content-type': 'application/json'}
r = requests.post(url, headers=headers, data=req)
res = json.loads(r.text)
return res['choices'][0]['message']['content']
if __name__ == "__main__":
vllm = VllmGPT('192.168.1.3','8101')
req = vllm.chat("你叫什么名字啊今年多大了")
print(req)