Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
J
jinchat-server
概览
概览
详情
活动
周期分析
版本库
存储库
文件
提交
分支
标签
贡献者
分支图
比较
统计图
问题
0
议题
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
CI / CD
CI / CD
流水线
作业
日程表
图表
维基
Wiki
代码片段
代码片段
成员
成员
折叠边栏
关闭边栏
活动
图像
聊天
创建新问题
作业
提交
问题看板
Open sidebar
aigc-pioneer
jinchat-server
Commits
55504fcd
Unverified
提交
55504fcd
authored
5月 10, 2023
作者:
shrimp
提交者:
GitHub
5月 10, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
新增加知识库测试能力 (#302)
上级
466bfb7a
全部展开
隐藏空白字符变更
内嵌
并排
正在显示
5 个修改的文件
包含
13 行增加
和
10 行删除
+13
-10
local_doc_qa.py
chains/local_doc_qa.py
+0
-0
model_config.py
configs/model_config.py
+4
-2
chatglm_llm.py
models/chatglm_llm.py
+3
-3
chinese_text_splitter.py
textsplitter/chinese_text_splitter.py
+6
-5
webui.py
webui.py
+0
-0
没有找到文件。
chains/local_doc_qa.py
浏览文件 @
55504fcd
差异被折叠。
点击展开。
configs/model_config.py
浏览文件 @
55504fcd
...
@@ -69,6 +69,9 @@ LLM_HISTORY_LEN = 3
...
@@ -69,6 +69,9 @@ LLM_HISTORY_LEN = 3
# return top-k text chunk from vector store
# return top-k text chunk from vector store
VECTOR_SEARCH_TOP_K
=
5
VECTOR_SEARCH_TOP_K
=
5
# 如果为0,则不生效,经测试小于500值的结果更精准
VECTOR_SEARCH_SCORE_THRESHOLD
=
0
NLTK_DATA_PATH
=
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
dirname
(
__file__
)),
"nltk_data"
)
NLTK_DATA_PATH
=
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
dirname
(
__file__
)),
"nltk_data"
)
FLAG_USER_NAME
=
uuid
.
uuid4
()
.
hex
FLAG_USER_NAME
=
uuid
.
uuid4
()
.
hex
...
@@ -79,4 +82,4 @@ llm device: {LLM_DEVICE}
...
@@ -79,4 +82,4 @@ llm device: {LLM_DEVICE}
embedding device: {EMBEDDING_DEVICE}
embedding device: {EMBEDDING_DEVICE}
dir: {os.path.dirname(os.path.dirname(__file__))}
dir: {os.path.dirname(os.path.dirname(__file__))}
flagging username: {FLAG_USER_NAME}
flagging username: {FLAG_USER_NAME}
"""
)
"""
)
\ No newline at end of file
models/chatglm_llm.py
浏览文件 @
55504fcd
...
@@ -125,7 +125,7 @@ class ChatGLM(LLM):
...
@@ -125,7 +125,7 @@ class ChatGLM(LLM):
prefix_encoder_file
.
close
()
prefix_encoder_file
.
close
()
model_config
.
pre_seq_len
=
prefix_encoder_config
[
'pre_seq_len'
]
model_config
.
pre_seq_len
=
prefix_encoder_config
[
'pre_seq_len'
]
model_config
.
prefix_projection
=
prefix_encoder_config
[
'prefix_projection'
]
model_config
.
prefix_projection
=
prefix_encoder_config
[
'prefix_projection'
]
except
Exception
as
e
:
except
Exception
as
e
:
logger
.
error
(
f
"加载PrefixEncoder config.json失败: {e}"
)
logger
.
error
(
f
"加载PrefixEncoder config.json失败: {e}"
)
self
.
model
=
AutoModel
.
from_pretrained
(
model_name_or_path
,
config
=
model_config
,
trust_remote_code
=
True
,
self
.
model
=
AutoModel
.
from_pretrained
(
model_name_or_path
,
config
=
model_config
,
trust_remote_code
=
True
,
**
kwargs
)
**
kwargs
)
...
@@ -163,7 +163,7 @@ class ChatGLM(LLM):
...
@@ -163,7 +163,7 @@ class ChatGLM(LLM):
new_prefix_state_dict
[
k
[
len
(
"transformer.prefix_encoder."
):]]
=
v
new_prefix_state_dict
[
k
[
len
(
"transformer.prefix_encoder."
):]]
=
v
self
.
model
.
transformer
.
prefix_encoder
.
load_state_dict
(
new_prefix_state_dict
)
self
.
model
.
transformer
.
prefix_encoder
.
load_state_dict
(
new_prefix_state_dict
)
self
.
model
.
transformer
.
prefix_encoder
.
float
()
self
.
model
.
transformer
.
prefix_encoder
.
float
()
except
Exception
as
e
:
except
Exception
as
e
:
logger
.
error
(
f
"加载PrefixEncoder模型参数失败:{e}"
)
logger
.
error
(
f
"加载PrefixEncoder模型参数失败:{e}"
)
self
.
model
=
self
.
model
.
eval
()
self
.
model
=
self
.
model
.
eval
()
...
@@ -175,7 +175,7 @@ if __name__ == "__main__":
...
@@ -175,7 +175,7 @@ if __name__ == "__main__":
llm_device
=
LLM_DEVICE
,
)
llm_device
=
LLM_DEVICE
,
)
last_print_len
=
0
last_print_len
=
0
for
resp
,
history
in
llm
.
_call
(
"你好"
,
streaming
=
True
):
for
resp
,
history
in
llm
.
_call
(
"你好"
,
streaming
=
True
):
logger
.
info
(
resp
[
last_print_len
:])
logger
.
info
(
resp
[
last_print_len
:]
,
end
=
""
,
flush
=
True
)
last_print_len
=
len
(
resp
)
last_print_len
=
len
(
resp
)
for
resp
,
history
in
llm
.
_call
(
"你好"
,
streaming
=
False
):
for
resp
,
history
in
llm
.
_call
(
"你好"
,
streaming
=
False
):
logger
.
info
(
resp
)
logger
.
info
(
resp
)
...
...
textsplitter/chinese_text_splitter.py
浏览文件 @
55504fcd
...
@@ -5,9 +5,10 @@ from configs.model_config import SENTENCE_SIZE
...
@@ -5,9 +5,10 @@ from configs.model_config import SENTENCE_SIZE
class
ChineseTextSplitter
(
CharacterTextSplitter
):
class
ChineseTextSplitter
(
CharacterTextSplitter
):
def
__init__
(
self
,
pdf
:
bool
=
False
,
**
kwargs
):
def
__init__
(
self
,
pdf
:
bool
=
False
,
sentence_size
:
int
=
None
,
**
kwargs
):
super
()
.
__init__
(
**
kwargs
)
super
()
.
__init__
(
**
kwargs
)
self
.
pdf
=
pdf
self
.
pdf
=
pdf
self
.
sentence_size
=
sentence_size
def
split_text1
(
self
,
text
:
str
)
->
List
[
str
]:
def
split_text1
(
self
,
text
:
str
)
->
List
[
str
]:
if
self
.
pdf
:
if
self
.
pdf
:
...
@@ -23,7 +24,7 @@ class ChineseTextSplitter(CharacterTextSplitter):
...
@@ -23,7 +24,7 @@ class ChineseTextSplitter(CharacterTextSplitter):
sent_list
.
append
(
ele
)
sent_list
.
append
(
ele
)
return
sent_list
return
sent_list
def
split_text
(
self
,
text
:
str
)
->
List
[
str
]:
def
split_text
(
self
,
text
:
str
)
->
List
[
str
]:
##此处需要进一步优化逻辑
if
self
.
pdf
:
if
self
.
pdf
:
text
=
re
.
sub
(
r"\n{3,}"
,
r"\n"
,
text
)
text
=
re
.
sub
(
r"\n{3,}"
,
r"\n"
,
text
)
text
=
re
.
sub
(
'
\
s'
,
" "
,
text
)
text
=
re
.
sub
(
'
\
s'
,
" "
,
text
)
...
@@ -38,15 +39,15 @@ class ChineseTextSplitter(CharacterTextSplitter):
...
@@ -38,15 +39,15 @@ class ChineseTextSplitter(CharacterTextSplitter):
# 很多规则中会考虑分号;,但是这里我把它忽略不计,破折号、英文双引号等同样忽略,需要的再做些简单调整即可。
# 很多规则中会考虑分号;,但是这里我把它忽略不计,破折号、英文双引号等同样忽略,需要的再做些简单调整即可。
ls
=
[
i
for
i
in
text
.
split
(
"
\n
"
)
if
i
]
ls
=
[
i
for
i
in
text
.
split
(
"
\n
"
)
if
i
]
for
ele
in
ls
:
for
ele
in
ls
:
if
len
(
ele
)
>
SENTENCE_SIZE
:
if
len
(
ele
)
>
self
.
sentence_size
:
ele1
=
re
.
sub
(
r'([,,.]["’”」』]{0,2})([^,,.])'
,
r'\1\n\2'
,
ele
)
ele1
=
re
.
sub
(
r'([,,.]["’”」』]{0,2})([^,,.])'
,
r'\1\n\2'
,
ele
)
ele1_ls
=
ele1
.
split
(
"
\n
"
)
ele1_ls
=
ele1
.
split
(
"
\n
"
)
for
ele_ele1
in
ele1_ls
:
for
ele_ele1
in
ele1_ls
:
if
len
(
ele_ele1
)
>
SENTENCE_SIZE
:
if
len
(
ele_ele1
)
>
self
.
sentence_size
:
ele_ele2
=
re
.
sub
(
r'([\n]{1,}| {2,}["’”」』]{0,2})([^\s])'
,
r'\1\n\2'
,
ele_ele1
)
ele_ele2
=
re
.
sub
(
r'([\n]{1,}| {2,}["’”」』]{0,2})([^\s])'
,
r'\1\n\2'
,
ele_ele1
)
ele2_ls
=
ele_ele2
.
split
(
"
\n
"
)
ele2_ls
=
ele_ele2
.
split
(
"
\n
"
)
for
ele_ele2
in
ele2_ls
:
for
ele_ele2
in
ele2_ls
:
if
len
(
ele_ele2
)
>
SENTENCE_SIZE
:
if
len
(
ele_ele2
)
>
self
.
sentence_size
:
ele_ele3
=
re
.
sub
(
'( ["’”」』]{0,2})([^ ])'
,
r'\1\n\2'
,
ele_ele2
)
ele_ele3
=
re
.
sub
(
'( ["’”」』]{0,2})([^ ])'
,
r'\1\n\2'
,
ele_ele2
)
ele2_id
=
ele2_ls
.
index
(
ele_ele2
)
ele2_id
=
ele2_ls
.
index
(
ele_ele2
)
ele2_ls
=
ele2_ls
[:
ele2_id
]
+
[
i
for
i
in
ele_ele3
.
split
(
"
\n
"
)
if
i
]
+
ele2_ls
[
ele2_ls
=
ele2_ls
[:
ele2_id
]
+
[
i
for
i
in
ele_ele3
.
split
(
"
\n
"
)
if
i
]
+
ele2_ls
[
...
...
webui.py
浏览文件 @
55504fcd
差异被折叠。
点击展开。
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论