调整项目结构,适配远程LLM调用生成问题。新增fastchat_openai_llm.py实现fastchat openai报文报文形式调用

configs/model_config.py

@@ -69,11 +69,11 @@ llm_model_dict = {
         "local_model_path": None,
         "provides": "LLamaLLM"
     },
-    "fastChat": {
-        "name": "fastChat",
-        "pretrained_model_name": "fastChat",
+    "fastChatOpenAI": {
+        "name": "FastChatOpenAI",
+        "pretrained_model_name": "FastChatOpenAI",
         "local_model_path": None,
-        "provides": "FastChatLLM"
+        "provides": "FastChatOpenAILLM"
     }
 }
 

models/__init__.py

 from .chatglm_llm import ChatGLM
 from .llama_llm import LLamaLLM
 from .moss_llm import MOSSLLM
-from .fastchat_llm import FastChatLLM
+from .fastchat_openai_llm import FastChatOpenAILLM

models/base/__init__.py

@@ -2,8 +2,12 @@ from models.base.base import (
     AnswerResult,
     BaseAnswer
 )
+from models.base.remote_rpc_model import (
+    RemoteRpcModel
+)
 
 __all__ = [
     "AnswerResult",
     "BaseAnswer",
+    "RemoteRpcModel",
 ]

models/base/lavis_blip2_multimodel.py

+from abc import ABC, abstractmethod
+import torch
+
+from models.base import (BaseAnswer,
+                         AnswerResult)
+
+
+class MultimodalAnswerResult(AnswerResult):
+    image: str = None
+
+
+class LavisBlip2Multimodal(BaseAnswer, ABC):
+
+    @property
+    @abstractmethod
+    def _blip2_instruct(self) -> any:
+        """Return _blip2_instruct of blip2."""
+
+    @property
+    @abstractmethod
+    def _image_blip2_vis_processors(self) -> dict:
+        """Return _image_blip2_vis_processors of blip2 image processors."""
+
+    @abstractmethod
+    def set_image_path(self, image_path: str):
+        """set set_image_path"""

models/base/remote_rpc_model.py

+from abc import ABC, abstractmethod
+import torch
+
+from models.base import (BaseAnswer,
+                         AnswerResult)
+
+
+class MultimodalAnswerResult(AnswerResult):
+    image: str = None
+
+
+class RemoteRpcModel(BaseAnswer, ABC):
+
+    @property
+    @abstractmethod
+    def _api_key(self) -> str:
+        """Return _api_key of client."""
+
+    @property
+    @abstractmethod
+    def _api_base_url(self) -> str:
+        """Return _api_base of client host bash url."""
+
+    @abstractmethod
+    def set_api_key(self, api_key: str):
+        """set set_api_key"""
+
+    @abstractmethod
+    def set_api_base_url(self, api_base_url: str):
+        """set api_base_url"""
+    @abstractmethod
+    def call_model_name(self, model_name):
+        """call model name of client"""

models/fastchat_llm.py

-from abc import ABC
-import requests
-from typing import Optional, List
-from langchain.llms.base import LLM
-
-from models.loader import LoaderCheckPoint
-from models.base import (BaseAnswer,
-                         AnswerResult)
-
-
-class FastChatLLM(BaseAnswer, LLM, ABC):
-    max_token: int = 10000
-    temperature: float = 0.01
-    top_p = 0.9
-    checkPoint: LoaderCheckPoint = None
-    # history = []
-    history_len: int = 10
-
-    def __init__(self, checkPoint: LoaderCheckPoint = None):
-        super().__init__()
-        self.checkPoint = checkPoint
-
-    @property
-    def _llm_type(self) -> str:
-        return "FastChat"
-
-    @property
-    def _check_point(self) -> LoaderCheckPoint:
-        return self.checkPoint
-
-    @property
-    def _history_len(self) -> int:
-        return self.history_len
-
-    def set_history_len(self, history_len: int = 10) -> None:
-        self.history_len = history_len
-
-    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
-        pass
-
-    def generatorAnswer(self, prompt: str,
-                         history: List[List[str]] = [],
-                         streaming: bool = False):
-
-        response = "fastchat 响应结果"
-        history += [[prompt, response]]
-        answer_result = AnswerResult()
-        answer_result.history = history
-        answer_result.llm_output = {"answer": response}
-
-        yield answer_result

models/fastchat_openai_llm.py

+from abc import ABC
+import requests
+from typing import Optional, List
+from langchain.llms.base import LLM
+
+from models.loader import LoaderCheckPoint
+from models.base import (RemoteRpcModel,
+                         AnswerResult)
+from typing import (
+    Collection,
+    Dict
+)
+
+
+def _build_message_template() -> Dict[str, str]:
+    """
+    :return: 结构
+    """
+    return {
+        "role": "",
+        "content": "",
+    }
+
+
+class FastChatOpenAILLM(RemoteRpcModel, LLM, ABC):
+    api_base_url: str = "http://localhost:8000/v1"
+    model_name: str = "chatglm-6b"
+    max_token: int = 10000
+    temperature: float = 0.01
+    top_p = 0.9
+    checkPoint: LoaderCheckPoint = None
+    history = []
+    history_len: int = 10
+
+    def __init__(self, checkPoint: LoaderCheckPoint = None):
+        super().__init__()
+        self.checkPoint = checkPoint
+
+    @property
+    def _llm_type(self) -> str:
+        return "FastChat"
+
+    @property
+    def _check_point(self) -> LoaderCheckPoint:
+        return self.checkPoint
+
+    @property
+    def _history_len(self) -> int:
+        return self.history_len
+
+    def set_history_len(self, history_len: int = 10) -> None:
+        self.history_len = history_len
+
+    @property
+    def _api_key(self) -> str:
+        pass
+
+    @property
+    def _api_base_url(self) -> str:
+        return self.api_base_url
+
+    def set_api_key(self, api_key: str):
+        pass
+
+    def set_api_base_url(self, api_base_url: str):
+        self.api_base_url = api_base_url
+
+    def call_model_name(self, model_name):
+        self.model_name = model_name
+
+    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
+        pass
+
+    # 将历史对话数组转换为文本格式
+    def build_message_list(self, query) -> Collection[Dict[str, str]]:
+        build_message_list: Collection[Dict[str, str]] = []
+        history = self.history[-self.history_len:] if self.history_len > 0 else []
+        for i, (old_query, response) in enumerate(history):
+            user_build_message = _build_message_template()
+            user_build_message['role'] = 'user'
+            user_build_message['content'] = old_query
+            system_build_message = _build_message_template()
+            system_build_message['role'] = 'system'
+            system_build_message['content'] = response
+            build_message_list.append(user_build_message)
+            build_message_list.append(system_build_message)
+
+        user_build_message = _build_message_template()
+        user_build_message['role'] = 'user'
+        user_build_message['content'] = query
+        build_message_list.append(user_build_message)
+        return build_message_list
+
+    def generatorAnswer(self, prompt: str,
+                        history: List[List[str]] = [],
+                        streaming: bool = False):
+
+        try:
+            import openai
+            # Not support yet
+            openai.api_key = "EMPTY"
+            openai.api_base = self.api_base_url
+        except ImportError:
+            raise ValueError(
+                "Could not import openai python package. "
+                "Please install it with `pip install openai`."
+            )
+        # create a chat completion
+        completion = openai.ChatCompletion.create(
+            model=self.model_name,
+            messages=self.build_message_list(prompt)
+        )
+
+        self.history += [[prompt, completion.choices[0].message.content]]
+        answer_result = AnswerResult()
+        answer_result.history = self.history
+        answer_result.llm_output = {"answer": completion.choices[0].message.content}
+
+        yield answer_result

models/shared.py

 import sys
-
+from typing import Any
 from models.loader.args import parser
 from models.loader import LoaderCheckPoint
 from configs.model_config import (llm_model_dict, LLM_MODEL)
@@ -8,7 +8,7 @@ from models.base import BaseAnswer
 loaderCheckPoint: LoaderCheckPoint = None
 
 
-def loaderLLM(llm_model: str = None, no_remote_model: bool = False, use_ptuning_v2: bool = False) -> BaseAnswer:
+def loaderLLM(llm_model: str = None, no_remote_model: bool = False, use_ptuning_v2: bool = False) -> Any:
     """
     init llm_model_ins LLM
     :param llm_model: model_name
@@ -34,7 +34,7 @@ def loaderLLM(llm_model: str = None, no_remote_model: bool = False, use_ptuning_
 
     loaderCheckPoint.model_path = llm_model_info["local_model_path"]
 
-    if 'fastChat' in loaderCheckPoint.model_name:
+    if 'FastChat' in loaderCheckPoint.model_name:
         loaderCheckPoint.unload_model()
     else:
         loaderCheckPoint.reload_model()

test/models/test_fastchat_openai_llm.py

+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../../')
+import asyncio
+from argparse import Namespace
+from models.loader.args import parser
+from models.loader import LoaderCheckPoint
+
+
+import models.shared as shared
+
+
+
+async def dispatch(args: Namespace):
+    args_dict = vars(args)
+
+    shared.loaderCheckPoint = LoaderCheckPoint(args_dict)
+
+    llm_model_ins = shared.loaderLLM()
+    llm_model_ins.set_api_base_url("http://localhost:8000/v1")
+    llm_model_ins.call_model_name("chatglm-6b")
+    history = [
+        ("which city is this?", "tokyo"),
+        ("why?", "she's japanese"),
+
+    ]
+    for answer_result in llm_model_ins.generatorAnswer(prompt="她在做什么? ", history=history,
+                                                       streaming=False):
+        resp = answer_result.llm_output["answer"]
+
+        print(resp)
+
+if __name__ == '__main__':
+    args = None
+    args = parser.parse_args(args=['--model-dir', '/media/checkpoint/',  '--model', 'fastChatOpenAI', '--no-remote-model'])
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(dispatch(args))

models/__main__.py → test/models/test_vicuna_chain_agent.py

 import sys
 import os
 
-sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../')
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../../')
 import asyncio
 from argparse import Namespace
 from models.loader.args import parser
 from models.loader import LoaderCheckPoint
 
-from langchain.agents import initialize_agent, Tool
-from langchain.agents import AgentType
 
 import models.shared as shared