updates

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,7 +1,29 @@
 FROM python:3.11-slim
 WORKDIR $HOME/app
 COPY . .
+RUN pip3 uninstall ffmpeg
+RUN pip3 uninstall ffmpeg-python
+RUN pip uninstall ffmpeg
+RUN pip uninstall ffmpeg-python
+RUN pip3 install ffmpeg
+RUN pip3 install ffmpeg-python
+RUN pip install ffmpeg
+RUN pip install ffmpeg-python
 RUN pip install -r requirements.txt
 VOLUME /data
 EXPOSE 23333
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+RUN mkdir -p $HOME/app/models
+RUN chmod 777 $HOME/app/models
+ENV MODELS_PATH=$HOME/app/models
+RUN mkdir -p $HOME/app/uploads
+RUN chmod 777 $HOME/app/uploads
+
 CMD ["python", "-m", "apis.chat_api"]

README.md

@@ -10,19 +10,20 @@ app_port: 23333
 ## HF-LLM-API-COLLECTION
 Huggingface LLM Inference API in OpenAI message format.
 
-Project link: https://github.com/ruslanmv/hf-llm-api-collection
-
 ## Features
 
-- Available Models (2024/01/22): [#5](https://github.com/ruslanmv/hf-llm-api-collection/issues/5)
-  - `mistral-7b`, `mixtral-8x7b`, `nous-mixtral-8x7b`
-  - Adaptive prompt templates for different models
+✅ Implemented:
+
+- Support Models
+  - `mixtral-8x7b`, `mistral-7b`
 - Support OpenAI API format
-  - Enable api endpoint via official `openai-python` package
-- Support both stream and no-stream response
-- Support API Key via both HTTP auth header and env varible [#4](https://github.com/ruslanmv/hf-llm-api-collection/issues/4)
+  - Can use api endpoint via official `openai-python` package
+- Stream response
 - Docker deployment
 
+🔨 In progress:
+- [x] Support more models
+
 ## Run API service
 
 ### Run in Command Line
@@ -62,17 +63,14 @@ sudo docker run -p 23333:23333 --env http_proxy="http://<server>:<port>" hf-llm-
 
 ### Using `openai-python`
 
-See: [`examples/chat_with_openai.py`](https://github.com/ruslanmv/hf-llm-api-collection/blob/main/examples/chat_with_openai.py)
+See: [examples/chat_with_openai.py](https://github.com/ruslanmv/hf-llm-api-collection/blob/main/examples/chat_with_openai.py)
 
 ```py
 from openai import OpenAI
 
 # If runnning this service with proxy, you might need to unset `http(s)_proxy`.
 base_url = "http://127.0.0.1:23333"
-# Your own HF_TOKEN
-api_key = "hf_xxxxxxxxxxxxxxxx"
-# use below as non-auth user
-# api_key = "sk-xxx"
+api_key = "sk-xxxxx"
 
 client = OpenAI(base_url=base_url, api_key=api_key)
 response = client.chat.completions.create(
@@ -97,7 +95,7 @@ for chunk in response:
 
 ### Using post requests
 
-See: [`examples/chat_with_post.py`](https://github.com/ruslanmv/hf-llm-api-collection/blob/main/examples/chat_with_post.py)
+See: [examples/chat_with_post.py](https://github.com/ruslanmv/hf-llm-api-collection/blob/main/examples/chat_with_post.py)
 
 
 ```py
@@ -108,11 +106,7 @@ import re
 
 # If runnning this service with proxy, you might need to unset `http(s)_proxy`.
 chat_api = "http://127.0.0.1:23333"
-# Your own HF_TOKEN
-api_key = "hf_xxxxxxxxxxxxxxxx"
-# use below as non-auth user
-# api_key = "sk-xxx"
-
+api_key = "sk-xxxxx"
 requests_headers = {}
 requests_payload = {
     "model": "mixtral-8x7b",

apis/chat_api.py

@@ -1,21 +1,40 @@
 import argparse
-import markdown2
-import os
-import sys
 import uvicorn
+import sys
+import os
+import io
+from transformers import M2M100Tokenizer, M2M100ForConditionalGeneration
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+import time
+import json
+from typing import List
+import torch
+import logging
+import string
+import random
+import base64
+import re
+import requests
+from utils.enver import enver
+import shutil
+import tempfile
+import numpy as np
 
-from pathlib import Path
-from fastapi import FastAPI, Depends
-from fastapi.responses import HTMLResponse
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+
+from fastapi import FastAPI, Response, File, UploadFile, Form
+from fastapi.encoders import jsonable_encoder
+from fastapi.responses import JSONResponse, StreamingResponse
 from pydantic import BaseModel, Field
-from typing import Union
-from sse_starlette.sse import EventSourceResponse, ServerSentEvent
+from sse_starlette.sse import EventSourceResponse
 from utils.logger import logger
 from networks.message_streamer import MessageStreamer
 from messagers.message_composer import MessageComposer
-from mocks.stream_chat_mocker import stream_chat_mock
-
+from googletrans import Translator
+from io import BytesIO
+from gtts import gTTS
+from fastapi.middleware.cors import CORSMiddleware
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 
 class ChatAPIApp:
     def __init__(self):
@@ -27,148 +46,229 @@ class ChatAPIApp:
         )
         self.setup_routes()
 
-    def get_available_models(self):
-        # https://platform.openai.com/docs/api-reference/models/list
-        # ANCHOR[id=available-models]: Available models
-        self.available_models = {
-            "object": "list",
-            "data": [
-                {
-                    "id": "mixtral-8x7b",
-                    "description": "[mistralai/Mixtral-8x7B-Instruct-v0.1]: https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1",
-                    "object": "model",
-                    "created": 1700000000,
-                    "owned_by": "mistralai",
-                },
-                {
-                    "id": "mistral-7b",
-                    "description": "[mistralai/Mistral-7B-Instruct-v0.2]: https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
-                    "object": "model",
-                    "created": 1700000000,
-                    "owned_by": "mistralai",
-                },
-                {
-                    "id": "nous-mixtral-8x7b",
-                    "description": "[NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO]: https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
-                    "object": "model",
-                    "created": 1700000000,
-                    "owned_by": "NousResearch",
-                },
-            ],
-        }
+    def get_available_langs(self):
+        f = open('apis/lang_name.json', "r")
+        self.available_models = json.loads(f.read())
         return self.available_models
 
-    def extract_api_key(
-        credentials: HTTPAuthorizationCredentials = Depends(
-            HTTPBearer(auto_error=False)
-        ),
-    ):
-        api_key = None
-        if credentials:
-            api_key = credentials.credentials
-        else:
-            api_key = os.getenv("HF_TOKEN")
+    class TranslateCompletionsPostItem(BaseModel):
+        from_language: str = Field(
+            default="en",
+            description="(str) `Detect`",
+        )
+        to_language: str = Field(
+            default="fa",
+            description="(str) `en`",
+        )
+        input_text: str = Field(
+            default="Hello",
+            description="(str) `Text for translate`",
+        )
+   
+
+    def translate_completions(self, item: TranslateCompletionsPostItem):
+        translator = Translator()
+        f = open('apis/lang_name.json', "r")
+        available_langs = json.loads(f.read())
+        from_lang = 'en'
+        to_lang = 'en'
+        for lang_item in available_langs:
+          if item.to_language == lang_item['code']:
+              to_lang = item.to_language
+              break
+              
+          
+        translated = translator.translate(item.input_text, dest=to_lang)
+        item_response = {
+            "from_language": translated.src,
+            "to_language": translated.dest,
+            "text": item.input_text,
+            "translate": translated.text
+        }
+        json_compatible_item_data = jsonable_encoder(item_response)
+        return JSONResponse(content=json_compatible_item_data)
+
+    def translate_ai_completions(self, item: TranslateCompletionsPostItem):
+        translator = Translator()
+        #print(os.getcwd())
+        f = open('apis/lang_name.json', "r")
+        available_langs = json.loads(f.read())
+        from_lang = 'en'
+        to_lang = 'en'
+        for lang_item in available_langs:
+          if item.to_language == lang_item['code']:
+              to_lang = item.to_language
+          if item.from_language == lang_item['code']:
+              from_lang = item.from_language
+
+        if to_lang == 'auto':
+            to_lang = 'en'
 
-        if api_key:
-            if api_key.startswith("hf_"):
-                return api_key
-            else:
-                logger.warn(f"Invalid HF Token!")
+        if from_lang == 'auto':
+            from_lang = translator.detect(item.input_text).lang
+            
+        if torch.cuda.is_available():
+            device = torch.device("cuda:0")
         else:
-            logger.warn("Not provide HF Token!")
-        return None
+            device = torch.device("cpu")
+            logging.warning("GPU not found, using CPU, translation will be very slow.")
+
+        time_start = time.time()
+        #TRANSFORMERS_CACHE
+        pretrained_model = "facebook/m2m100_1.2B"
+        cache_dir = "models/"
+        tokenizer = M2M100Tokenizer.from_pretrained(pretrained_model, cache_dir=cache_dir)
+        model = M2M100ForConditionalGeneration.from_pretrained(
+            pretrained_model, cache_dir=cache_dir
+        ).to(device)
+        model.eval()
+
+        tokenizer.src_lang = from_lang
+        with torch.no_grad():
+            encoded_input = tokenizer(item.input_text, return_tensors="pt").to(device)
+            generated_tokens = model.generate(
+               **encoded_input, forced_bos_token_id=tokenizer.get_lang_id(to_lang)
+            )
+            translated_text = tokenizer.batch_decode(
+            generated_tokens, skip_special_tokens=True
+            )[0]
+
+        time_end = time.time()
+        translated = translated_text
+        item_response = {
+            "from_language": from_lang,
+            "to_language": to_lang,
+            "text": item.input_text,
+            "translate": translated,
+            "start": str(time_start),
+            "end": str(time_end)
+        }
+        json_compatible_item_data = jsonable_encoder(item_response)
+        return JSONResponse(content=json_compatible_item_data)
 
-    class ChatCompletionsPostItem(BaseModel):
+    class TranslateAiPostItem(BaseModel):
         model: str = Field(
-            default="mixtral-8x7b",
-            description="(str) `mixtral-8x7b`",
+            default="t5-base",
+            description="(str) `Model Name`",
         )
-        messages: list = Field(
-            default=[{"role": "user", "content": "Hello, who are you?"}],
-            description="(list) Messages",
+        from_language: str = Field(
+            default="en",
+            description="(str) `translate from`",
         )
-        temperature: Union[float, None] = Field(
-            default=0.5,
-            description="(float) Temperature",
+        to_language: str = Field(
+            default="fa",
+            description="(str) `translate to`",
         )
-        top_p: Union[float, None] = Field(
-            default=0.95,
-            description="(float) top p",
-        )
-        max_tokens: Union[int, None] = Field(
-            default=-1,
-            description="(int) Max tokens",
-        )
-        use_cache: bool = Field(
-            default=False,
-            description="(bool) Use cache",
+        input_text: str = Field(
+            default="Hello",
+            description="(str) `Text for translate`",
+        )    
+    def ai_translate(self, item:TranslateAiPostItem):
+        MODEL_MAP = {
+        "t5-base": "t5-base",
+        "t5-small": "t5-small",
+        "t5-large": "t5-large",
+        "t5-3b": "t5-3b",
+        "mbart-large-50-many-to-many-mmt": "facebook/mbart-large-50-many-to-many-mmt",
+        "nllb-200-distilled-600M": "facebook/nllb-200-distilled-600M",
+        "madlad400-3b-mt": "jbochi/madlad400-3b-mt",    
+        "default": "t5-base",
+        }
+        if item.model in MODEL_MAP.keys():
+            target_model = item.model
+        else:
+            target_model = "default"
+
+        real_name = MODEL_MAP[target_model]
+        read_model = AutoModelForSeq2SeqLM.from_pretrained(real_name)
+        tokenizer = AutoTokenizer.from_pretrained(real_name)
+        #translator = pipeline("translation", model=read_model, tokenizer=tokenizer, src_lang=item.from_language, tgt_lang=item.to_language)
+        translate_query = (
+            f"translation_{item.from_language}_to_{item.to_language}"
         )
-        stream: bool = Field(
-            default=True,
-            description="(bool) Stream",
+        translator = pipeline(translate_query)
+        result = translator(item.input_text)    
+           
+        item_response = {
+            "statue": 200,
+            "result": result,
+            }
+        json_compatible_item_data = jsonable_encoder(item_response)
+        return JSONResponse(content=json_compatible_item_data)
+    class DetectLanguagePostItem(BaseModel):
+        input_text: str = Field(
+            default="Hello, how are you?",
+            description="(str) `Text for detection`",
         )
 
-    def chat_completions(
-        self, item: ChatCompletionsPostItem, api_key: str = Depends(extract_api_key)
-    ):
-        streamer = MessageStreamer(model=item.model)
-        composer = MessageComposer(model=item.model)
-        composer.merge(messages=item.messages)
-        # streamer.chat = stream_chat_mock
-
-        stream_response = streamer.chat_response(
-            prompt=composer.merged_str,
-            temperature=item.temperature,
-            top_p=item.top_p,
-            max_new_tokens=item.max_tokens,
-            api_key=api_key,
-            use_cache=item.use_cache,
+    def detect_language(self, item: DetectLanguagePostItem):
+        translator = Translator()
+        detected = translator.detect(item.input_text)
+
+        item_response = {
+            "lang": detected.lang,
+            "confidence": detected.confidence,
+        }
+        json_compatible_item_data = jsonable_encoder(item_response)
+        return JSONResponse(content=json_compatible_item_data)
+        
+    class TTSPostItem(BaseModel):
+        input_text: str = Field(
+            default="Hello",
+            description="(str) `Text for TTS`",
         )
-        if item.stream:
-            event_source_response = EventSourceResponse(
-                streamer.chat_return_generator(stream_response),
-                media_type="text/event-stream",
-                ping=2000,
-                ping_message_factory=lambda: ServerSentEvent(**{"comment": ""}),
-            )
-            return event_source_response
-        else:
-            data_response = streamer.chat_return_dict(stream_response)
-            return data_response
-
-    def get_readme(self):
-        readme_path = Path(__file__).parents[1] / "README.md"
-        with open(readme_path, "r", encoding="utf-8") as rf:
-            readme_str = rf.read()
-        readme_html = markdown2.markdown(
-            readme_str, extras=["table", "fenced-code-blocks", "highlightjs-lang"]
+        from_language: str = Field(
+            default="en",
+            description="(str) `TTS language`",
         )
-        return readme_html
-
+        
+    def text_to_speech(self, item: TTSPostItem):
+        try:
+            audioobj = gTTS(text = item.input_text, lang = item.from_language, slow = False)
+            fileName = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(10));
+            fileName = fileName + ".mp3";
+            mp3_fp = BytesIO()
+            #audioobj.save(fileName)
+            #audioobj.write_to_fp(mp3_fp)
+            #buffer = bytearray(mp3_fp.read())
+            #base64EncodedStr = base64.encodebytes(buffer)
+            #mp3_fp.read()
+            #return Response(content=mp3_fp.tell(), media_type="audio/mpeg")
+            return StreamingResponse(audioobj.stream())
+        except:
+               item_response = {
+                 "status": 400
+               }
+               json_compatible_item_data = jsonable_encoder(item_response)
+               return JSONResponse(content=json_compatible_item_data)
+           
+        
     def setup_routes(self):
-        for prefix in ["", "/v1", "/api", "/api/v1"]:
-            if prefix in ["/api/v1"]:
-                include_in_schema = True
-            else:
-                include_in_schema = False
-
+        for prefix in ["", "/v1"]:
             self.app.get(
-                prefix + "/models",
-                summary="Get available models",
-                include_in_schema=include_in_schema,
-            )(self.get_available_models)
+                prefix + "/langs",
+                summary="Get available languages",
+            )(self.get_available_langs)
+
+            self.app.post(
+                prefix + "/translate",
+                summary="translate text",
+            )(self.translate_completions)
+
+            self.app.post(
+                prefix + "/translate/ai",
+                summary="translate text with ai",
+            )(self.translate_ai_completions)
+            
+            self.app.post(
+                prefix + "/detect",
+                summary="detect language",
+            )(self.detect_language)
 
             self.app.post(
-                prefix + "/chat/completions",
-                summary="Chat completions in conversation session",
-                include_in_schema=include_in_schema,
-            )(self.chat_completions)
-        self.app.get(
-            "/readme",
-            summary="README of HF LLM API",
-            response_class=HTMLResponse,
-            include_in_schema=False,
-        )(self.get_readme)
+                prefix + "/tts",
+                summary="text to speech",
+            )(self.text_to_speech)
 
 
 class ArgParser(argparse.ArgumentParser):
@@ -203,6 +303,77 @@ class ArgParser(argparse.ArgumentParser):
 
 app = ChatAPIApp().app
 
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+@app.post("/transcribe")
+async def whisper_transcribe(
+    audio_file: UploadFile = File(description="Audio file for transcribe"),
+    language: str = Form(),
+    model: str = Form(),
+):
+    MODEL_MAP = {
+        "whisper-small": "openai/whisper-small",
+        "whisper-medium": "openai/whisper-medium",
+        "whisper-large": "openai/whisper-large",   
+        "default": "openai/whisper-small",
+    }
+    AUDIO_MAP = {
+        "audio/wav": "audio/wav",
+        "audio/mpeg": "audio/mpeg",
+        "audio/x-flac": "audio/x-flac",   
+    }
+    item_response = {
+            "statue": 200,
+            "result": "",
+            "start": 0,
+            "end": 0
+    }
+    if audio_file.content_type in AUDIO_MAP.keys():
+        if model in MODEL_MAP.keys():
+            target_model = model
+        else:
+            target_model = "default"
+
+        real_name = MODEL_MAP[target_model]
+        device = 0 if torch.cuda.is_available() else "cpu"
+        pipe = pipeline(
+           task="automatic-speech-recognition",
+           model=real_name,
+           chunk_length_s=30,
+           device=device,
+        )
+        time_start = time.time()
+        pipe.model.config.forced_decoder_ids = pipe.tokenizer.get_decoder_prompt_ids(language=language, task="transcribe")
+        try:
+           suffix = Path(audio_file.filename).suffix
+           with NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+            shutil.copyfileobj(audio_file.file, tmp)
+            tmp_path = Path(tmp.name)
+        finally:
+           audio_file.file.close()
+        #file_data = await audio_file.read()
+        # rv = data.encode('utf-8')
+        #rv = base64.b64encode(file_data).decode()
+        #print(rv, "rvrvrvrvr")
+        audio_data = np.fromfile(tmp_path)    
+        text = pipe(audio_data)["text"]
+        time_end = time.time()
+        item_response["status"] = 200
+        item_response["result"] = text
+        item_response["start"] = time_start
+        item_response["end"] = time_end
+    else:
+        item_response["status"] = 400
+        item_response["result"] = 'Acceptable files: audio/wav,audio/mpeg,audio/x-flac'
+        
+    
+    return item_response
+    
 if __name__ == "__main__":
     args = ArgParser().args
     if args.dev:
@@ -211,4 +382,4 @@ if __name__ == "__main__":
         uvicorn.run("__main__:app", host=args.server, port=args.port, reload=False)
 
     # python -m apis.chat_api      # [Docker] on product mode
-    # python -m apis.chat_api -d   # [Dev]    on develop mode
+    # python -m apis.chat_api -d   # [Dev]    on develop mode

apis/lang_name.json

@@ -0,0 +1 @@
+[{"code":"auto","name":"Detect language"},{"code":"fa","name":"Persian"},{"code":"en","name":"English"},{"code":"af","name":"Afrikaans"},{"code":"sq","name":"Albanian"},{"code":"am","name":"Amharic"},{"code":"ar","name":"Arabic"},{"code":"hy","name":"Armenian"},{"code":"as","name":"Assamese"},{"code":"ay","name":"Aymara"},{"code":"az","name":"Azerbaijani"},{"code":"bm","name":"Bambara"},{"code":"eu","name":"Basque"},{"code":"be","name":"Belarusian"},{"code":"bn","name":"Bengali"},{"code":"bho","name":"Bhojpuri"},{"code":"bs","name":"Bosnian"},{"code":"bg","name":"Bulgarian"},{"code":"ca","name":"Catalan"},{"code":"ceb","name":"Cebuano"},{"code":"ny","name":"Chichewa"},{"code":"zh-CN","name":"Chinese (Simplified)"},{"code":"zh-TW","name":"Chinese (Traditional)"},{"code":"co","name":"Corsican"},{"code":"hr","name":"Croatian"},{"code":"cs","name":"Czech"},{"code":"da","name":"Danish"},{"code":"dv","name":"Dhivehi"},{"code":"doi","name":"Dogri"},{"code":"nl","name":"Dutch"},{"code":"en","name":"English(recently used language)"},{"code":"eo","name":"Esperanto"},{"code":"et","name":"Estonian"},{"code":"ee","name":"Ewe"},{"code":"tl","name":"Filipino"},{"code":"fi","name":"Finnish"},{"code":"fr","name":"French"},{"code":"fy","name":"Frisian"},{"code":"gl","name":"Galician"},{"code":"ka","name":"Georgian"},{"code":"de","name":"German"},{"code":"el","name":"Greek"},{"code":"gn","name":"Guarani"},{"code":"gu","name":"Gujarati"},{"code":"ht","name":"Haitian Creole"},{"code":"ha","name":"Hausa"},{"code":"haw","name":"Hawaiian"},{"code":"iw","name":"Hebrew"},{"code":"hi","name":"Hindi"},{"code":"hmn","name":"Hmong"},{"code":"hu","name":"Hungarian"},{"code":"is","name":"Icelandic"},{"code":"ig","name":"Igbo"},{"code":"ilo","name":"Ilocano"},{"code":"id","name":"Indonesian"},{"code":"ga","name":"Irish"},{"code":"it","name":"Italian"},{"code":"ja","name":"Japanese"},{"code":"jw","name":"Javanese"},{"code":"kn","name":"Kannada"},{"code":"kk","name":"Kazakh"},{"code":"km","name":"Khmer"},{"code":"rw","name":"Kinyarwanda"},{"code":"gom","name":"Konkani"},{"code":"ko","name":"Korean"},{"code":"kri","name":"Krio"},{"code":"ku","name":"Kurdish (Kurmanji)"},{"code":"ckb","name":"Kurdish (Sorani)"},{"code":"ky","name":"Kyrgyz"},{"code":"lo","name":"Lao"},{"code":"la","name":"Latin"},{"code":"lv","name":"Latvian"},{"code":"ln","name":"Lingala"},{"code":"lt","name":"Lithuanian"},{"code":"lg","name":"Luganda"},{"code":"lb","name":"Luxembourgish"},{"code":"mk","name":"Macedonian"},{"code":"mai","name":"Maithili"},{"code":"mg","name":"Malagasy"},{"code":"ms","name":"Malay"},{"code":"ml","name":"Malayalam"},{"code":"mt","name":"Maltese"},{"code":"mi","name":"Maori"},{"code":"mr","name":"Marathi"},{"code":"mni-Mtei","name":"Meiteilon (Manipuri)"},{"code":"lus","name":"Mizo"},{"code":"mn","name":"Mongolian"},{"code":"my","name":"Myanmar (Burmese)"},{"code":"ne","name":"Nepali"},{"code":"no","name":"Norwegian"},{"code":"or","name":"Odia (Oriya)"},{"code":"om","name":"Oromo"},{"code":"ps","name":"Pashto"},{"code":"fa","name":"Persian(recently used language)"},{"code":"pl","name":"Polish"},{"code":"pt","name":"Portuguese"},{"code":"pa","name":"Punjabi"},{"code":"qu","name":"Quechua"},{"code":"ro","name":"Romanian"},{"code":"ru","name":"Russian"},{"code":"sm","name":"Samoan"},{"code":"sa","name":"Sanskrit"},{"code":"gd","name":"Scots Gaelic"},{"code":"nso","name":"Sepedi"},{"code":"sr","name":"Serbian"},{"code":"st","name":"Sesotho"},{"code":"sn","name":"Shona"},{"code":"sd","name":"Sindhi"},{"code":"si","name":"Sinhala"},{"code":"sk","name":"Slovak"},{"code":"sl","name":"Slovenian"},{"code":"so","name":"Somali"},{"code":"es","name":"Spanish"},{"code":"su","name":"Sundanese"},{"code":"sw","name":"Swahili"},{"code":"sv","name":"Swedish"},{"code":"tg","name":"Tajik"},{"code":"ta","name":"Tamil"},{"code":"tt","name":"Tatar"},{"code":"te","name":"Telugu"},{"code":"th","name":"Thai"},{"code":"ti","name":"Tigrinya"},{"code":"ts","name":"Tsonga"},{"code":"tr","name":"Turkish"},{"code":"tk","name":"Turkmen"},{"code":"ak","name":"Twi"},{"code":"uk","name":"Ukrainian"},{"code":"ur","name":"Urdu"},{"code":"ug","name":"Uyghur"},{"code":"uz","name":"Uzbek"},{"code":"vi","name":"Vietnamese"},{"code":"cy","name":"Welsh"},{"code":"xh","name":"Xhosa"},{"code":"yi","name":"Yiddish"},{"code":"yo","name":"Yoruba"},{"code":"zu","name":"Zulu"},{"code":"ckb","name":"Kurdish (Sorani)"},{"code":"ku","name":"Kurdish (Kurmanji)"},{"code":"en","name":"English"},{"code":"fa","name":"Persian"},{"code":"af","name":"Afrikaans"},{"code":"sq","name":"Albanian"},{"code":"am","name":"Amharic"},{"code":"ar","name":"Arabic"},{"code":"hy","name":"Armenian"},{"code":"as","name":"Assamese"},{"code":"ay","name":"Aymara"},{"code":"az","name":"Azerbaijani"},{"code":"bm","name":"Bambara"},{"code":"eu","name":"Basque"},{"code":"be","name":"Belarusian"},{"code":"bn","name":"Bengali"},{"code":"bho","name":"Bhojpuri"},{"code":"bs","name":"Bosnian"},{"code":"bg","name":"Bulgarian"},{"code":"ca","name":"Catalan"},{"code":"ceb","name":"Cebuano"},{"code":"ny","name":"Chichewa"},{"code":"zh-CN","name":"Chinese (Simplified)"},{"code":"zh-TW","name":"Chinese (Traditional)"},{"code":"co","name":"Corsican"},{"code":"hr","name":"Croatian"},{"code":"cs","name":"Czech"},{"code":"da","name":"Danish"},{"code":"dv","name":"Dhivehi"},{"code":"doi","name":"Dogri"},{"code":"nl","name":"Dutch"},{"code":"en","name":"English(recently used language)"},{"code":"eo","name":"Esperanto"},{"code":"et","name":"Estonian"},{"code":"ee","name":"Ewe"},{"code":"tl","name":"Filipino"},{"code":"fi","name":"Finnish"},{"code":"fr","name":"French"},{"code":"fy","name":"Frisian"},{"code":"gl","name":"Galician"},{"code":"ka","name":"Georgian"},{"code":"de","name":"German"},{"code":"el","name":"Greek"},{"code":"gn","name":"Guarani"},{"code":"gu","name":"Gujarati"},{"code":"ht","name":"Haitian Creole"},{"code":"ha","name":"Hausa"},{"code":"haw","name":"Hawaiian"},{"code":"iw","name":"Hebrew"},{"code":"hi","name":"Hindi"},{"code":"hmn","name":"Hmong"},{"code":"hu","name":"Hungarian"},{"code":"is","name":"Icelandic"},{"code":"ig","name":"Igbo"},{"code":"ilo","name":"Ilocano"},{"code":"id","name":"Indonesian"},{"code":"ga","name":"Irish"},{"code":"it","name":"Italian"},{"code":"ja","name":"Japanese"},{"code":"jw","name":"Javanese"},{"code":"kn","name":"Kannada"},{"code":"kk","name":"Kazakh"},{"code":"km","name":"Khmer"},{"code":"rw","name":"Kinyarwanda"},{"code":"gom","name":"Konkani"},{"code":"ko","name":"Korean"},{"code":"kri","name":"Krio"},{"code":"ku","name":"Kurdish (Kurmanji)(recently used language)"},{"code":"ckb","name":"Kurdish (Sorani)(recently used language)"},{"code":"ky","name":"Kyrgyz"},{"code":"lo","name":"Lao"},{"code":"la","name":"Latin"},{"code":"lv","name":"Latvian"},{"code":"ln","name":"Lingala"},{"code":"lt","name":"Lithuanian"},{"code":"lg","name":"Luganda"},{"code":"lb","name":"Luxembourgish"},{"code":"mk","name":"Macedonian"},{"code":"mai","name":"Maithili"},{"code":"mg","name":"Malagasy"},{"code":"ms","name":"Malay"},{"code":"ml","name":"Malayalam"},{"code":"mt","name":"Maltese"},{"code":"mi","name":"Maori"},{"code":"mr","name":"Marathi"},{"code":"mni-Mtei","name":"Meiteilon (Manipuri)"},{"code":"lus","name":"Mizo"},{"code":"mn","name":"Mongolian"},{"code":"my","name":"Myanmar (Burmese)"},{"code":"ne","name":"Nepali"},{"code":"no","name":"Norwegian"},{"code":"or","name":"Odia (Oriya)"},{"code":"om","name":"Oromo"},{"code":"ps","name":"Pashto"},{"code":"fa","name":"Persian(recently used language)"},{"code":"pl","name":"Polish"},{"code":"pt","name":"Portuguese"},{"code":"pa","name":"Punjabi"},{"code":"qu","name":"Quechua"},{"code":"ro","name":"Romanian"},{"code":"ru","name":"Russian"},{"code":"sm","name":"Samoan"},{"code":"sa","name":"Sanskrit"},{"code":"gd","name":"Scots Gaelic"},{"code":"nso","name":"Sepedi"},{"code":"sr","name":"Serbian"},{"code":"st","name":"Sesotho"},{"code":"sn","name":"Shona"},{"code":"sd","name":"Sindhi"},{"code":"si","name":"Sinhala"},{"code":"sk","name":"Slovak"},{"code":"sl","name":"Slovenian"},{"code":"so","name":"Somali"},{"code":"es","name":"Spanish"},{"code":"su","name":"Sundanese"},{"code":"sw","name":"Swahili"},{"code":"sv","name":"Swedish"},{"code":"tg","name":"Tajik"},{"code":"ta","name":"Tamil"},{"code":"tt","name":"Tatar"},{"code":"te","name":"Telugu"},{"code":"th","name":"Thai"},{"code":"ti","name":"Tigrinya"},{"code":"ts","name":"Tsonga"},{"code":"tr","name":"Turkish"},{"code":"tk","name":"Turkmen"},{"code":"ak","name":"Twi"},{"code":"uk","name":"Ukrainian"},{"code":"ur","name":"Urdu"},{"code":"ug","name":"Uyghur"},{"code":"uz","name":"Uzbek"},{"code":"vi","name":"Vietnamese"},{"code":"cy","name":"Welsh"},{"code":"xh","name":"Xhosa"},{"code":"yi","name":"Yiddish"},{"code":"yo","name":"Yoruba"},{"code":"zu","name":"Zulu"},{"code":"auto","name":"Detect language"},{"code":"fa","name":"Persian"},{"code":"en","name":"English"},{"code":"af","name":"Afrikaans"},{"code":"sq","name":"Albanian"},{"code":"am","name":"Amharic"},{"code":"ar","name":"Arabic"},{"code":"hy","name":"Armenian"},{"code":"as","name":"Assamese"},{"code":"ay","name":"Aymara"},{"code":"az","name":"Azerbaijani"},{"code":"bm","name":"Bambara"},{"code":"eu","name":"Basque"},{"code":"be","name":"Belarusian"},{"code":"bn","name":"Bengali"},{"code":"bho","name":"Bhojpuri"},{"code":"bs","name":"Bosnian"},{"code":"bg","name":"Bulgarian"},{"code":"ca","name":"Catalan"},{"code":"ceb","name":"Cebuano"},{"code":"ny","name":"Chichewa"},{"code":"zh-CN","name":"Chinese (Simplified)"},{"code":"zh-TW","name":"Chinese (Traditional)"},{"code":"co","name":"Corsican"},{"code":"hr","name":"Croatian"},{"code":"cs","name":"Czech"},{"code":"da","name":"Danish"},{"code":"dv","name":"Dhivehi"},{"code":"doi","name":"Dogri"},{"code":"nl","name":"Dutch"},{"code":"en","name":"English(recently used language)"},{"code":"eo","name":"Esperanto"},{"code":"et","name":"Estonian"},{"code":"ee","name":"Ewe"},{"code":"tl","name":"Filipino"},{"code":"fi","name":"Finnish"},{"code":"fr","name":"French"},{"code":"fy","name":"Frisian"},{"code":"gl","name":"Galician"},{"code":"ka","name":"Georgian"},{"code":"de","name":"German"},{"code":"el","name":"Greek"},{"code":"gn","name":"Guarani"},{"code":"gu","name":"Gujarati"},{"code":"ht","name":"Haitian Creole"},{"code":"ha","name":"Hausa"},{"code":"haw","name":"Hawaiian"},{"code":"iw","name":"Hebrew"},{"code":"hi","name":"Hindi"},{"code":"hmn","name":"Hmong"},{"code":"hu","name":"Hungarian"},{"code":"is","name":"Icelandic"},{"code":"ig","name":"Igbo"},{"code":"ilo","name":"Ilocano"},{"code":"id","name":"Indonesian"},{"code":"ga","name":"Irish"},{"code":"it","name":"Italian"},{"code":"ja","name":"Japanese"},{"code":"jw","name":"Javanese"},{"code":"kn","name":"Kannada"},{"code":"kk","name":"Kazakh"},{"code":"km","name":"Khmer"},{"code":"rw","name":"Kinyarwanda"},{"code":"gom","name":"Konkani"},{"code":"ko","name":"Korean"},{"code":"kri","name":"Krio"},{"code":"ku","name":"Kurdish (Kurmanji)"},{"code":"ckb","name":"Kurdish (Sorani)"},{"code":"ky","name":"Kyrgyz"},{"code":"lo","name":"Lao"},{"code":"la","name":"Latin"},{"code":"lv","name":"Latvian"},{"code":"ln","name":"Lingala"},{"code":"lt","name":"Lithuanian"},{"code":"lg","name":"Luganda"},{"code":"lb","name":"Luxembourgish"},{"code":"mk","name":"Macedonian"},{"code":"mai","name":"Maithili"},{"code":"mg","name":"Malagasy"},{"code":"ms","name":"Malay"},{"code":"ml","name":"Malayalam"},{"code":"mt","name":"Maltese"},{"code":"mi","name":"Maori"},{"code":"mr","name":"Marathi"},{"code":"mni-Mtei","name":"Meiteilon (Manipuri)"},{"code":"lus","name":"Mizo"},{"code":"mn","name":"Mongolian"},{"code":"my","name":"Myanmar (Burmese)"},{"code":"ne","name":"Nepali"},{"code":"no","name":"Norwegian"},{"code":"or","name":"Odia (Oriya)"},{"code":"om","name":"Oromo"},{"code":"ps","name":"Pashto"},{"code":"fa","name":"Persian(recently used language)"},{"code":"pl","name":"Polish"},{"code":"pt","name":"Portuguese"},{"code":"pa","name":"Punjabi"},{"code":"qu","name":"Quechua"},{"code":"ro","name":"Romanian"},{"code":"ru","name":"Russian"},{"code":"sm","name":"Samoan"},{"code":"sa","name":"Sanskrit"},{"code":"gd","name":"Scots Gaelic"},{"code":"nso","name":"Sepedi"},{"code":"sr","name":"Serbian"},{"code":"st","name":"Sesotho"},{"code":"sn","name":"Shona"},{"code":"sd","name":"Sindhi"},{"code":"si","name":"Sinhala"},{"code":"sk","name":"Slovak"},{"code":"sl","name":"Slovenian"},{"code":"so","name":"Somali"},{"code":"es","name":"Spanish"},{"code":"su","name":"Sundanese"},{"code":"sw","name":"Swahili"},{"code":"sv","name":"Swedish"},{"code":"tg","name":"Tajik"},{"code":"ta","name":"Tamil"},{"code":"tt","name":"Tatar"},{"code":"te","name":"Telugu"},{"code":"th","name":"Thai"},{"code":"ti","name":"Tigrinya"},{"code":"ts","name":"Tsonga"},{"code":"tr","name":"Turkish"},{"code":"tk","name":"Turkmen"},{"code":"ak","name":"Twi"},{"code":"uk","name":"Ukrainian"},{"code":"ur","name":"Urdu"},{"code":"ug","name":"Uyghur"},{"code":"uz","name":"Uzbek"},{"code":"vi","name":"Vietnamese"},{"code":"cy","name":"Welsh"},{"code":"xh","name":"Xhosa"},{"code":"yi","name":"Yiddish"},{"code":"yo","name":"Yoruba"},{"code":"zu","name":"Zulu"},{"code":"ckb","name":"Kurdish (Sorani)"},{"code":"ku","name":"Kurdish (Kurmanji)"},{"code":"en","name":"English"},{"code":"fa","name":"Persian"},{"code":"af","name":"Afrikaans"},{"code":"sq","name":"Albanian"},{"code":"am","name":"Amharic"},{"code":"ar","name":"Arabic"},{"code":"hy","name":"Armenian"},{"code":"as","name":"Assamese"},{"code":"ay","name":"Aymara"},{"code":"az","name":"Azerbaijani"},{"code":"bm","name":"Bambara"},{"code":"eu","name":"Basque"},{"code":"be","name":"Belarusian"},{"code":"bn","name":"Bengali"},{"code":"bho","name":"Bhojpuri"},{"code":"bs","name":"Bosnian"},{"code":"bg","name":"Bulgarian"},{"code":"ca","name":"Catalan"},{"code":"ceb","name":"Cebuano"},{"code":"ny","name":"Chichewa"},{"code":"zh-CN","name":"Chinese (Simplified)"},{"code":"zh-TW","name":"Chinese (Traditional)"},{"code":"co","name":"Corsican"},{"code":"hr","name":"Croatian"},{"code":"cs","name":"Czech"},{"code":"da","name":"Danish"},{"code":"dv","name":"Dhivehi"},{"code":"doi","name":"Dogri"},{"code":"nl","name":"Dutch"},{"code":"en","name":"English(recently used language)"},{"code":"eo","name":"Esperanto"},{"code":"et","name":"Estonian"},{"code":"ee","name":"Ewe"},{"code":"tl","name":"Filipino"},{"code":"fi","name":"Finnish"},{"code":"fr","name":"French"},{"code":"fy","name":"Frisian"},{"code":"gl","name":"Galician"},{"code":"ka","name":"Georgian"},{"code":"de","name":"German"},{"code":"el","name":"Greek"},{"code":"gn","name":"Guarani"},{"code":"gu","name":"Gujarati"},{"code":"ht","name":"Haitian Creole"},{"code":"ha","name":"Hausa"},{"code":"haw","name":"Hawaiian"},{"code":"iw","name":"Hebrew"},{"code":"hi","name":"Hindi"},{"code":"hmn","name":"Hmong"},{"code":"hu","name":"Hungarian"},{"code":"is","name":"Icelandic"},{"code":"ig","name":"Igbo"},{"code":"ilo","name":"Ilocano"},{"code":"id","name":"Indonesian"},{"code":"ga","name":"Irish"},{"code":"it","name":"Italian"},{"code":"ja","name":"Japanese"},{"code":"jw","name":"Javanese"},{"code":"kn","name":"Kannada"},{"code":"kk","name":"Kazakh"},{"code":"km","name":"Khmer"},{"code":"rw","name":"Kinyarwanda"},{"code":"gom","name":"Konkani"},{"code":"ko","name":"Korean"},{"code":"kri","name":"Krio"},{"code":"ku","name":"Kurdish (Kurmanji)(recently used language)"},{"code":"ckb","name":"Kurdish (Sorani)(recently used language)"},{"code":"ky","name":"Kyrgyz"},{"code":"lo","name":"Lao"},{"code":"la","name":"Latin"},{"code":"lv","name":"Latvian"},{"code":"ln","name":"Lingala"},{"code":"lt","name":"Lithuanian"},{"code":"lg","name":"Luganda"},{"code":"lb","name":"Luxembourgish"},{"code":"mk","name":"Macedonian"},{"code":"mai","name":"Maithili"},{"code":"mg","name":"Malagasy"},{"code":"ms","name":"Malay"},{"code":"ml","name":"Malayalam"},{"code":"mt","name":"Maltese"},{"code":"mi","name":"Maori"},{"code":"mr","name":"Marathi"},{"code":"mni-Mtei","name":"Meiteilon (Manipuri)"},{"code":"lus","name":"Mizo"},{"code":"mn","name":"Mongolian"},{"code":"my","name":"Myanmar (Burmese)"},{"code":"ne","name":"Nepali"},{"code":"no","name":"Norwegian"},{"code":"or","name":"Odia (Oriya)"},{"code":"om","name":"Oromo"},{"code":"ps","name":"Pashto"},{"code":"fa","name":"Persian(recently used language)"},{"code":"pl","name":"Polish"},{"code":"pt","name":"Portuguese"},{"code":"pa","name":"Punjabi"},{"code":"qu","name":"Quechua"},{"code":"ro","name":"Romanian"},{"code":"ru","name":"Russian"},{"code":"sm","name":"Samoan"},{"code":"sa","name":"Sanskrit"},{"code":"gd","name":"Scots Gaelic"},{"code":"nso","name":"Sepedi"},{"code":"sr","name":"Serbian"},{"code":"st","name":"Sesotho"},{"code":"sn","name":"Shona"},{"code":"sd","name":"Sindhi"},{"code":"si","name":"Sinhala"},{"code":"sk","name":"Slovak"},{"code":"sl","name":"Slovenian"},{"code":"so","name":"Somali"},{"code":"es","name":"Spanish"},{"code":"su","name":"Sundanese"},{"code":"sw","name":"Swahili"},{"code":"sv","name":"Swedish"},{"code":"tg","name":"Tajik"},{"code":"ta","name":"Tamil"},{"code":"tt","name":"Tatar"},{"code":"te","name":"Telugu"},{"code":"th","name":"Thai"},{"code":"ti","name":"Tigrinya"},{"code":"ts","name":"Tsonga"},{"code":"tr","name":"Turkish"},{"code":"tk","name":"Turkmen"},{"code":"ak","name":"Twi"},{"code":"uk","name":"Ukrainian"},{"code":"ur","name":"Urdu"},{"code":"ug","name":"Uyghur"},{"code":"uz","name":"Uzbek"},{"code":"vi","name":"Vietnamese"},{"code":"cy","name":"Welsh"},{"code":"xh","name":"Xhosa"},{"code":"yi","name":"Yiddish"},{"code":"yo","name":"Yoruba"},{"code":"zu","name":"Zulu"},{"code":"auto","name":"Detect language"},{"code":"fa","name":"Persian"},{"code":"en","name":"English"},{"code":"af","name":"Afrikaans"},{"code":"sq","name":"Albanian"},{"code":"am","name":"Amharic"},{"code":"ar","name":"Arabic"},{"code":"hy","name":"Armenian"},{"code":"as","name":"Assamese"},{"code":"ay","name":"Aymara"},{"code":"az","name":"Azerbaijani"},{"code":"bm","name":"Bambara"},{"code":"eu","name":"Basque"},{"code":"be","name":"Belarusian"},{"code":"bn","name":"Bengali"},{"code":"bho","name":"Bhojpuri"},{"code":"bs","name":"Bosnian"},{"code":"bg","name":"Bulgarian"},{"code":"ca","name":"Catalan"},{"code":"ceb","name":"Cebuano"},{"code":"ny","name":"Chichewa"},{"code":"zh-CN","name":"Chinese (Simplified)"},{"code":"zh-TW","name":"Chinese (Traditional)"},{"code":"co","name":"Corsican"},{"code":"hr","name":"Croatian"},{"code":"cs","name":"Czech"},{"code":"da","name":"Danish"},{"code":"dv","name":"Dhivehi"},{"code":"doi","name":"Dogri"},{"code":"nl","name":"Dutch"},{"code":"en","name":"English(recently used language)"},{"code":"eo","name":"Esperanto"},{"code":"et","name":"Estonian"},{"code":"ee","name":"Ewe"},{"code":"tl","name":"Filipino"},{"code":"fi","name":"Finnish"},{"code":"fr","name":"French"},{"code":"fy","name":"Frisian"},{"code":"gl","name":"Galician"},{"code":"ka","name":"Georgian"},{"code":"de","name":"German"},{"code":"el","name":"Greek"},{"code":"gn","name":"Guarani"},{"code":"gu","name":"Gujarati"},{"code":"ht","name":"Haitian Creole"},{"code":"ha","name":"Hausa"},{"code":"haw","name":"Hawaiian"},{"code":"iw","name":"Hebrew"},{"code":"hi","name":"Hindi"},{"code":"hmn","name":"Hmong"},{"code":"hu","name":"Hungarian"},{"code":"is","name":"Icelandic"},{"code":"ig","name":"Igbo"},{"code":"ilo","name":"Ilocano"},{"code":"id","name":"Indonesian"},{"code":"ga","name":"Irish"},{"code":"it","name":"Italian"},{"code":"ja","name":"Japanese"},{"code":"jw","name":"Javanese"},{"code":"kn","name":"Kannada"},{"code":"kk","name":"Kazakh"},{"code":"km","name":"Khmer"},{"code":"rw","name":"Kinyarwanda"},{"code":"gom","name":"Konkani"},{"code":"ko","name":"Korean"},{"code":"kri","name":"Krio"},{"code":"ku","name":"Kurdish (Kurmanji)"},{"code":"ckb","name":"Kurdish (Sorani)"},{"code":"ky","name":"Kyrgyz"},{"code":"lo","name":"Lao"},{"code":"la","name":"Latin"},{"code":"lv","name":"Latvian"},{"code":"ln","name":"Lingala"},{"code":"lt","name":"Lithuanian"},{"code":"lg","name":"Luganda"},{"code":"lb","name":"Luxembourgish"},{"code":"mk","name":"Macedonian"},{"code":"mai","name":"Maithili"},{"code":"mg","name":"Malagasy"},{"code":"ms","name":"Malay"},{"code":"ml","name":"Malayalam"},{"code":"mt","name":"Maltese"},{"code":"mi","name":"Maori"},{"code":"mr","name":"Marathi"},{"code":"mni-Mtei","name":"Meiteilon (Manipuri)"},{"code":"lus","name":"Mizo"},{"code":"mn","name":"Mongolian"},{"code":"my","name":"Myanmar (Burmese)"},{"code":"ne","name":"Nepali"},{"code":"no","name":"Norwegian"},{"code":"or","name":"Odia (Oriya)"},{"code":"om","name":"Oromo"},{"code":"ps","name":"Pashto"},{"code":"fa","name":"Persian(recently used language)"},{"code":"pl","name":"Polish"},{"code":"pt","name":"Portuguese"},{"code":"pa","name":"Punjabi"},{"code":"qu","name":"Quechua"},{"code":"ro","name":"Romanian"},{"code":"ru","name":"Russian"},{"code":"sm","name":"Samoan"},{"code":"sa","name":"Sanskrit"},{"code":"gd","name":"Scots Gaelic"},{"code":"nso","name":"Sepedi"},{"code":"sr","name":"Serbian"},{"code":"st","name":"Sesotho"},{"code":"sn","name":"Shona"},{"code":"sd","name":"Sindhi"},{"code":"si","name":"Sinhala"},{"code":"sk","name":"Slovak"},{"code":"sl","name":"Slovenian"},{"code":"so","name":"Somali"},{"code":"es","name":"Spanish"},{"code":"su","name":"Sundanese"},{"code":"sw","name":"Swahili"},{"code":"sv","name":"Swedish"},{"code":"tg","name":"Tajik"},{"code":"ta","name":"Tamil"},{"code":"tt","name":"Tatar"},{"code":"te","name":"Telugu"},{"code":"th","name":"Thai"},{"code":"ti","name":"Tigrinya"},{"code":"ts","name":"Tsonga"},{"code":"tr","name":"Turkish"},{"code":"tk","name":"Turkmen"},{"code":"ak","name":"Twi"},{"code":"uk","name":"Ukrainian"},{"code":"ur","name":"Urdu"},{"code":"ug","name":"Uyghur"},{"code":"uz","name":"Uzbek"},{"code":"vi","name":"Vietnamese"},{"code":"cy","name":"Welsh"},{"code":"xh","name":"Xhosa"},{"code":"yi","name":"Yiddish"},{"code":"yo","name":"Yoruba"},{"code":"zu","name":"Zulu"},{"code":"ckb","name":"Kurdish (Sorani)"},{"code":"ku","name":"Kurdish (Kurmanji)"},{"code":"en","name":"English"},{"code":"fa","name":"Persian"},{"code":"af","name":"Afrikaans"},{"code":"sq","name":"Albanian"},{"code":"am","name":"Amharic"},{"code":"ar","name":"Arabic"},{"code":"hy","name":"Armenian"},{"code":"as","name":"Assamese"},{"code":"ay","name":"Aymara"},{"code":"az","name":"Azerbaijani"},{"code":"bm","name":"Bambara"},{"code":"eu","name":"Basque"},{"code":"be","name":"Belarusian"},{"code":"bn","name":"Bengali"},{"code":"bho","name":"Bhojpuri"},{"code":"bs","name":"Bosnian"},{"code":"bg","name":"Bulgarian"},{"code":"ca","name":"Catalan"},{"code":"ceb","name":"Cebuano"},{"code":"ny","name":"Chichewa"},{"code":"zh-CN","name":"Chinese (Simplified)"},{"code":"zh-TW","name":"Chinese (Traditional)"},{"code":"co","name":"Corsican"},{"code":"hr","name":"Croatian"},{"code":"cs","name":"Czech"},{"code":"da","name":"Danish"},{"code":"dv","name":"Dhivehi"},{"code":"doi","name":"Dogri"},{"code":"nl","name":"Dutch"},{"code":"en","name":"English(recently used language)"},{"code":"eo","name":"Esperanto"},{"code":"et","name":"Estonian"},{"code":"ee","name":"Ewe"},{"code":"tl","name":"Filipino"},{"code":"fi","name":"Finnish"},{"code":"fr","name":"French"},{"code":"fy","name":"Frisian"},{"code":"gl","name":"Galician"},{"code":"ka","name":"Georgian"},{"code":"de","name":"German"},{"code":"el","name":"Greek"},{"code":"gn","name":"Guarani"},{"code":"gu","name":"Gujarati"},{"code":"ht","name":"Haitian Creole"},{"code":"ha","name":"Hausa"},{"code":"haw","name":"Hawaiian"},{"code":"iw","name":"Hebrew"},{"code":"hi","name":"Hindi"},{"code":"hmn","name":"Hmong"},{"code":"hu","name":"Hungarian"},{"code":"is","name":"Icelandic"},{"code":"ig","name":"Igbo"},{"code":"ilo","name":"Ilocano"},{"code":"id","name":"Indonesian"},{"code":"ga","name":"Irish"},{"code":"it","name":"Italian"},{"code":"ja","name":"Japanese"},{"code":"jw","name":"Javanese"},{"code":"kn","name":"Kannada"},{"code":"kk","name":"Kazakh"},{"code":"km","name":"Khmer"},{"code":"rw","name":"Kinyarwanda"},{"code":"gom","name":"Konkani"},{"code":"ko","name":"Korean"},{"code":"kri","name":"Krio"},{"code":"ku","name":"Kurdish (Kurmanji)(recently used language)"},{"code":"ckb","name":"Kurdish (Sorani)(recently used language)"},{"code":"ky","name":"Kyrgyz"},{"code":"lo","name":"Lao"},{"code":"la","name":"Latin"},{"code":"lv","name":"Latvian"},{"code":"ln","name":"Lingala"},{"code":"lt","name":"Lithuanian"},{"code":"lg","name":"Luganda"},{"code":"lb","name":"Luxembourgish"},{"code":"mk","name":"Macedonian"},{"code":"mai","name":"Maithili"},{"code":"mg","name":"Malagasy"},{"code":"ms","name":"Malay"},{"code":"ml","name":"Malayalam"},{"code":"mt","name":"Maltese"},{"code":"mi","name":"Maori"},{"code":"mr","name":"Marathi"},{"code":"mni-Mtei","name":"Meiteilon (Manipuri)"},{"code":"lus","name":"Mizo"},{"code":"mn","name":"Mongolian"},{"code":"my","name":"Myanmar (Burmese)"},{"code":"ne","name":"Nepali"},{"code":"no","name":"Norwegian"},{"code":"or","name":"Odia (Oriya)"},{"code":"om","name":"Oromo"},{"code":"ps","name":"Pashto"},{"code":"fa","name":"Persian(recently used language)"},{"code":"pl","name":"Polish"},{"code":"pt","name":"Portuguese"},{"code":"pa","name":"Punjabi"},{"code":"qu","name":"Quechua"},{"code":"ro","name":"Romanian"},{"code":"ru","name":"Russian"},{"code":"sm","name":"Samoan"},{"code":"sa","name":"Sanskrit"},{"code":"gd","name":"Scots Gaelic"},{"code":"nso","name":"Sepedi"},{"code":"sr","name":"Serbian"},{"code":"st","name":"Sesotho"},{"code":"sn","name":"Shona"},{"code":"sd","name":"Sindhi"},{"code":"si","name":"Sinhala"},{"code":"sk","name":"Slovak"},{"code":"sl","name":"Slovenian"},{"code":"so","name":"Somali"},{"code":"es","name":"Spanish"},{"code":"su","name":"Sundanese"},{"code":"sw","name":"Swahili"},{"code":"sv","name":"Swedish"},{"code":"tg","name":"Tajik"},{"code":"ta","name":"Tamil"},{"code":"tt","name":"Tatar"},{"code":"te","name":"Telugu"},{"code":"th","name":"Thai"},{"code":"ti","name":"Tigrinya"},{"code":"ts","name":"Tsonga"},{"code":"tr","name":"Turkish"},{"code":"tk","name":"Turkmen"},{"code":"ak","name":"Twi"},{"code":"uk","name":"Ukrainian"},{"code":"ur","name":"Urdu"},{"code":"ug","name":"Uyghur"},{"code":"uz","name":"Uzbek"},{"code":"vi","name":"Vietnamese"},{"code":"cy","name":"Welsh"},{"code":"xh","name":"Xhosa"},{"code":"yi","name":"Yiddish"},{"code":"yo","name":"Yoruba"},{"code":"zu","name":"Zulu"},{"code":"auto","name":"Detect language"},{"code":"fa","name":"Persian"},{"code":"en","name":"English"},{"code":"af","name":"Afrikaans"},{"code":"sq","name":"Albanian"},{"code":"am","name":"Amharic"},{"code":"ar","name":"Arabic"},{"code":"hy","name":"Armenian"},{"code":"as","name":"Assamese"},{"code":"ay","name":"Aymara"},{"code":"az","name":"Azerbaijani"},{"code":"bm","name":"Bambara"},{"code":"eu","name":"Basque"},{"code":"be","name":"Belarusian"},{"code":"bn","name":"Bengali"},{"code":"bho","name":"Bhojpuri"},{"code":"bs","name":"Bosnian"},{"code":"bg","name":"Bulgarian"},{"code":"ca","name":"Catalan"},{"code":"ceb","name":"Cebuano"},{"code":"ny","name":"Chichewa"},{"code":"zh-CN","name":"Chinese (Simplified)"},{"code":"zh-TW","name":"Chinese (Traditional)"},{"code":"co","name":"Corsican"},{"code":"hr","name":"Croatian"},{"code":"cs","name":"Czech"},{"code":"da","name":"Danish"},{"code":"dv","name":"Dhivehi"},{"code":"doi","name":"Dogri"},{"code":"nl","name":"Dutch"},{"code":"en","name":"English(recently used language)"},{"code":"eo","name":"Esperanto"},{"code":"et","name":"Estonian"},{"code":"ee","name":"Ewe"},{"code":"tl","name":"Filipino"},{"code":"fi","name":"Finnish"},{"code":"fr","name":"French"},{"code":"fy","name":"Frisian"},{"code":"gl","name":"Galician"},{"code":"ka","name":"Georgian"},{"code":"de","name":"German"},{"code":"el","name":"Greek"},{"code":"gn","name":"Guarani"},{"code":"gu","name":"Gujarati"},{"code":"ht","name":"Haitian Creole"},{"code":"ha","name":"Hausa"},{"code":"haw","name":"Hawaiian"},{"code":"iw","name":"Hebrew"},{"code":"hi","name":"Hindi"},{"code":"hmn","name":"Hmong"},{"code":"hu","name":"Hungarian"},{"code":"is","name":"Icelandic"},{"code":"ig","name":"Igbo"},{"code":"ilo","name":"Ilocano"},{"code":"id","name":"Indonesian"},{"code":"ga","name":"Irish"},{"code":"it","name":"Italian"},{"code":"ja","name":"Japanese"},{"code":"jw","name":"Javanese"},{"code":"kn","name":"Kannada"},{"code":"kk","name":"Kazakh"},{"code":"km","name":"Khmer"},{"code":"rw","name":"Kinyarwanda"},{"code":"gom","name":"Konkani"},{"code":"ko","name":"Korean"},{"code":"kri","name":"Krio"},{"code":"ku","name":"Kurdish (Kurmanji)"},{"code":"ckb","name":"Kurdish (Sorani)"},{"code":"ky","name":"Kyrgyz"},{"code":"lo","name":"Lao"},{"code":"la","name":"Latin"},{"code":"lv","name":"Latvian"},{"code":"ln","name":"Lingala"},{"code":"lt","name":"Lithuanian"},{"code":"lg","name":"Luganda"},{"code":"lb","name":"Luxembourgish"},{"code":"mk","name":"Macedonian"},{"code":"mai","name":"Maithili"},{"code":"mg","name":"Malagasy"},{"code":"ms","name":"Malay"},{"code":"ml","name":"Malayalam"},{"code":"mt","name":"Maltese"},{"code":"mi","name":"Maori"},{"code":"mr","name":"Marathi"},{"code":"mni-Mtei","name":"Meiteilon (Manipuri)"},{"code":"lus","name":"Mizo"},{"code":"mn","name":"Mongolian"},{"code":"my","name":"Myanmar (Burmese)"},{"code":"ne","name":"Nepali"},{"code":"no","name":"Norwegian"},{"code":"or","name":"Odia (Oriya)"},{"code":"om","name":"Oromo"},{"code":"ps","name":"Pashto"},{"code":"fa","name":"Persian(recently used language)"},{"code":"pl","name":"Polish"},{"code":"pt","name":"Portuguese"},{"code":"pa","name":"Punjabi"},{"code":"qu","name":"Quechua"},{"code":"ro","name":"Romanian"},{"code":"ru","name":"Russian"},{"code":"sm","name":"Samoan"},{"code":"sa","name":"Sanskrit"},{"code":"gd","name":"Scots Gaelic"},{"code":"nso","name":"Sepedi"},{"code":"sr","name":"Serbian"},{"code":"st","name":"Sesotho"},{"code":"sn","name":"Shona"},{"code":"sd","name":"Sindhi"},{"code":"si","name":"Sinhala"},{"code":"sk","name":"Slovak"},{"code":"sl","name":"Slovenian"},{"code":"so","name":"Somali"},{"code":"es","name":"Spanish"},{"code":"su","name":"Sundanese"},{"code":"sw","name":"Swahili"},{"code":"sv","name":"Swedish"},{"code":"tg","name":"Tajik"},{"code":"ta","name":"Tamil"},{"code":"tt","name":"Tatar"},{"code":"te","name":"Telugu"},{"code":"th","name":"Thai"},{"code":"ti","name":"Tigrinya"},{"code":"ts","name":"Tsonga"},{"code":"tr","name":"Turkish"},{"code":"tk","name":"Turkmen"},{"code":"ak","name":"Twi"},{"code":"uk","name":"Ukrainian"},{"code":"ur","name":"Urdu"},{"code":"ug","name":"Uyghur"},{"code":"uz","name":"Uzbek"},{"code":"vi","name":"Vietnamese"},{"code":"cy","name":"Welsh"},{"code":"xh","name":"Xhosa"},{"code":"yi","name":"Yiddish"},{"code":"yo","name":"Yoruba"},{"code":"zu","name":"Zulu"},{"code":"ckb","name":"Kurdish (Sorani)"},{"code":"ku","name":"Kurdish (Kurmanji)"},{"code":"en","name":"English"},{"code":"fa","name":"Persian"},{"code":"af","name":"Afrikaans"},{"code":"sq","name":"Albanian"},{"code":"am","name":"Amharic"},{"code":"ar","name":"Arabic"},{"code":"hy","name":"Armenian"},{"code":"as","name":"Assamese"},{"code":"ay","name":"Aymara"},{"code":"az","name":"Azerbaijani"},{"code":"bm","name":"Bambara"},{"code":"eu","name":"Basque"},{"code":"be","name":"Belarusian"},{"code":"bn","name":"Bengali"},{"code":"bho","name":"Bhojpuri"},{"code":"bs","name":"Bosnian"},{"code":"bg","name":"Bulgarian"},{"code":"ca","name":"Catalan"},{"code":"ceb","name":"Cebuano"},{"code":"ny","name":"Chichewa"},{"code":"zh-CN","name":"Chinese (Simplified)"},{"code":"zh-TW","name":"Chinese (Traditional)"},{"code":"co","name":"Corsican"},{"code":"hr","name":"Croatian"},{"code":"cs","name":"Czech"},{"code":"da","name":"Danish"},{"code":"dv","name":"Dhivehi"},{"code":"doi","name":"Dogri"},{"code":"nl","name":"Dutch"},{"code":"en","name":"English(recently used language)"},{"code":"eo","name":"Esperanto"},{"code":"et","name":"Estonian"},{"code":"ee","name":"Ewe"},{"code":"tl","name":"Filipino"},{"code":"fi","name":"Finnish"},{"code":"fr","name":"French"},{"code":"fy","name":"Frisian"},{"code":"gl","name":"Galician"},{"code":"ka","name":"Georgian"},{"code":"de","name":"German"},{"code":"el","name":"Greek"},{"code":"gn","name":"Guarani"},{"code":"gu","name":"Gujarati"},{"code":"ht","name":"Haitian Creole"},{"code":"ha","name":"Hausa"},{"code":"haw","name":"Hawaiian"},{"code":"iw","name":"Hebrew"},{"code":"hi","name":"Hindi"},{"code":"hmn","name":"Hmong"},{"code":"hu","name":"Hungarian"},{"code":"is","name":"Icelandic"},{"code":"ig","name":"Igbo"},{"code":"ilo","name":"Ilocano"},{"code":"id","name":"Indonesian"},{"code":"ga","name":"Irish"},{"code":"it","name":"Italian"},{"code":"ja","name":"Japanese"},{"code":"jw","name":"Javanese"},{"code":"kn","name":"Kannada"},{"code":"kk","name":"Kazakh"},{"code":"km","name":"Khmer"},{"code":"rw","name":"Kinyarwanda"},{"code":"gom","name":"Konkani"},{"code":"ko","name":"Korean"},{"code":"kri","name":"Krio"},{"code":"ku","name":"Kurdish (Kurmanji)(recently used language)"},{"code":"ckb","name":"Kurdish (Sorani)(recently used language)"},{"code":"ky","name":"Kyrgyz"},{"code":"lo","name":"Lao"},{"code":"la","name":"Latin"},{"code":"lv","name":"Latvian"},{"code":"ln","name":"Lingala"},{"code":"lt","name":"Lithuanian"},{"code":"lg","name":"Luganda"},{"code":"lb","name":"Luxembourgish"},{"code":"mk","name":"Macedonian"},{"code":"mai","name":"Maithili"},{"code":"mg","name":"Malagasy"},{"code":"ms","name":"Malay"},{"code":"ml","name":"Malayalam"},{"code":"mt","name":"Maltese"},{"code":"mi","name":"Maori"},{"code":"mr","name":"Marathi"},{"code":"mni-Mtei","name":"Meiteilon (Manipuri)"},{"code":"lus","name":"Mizo"},{"code":"mn","name":"Mongolian"},{"code":"my","name":"Myanmar (Burmese)"},{"code":"ne","name":"Nepali"},{"code":"no","name":"Norwegian"},{"code":"or","name":"Odia (Oriya)"},{"code":"om","name":"Oromo"},{"code":"ps","name":"Pashto"},{"code":"fa","name":"Persian(recently used language)"},{"code":"pl","name":"Polish"},{"code":"pt","name":"Portuguese"},{"code":"pa","name":"Punjabi"},{"code":"qu","name":"Quechua"},{"code":"ro","name":"Romanian"},{"code":"ru","name":"Russian"},{"code":"sm","name":"Samoan"},{"code":"sa","name":"Sanskrit"},{"code":"gd","name":"Scots Gaelic"},{"code":"nso","name":"Sepedi"},{"code":"sr","name":"Serbian"},{"code":"st","name":"Sesotho"},{"code":"sn","name":"Shona"},{"code":"sd","name":"Sindhi"},{"code":"si","name":"Sinhala"},{"code":"sk","name":"Slovak"},{"code":"sl","name":"Slovenian"},{"code":"so","name":"Somali"},{"code":"es","name":"Spanish"},{"code":"su","name":"Sundanese"},{"code":"sw","name":"Swahili"},{"code":"sv","name":"Swedish"},{"code":"tg","name":"Tajik"},{"code":"ta","name":"Tamil"},{"code":"tt","name":"Tatar"},{"code":"te","name":"Telugu"},{"code":"th","name":"Thai"},{"code":"ti","name":"Tigrinya"},{"code":"ts","name":"Tsonga"},{"code":"tr","name":"Turkish"},{"code":"tk","name":"Turkmen"},{"code":"ak","name":"Twi"},{"code":"uk","name":"Ukrainian"},{"code":"ur","name":"Urdu"},{"code":"ug","name":"Uyghur"},{"code":"uz","name":"Uzbek"},{"code":"vi","name":"Vietnamese"},{"code":"cy","name":"Welsh"},{"code":"xh","name":"Xhosa"},{"code":"yi","name":"Yiddish"},{"code":"yo","name":"Yoruba"},{"code":"zu","name":"Zulu"}]

examples/__init__.py

@@ -0,0 +1 @@
+#source

messagers/message_composer.py

@@ -1,26 +1,16 @@
 import re
 from pprint import pprint
-from utils.logger import logger
 
 
 class MessageComposer:
-    # LINK - apis/chat_api.py#available-models
-    AVALAIBLE_MODELS = [
-        "mixtral-8x7b",
-        "mistral-7b",
-        "openchat-3.5",
-        "nous-mixtral-8x7b",
-    ]
+    """
+    models:
+    - mixtral-8x7b (mistralai/Mixtral-8x7B-Instruct-v0.1)
+    """
 
     def __init__(self, model: str = None):
-        if model in self.AVALAIBLE_MODELS:
-            self.model = model
-        else:
-            self.model = "mixtral-8x7b"
-        self.system_roles = ["system"]
         self.inst_roles = ["user", "system", "inst"]
         self.answer_roles = ["assistant", "bot", "answer"]
-        self.default_role = "user"
 
     def concat_messages_by_role(self, messages):
         def is_same_role(role1, role2):
@@ -50,86 +40,37 @@ class MessageComposer:
         return concat_messages
 
     def merge(self, messages) -> str:
-        # Mistral and Mixtral:
-        #   <s> [INST] Instruction [/INST] Model answer </s> [INST] Follow-up instruction [/INST]
-
-        # OpenChat:
-        #   GPT4 Correct User: Hello<|end_of_turn|>GPT4 Correct Assistant: Hi<|end_of_turn|>GPT4 Correct User: How are you today?<|end_of_turn|>GPT4 Correct Assistant:
+        # <s> [INST] Instruction [/INST] Model answer </s> [INST] Follow-up instruction [/INST]
 
-        # Nous Mixtral:
-        #   <|im_start|>system
-        #   You are "Hermes 2".<|im_end|>
-        #   <|im_start|>user
-        #   Hello, who are you?<|im_end|>
-        #   <|im_start|>assistant
-
-        self.messages = messages
+        self.messages = self.concat_messages_by_role(messages)
         self.merged_str = ""
-
-        # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1#instruction-format
-        if self.model in ["mixtral-8x7b", "mistral-7b"]:
-            self.messages = self.concat_messages_by_role(messages)
-            self.cached_str = ""
-            for message in self.messages:
-                role = message["role"]
-                content = message["content"]
-                if role in self.inst_roles:
-                    self.cached_str = f"[INST] {content} [/INST]"
-                elif role in self.answer_roles:
-                    self.merged_str += f"<s> {self.cached_str} {content} </s>\n"
-                    self.cached_str = ""
-                else:
-                    self.cached_str = f"[INST] {content} [/INST]"
-            if self.cached_str:
-                self.merged_str += f"{self.cached_str}"
-        # https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO#prompt-format
-        elif self.model in ["nous-mixtral-8x7b"]:
-            self.merged_str_list = []
-            for message in self.messages:
-                role = message["role"]
-                content = message["content"]
-                if role not in ["system", "user", "assistant"]:
-                    role = self.default_role
-                message_line = f"<|im_start|>{role}\n{content}<|im_end|>"
-                self.merged_str_list.append(message_line)
-            self.merged_str_list.append("<|im_start|>assistant")
-            self.merged_str = "\n".join(self.merged_str_list)
-        # https://huggingface.co/openchat/openchat-3.5-0106
-        elif self.model in ["openchat-3.5"]:
-            self.messages = self.concat_messages_by_role(messages)
-            self.merged_str_list = []
-            self.end_of_turn = "<|end_of_turn|>"
-            for message in self.messages:
-                role = message["role"]
-                content = message["content"]
-                if role in self.inst_roles:
-                    self.merged_str_list.append(
-                        f"GPT4 Correct User:\n{content}{self.end_of_turn}"
-                    )
-                elif role in self.answer_roles:
-                    self.merged_str_list.append(
-                        f"GPT4 Correct Assistant:\n{content}{self.end_of_turn}"
-                    )
-                else:
-                    self.merged_str_list.append(
-                        f"GPT4 Correct User: {content}{self.end_of_turn}"
-                    )
-            self.merged_str_list.append(f"GPT4 Correct Assistant:\n")
-            self.merged_str = "\n".join(self.merged_str_list)
-        else:
-            self.merged_str = "\n".join(
-                [
-                    f'`{message["role"]}`:\n{message["content"]}\n'
-                    for message in self.messages
-                ]
-            )
+        self.cached_str = ""
+        for message in self.messages:
+            role = message["role"]
+            content = message["content"]
+            if role in self.inst_roles:
+                self.cached_str = f"[INST] {content} [/INST]"
+            elif role in self.answer_roles:
+                self.merged_str += f"<s> {self.cached_str} {content} </s>\n"
+                self.cached_str = ""
+            else:
+                self.cached_str = f"[INST] {content} [/INST]"
+        if self.cached_str:
+            self.merged_str += f"{self.cached_str}"
 
         return self.merged_str
 
-    def convert_pair_matches_to_messages(self, pair_matches_list):
-        messages = []
+    def split(self, merged_str) -> list:
+        self.messages = []
+        self.merged_str = merged_str
+        pair_pattern = (
+            r"<s>\s*\[INST\](?P<inst>[\s\S]*?)\[/INST\](?P<answer>[\s\S]*?)</s>"
+        )
+        pair_matches = re.finditer(pair_pattern, self.merged_str, re.MULTILINE)
+        pair_matches_list = list(pair_matches)
+
         if len(pair_matches_list) <= 0:
-            messages = [
+            self.messages = [
                 {
                     "role": "user",
                     "content": self.merged_str,
@@ -139,15 +80,17 @@ class MessageComposer:
             for match in pair_matches_list:
                 inst = match.group("inst")
                 answer = match.group("answer")
-                messages.extend(
+                self.messages.extend(
                     [
                         {"role": "user", "content": inst.strip()},
                         {"role": "assistant", "content": answer.strip()},
                     ]
                 )
-        return messages
 
-    def append_last_instruction_to_messages(self, inst_matches_list, pair_matches_list):
+        inst_pattern = r"\[INST\](?P<inst>[\s\S]*?)\[/INST\]"
+        inst_matches = re.finditer(inst_pattern, self.merged_str, re.MULTILINE)
+        inst_matches_list = list(inst_matches)
+
         if len(inst_matches_list) > len(pair_matches_list):
             self.messages.extend(
                 [
@@ -158,78 +101,20 @@ class MessageComposer:
                 ]
             )
 
-    def split(self, merged_str) -> list:
-        self.merged_str = merged_str
-        self.messages = []
-
-        if self.model in ["mixtral-8x7b", "mistral-7b"]:
-            pair_pattern = (
-                r"<s>\s*\[INST\](?P<inst>[\s\S]*?)\[/INST\](?P<answer>[\s\S]*?)</s>"
-            )
-            pair_matches = re.finditer(pair_pattern, self.merged_str, re.MULTILINE)
-            pair_matches_list = list(pair_matches)
-
-            self.messages = self.convert_pair_matches_to_messages(pair_matches_list)
-
-            inst_pattern = r"\[INST\](?P<inst>[\s\S]*?)\[/INST\]"
-            inst_matches = re.finditer(inst_pattern, self.merged_str, re.MULTILINE)
-            inst_matches_list = list(inst_matches)
-
-            self.append_last_instruction_to_messages(
-                inst_matches_list, pair_matches_list
-            )
-        elif self.model in ["nous-mixtral-8x7b"]:
-            # https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO#prompt-format
-            # message_pattern = r"<\|im_start\|>(?P<role>system|user|assistant)[\s\n]*(?P<content>[\s\S]*?)<\|im_end\|>"
-            message_pattern = r"<\|im_start\|>(?P<role>system|user|assistant)[\s\n]*(?P<content>[\s\S]*?)<\|im_end\|>"
-            message_matches = re.finditer(
-                message_pattern, self.merged_str, flags=re.MULTILINE | re.IGNORECASE
-            )
-            message_matches_list = list(message_matches)
-            logger.note(f"message_matches_list: {message_matches_list}")
-            for match in message_matches_list:
-                role = match.group("role")
-                content = match.group("content")
-                self.messages.append({"role": role, "content": content.strip()})
-        elif self.model in ["openchat-3.5"]:
-            pair_pattern = r"GPT4 Correct User:(?P<inst>[\s\S]*?)<\|end_of_turn\|>\s*GPT4 Correct Assistant:(?P<answer>[\s\S]*?)<\|end_of_turn\|>"
-            pair_matches = re.finditer(
-                pair_pattern, self.merged_str, flags=re.MULTILINE | re.IGNORECASE
-            )
-            pair_matches_list = list(pair_matches)
-            self.messages = self.convert_pair_matches_to_messages(pair_matches_list)
-            inst_pattern = r"GPT4 Correct User:(?P<inst>[\s\S]*?)<\|end_of_turn\|>"
-            inst_matches = re.finditer(
-                inst_pattern, self.merged_str, flags=re.MULTILINE | re.IGNORECASE
-            )
-            inst_matches_list = list(inst_matches)
-            self.append_last_instruction_to_messages(
-                inst_matches_list, pair_matches_list
-            )
-        else:
-            self.messages = [
-                {
-                    "role": "user",
-                    "content": self.merged_str,
-                }
-            ]
-
         return self.messages
 
 
 if __name__ == "__main__":
-    model = "mixtral-8x7b"
-    # model = "nous-mixtral-8x7b"
-    composer = MessageComposer(model)
+    composer = MessageComposer()
     messages = [
         {
             "role": "system",
-            "content": "You are a LLM developed by OpenAI.\nYour name is GPT-4.",
+            "content": "You are a LLM developed by OpenAI. Your name is GPT-4.",
         },
         {"role": "user", "content": "Hello, who are you?"},
         {"role": "assistant", "content": "I am a bot."},
-        {"role": "user", "content": "What is your name?"},
-        # {"role": "assistant", "content": "My name is Bing."},
+        # {"role": "user", "content": "What is your name?"},
+        {"role": "assistant", "content": "My name is Bing."},
         # {"role": "user", "content": "Tell me a joke."},
         # {"role": "assistant", "content": "What is a robot's favorite type of music?"},
         # {
@@ -237,11 +122,7 @@ if __name__ == "__main__":
         #     "content": "How many questions have I asked? Please list them.",
         # },
     ]
-    logger.note(f"model: {composer.model}")
     merged_str = composer.merge(messages)
-    logger.note("merged_str:")
-    logger.mesg(merged_str)
-    logger.note("splitted messages:")
+    print(merged_str)
     pprint(composer.split(merged_str))
-    # logger.note("merged merged_str:")
-    # logger.mesg(composer.merge(composer.split(merged_str)))
+    # print(composer.merge(composer.split(merged_str)))

messagers/message_outputer.py

@@ -7,22 +7,20 @@ class OpenaiStreamOutputer:
     * https://platform.openai.com/docs/api-reference/chat/create
     """
 
-    def __init__(self):
-        self.default_data = {
+    def data_to_string(self, data={}, content_type=""):
+        data_str = f"{json.dumps(data)}"
+
+        return data_str
+
+    def output(self, content=None, content_type="Completions") -> str:
+        data = {
             "created": 1700000000,
             "id": "chatcmpl-hugginface",
             "object": "chat.completion.chunk",
-            # "content_type": "Completions",
+            # "content_type": content_type,
             "model": "hugginface",
             "choices": [],
         }
-
-    def data_to_string(self, data={}, content_type=""):
-        data_str = f"{json.dumps(data)}"
-        return data_str
-
-    def output(self, content=None, content_type="Completions") -> str:
-        data = self.default_data.copy()
         if content_type == "Role":
             data["choices"] = [
                 {

networks/message_streamer.py

@@ -1,7 +1,6 @@
 import json
 import re
 import requests
-from tiktoken import get_encoding as tiktoken_get_encoding
 from messagers.message_outputer import OpenaiStreamOutputer
 from utils.logger import logger
 from utils.enver import enver
@@ -11,27 +10,11 @@ class MessageStreamer:
     MODEL_MAP = {
         "mixtral-8x7b": "mistralai/Mixtral-8x7B-Instruct-v0.1",  # 72.62, fast [Recommended]
         "mistral-7b": "mistralai/Mistral-7B-Instruct-v0.2",  # 65.71, fast
-        "nous-mixtral-8x7b": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
-        # "openchat-3.5": "openchat/openchat-3.5-1210",  # 68.89, fast
-        # "zephyr-7b-beta": "HuggingFaceH4/zephyr-7b-beta",  # ❌ Too Slow
-        # "llama-70b": "meta-llama/Llama-2-70b-chat-hf",  # ❌ Require Pro User
-        # "codellama-34b": "codellama/CodeLlama-34b-Instruct-hf",  # ❌ Low Score
-        # "falcon-180b": "tiiuae/falcon-180B-chat",  # ❌ Require Pro User
+        "openchat-3.5": "openchat/openchat_3.5",  # 61.24, fast
+        # "zephyr-7b-alpha": "HuggingFaceH4/zephyr-7b-alpha",  # 59.5, fast
+        # "zephyr-7b-beta": "HuggingFaceH4/zephyr-7b-beta",  # 61.95, slow
         "default": "mistralai/Mixtral-8x7B-Instruct-v0.1",
     }
-    STOP_SEQUENCES_MAP = {
-        "mixtral-8x7b": "</s>",
-        "mistral-7b": "</s>",
-        "nous-mixtral-8x7b": "<|im_end|>",
-        "openchat-3.5": "<|end_of_turn|>",
-    }
-    TOKEN_LIMIT_MAP = {
-        "mixtral-8x7b": 32768,
-        "mistral-7b": 32768,
-        "nous-mixtral-8x7b": 32768,
-        "openchat-3.5": 8192,
-    }
-    TOKEN_RESERVED = 100
 
     def __init__(self, model: str):
         if model in self.MODEL_MAP.keys():
@@ -40,32 +23,21 @@ class MessageStreamer:
             self.model = "default"
         self.model_fullname = self.MODEL_MAP[self.model]
         self.message_outputer = OpenaiStreamOutputer()
-        self.tokenizer = tiktoken_get_encoding("cl100k_base")
 
     def parse_line(self, line):
         line = line.decode("utf-8")
         line = re.sub(r"data:\s*", "", line)
         data = json.loads(line)
-        try:
-            content = data["token"]["text"]
-        except:
-            logger.err(data)
+        content = data["token"]["text"]
         return content
 
-    def count_tokens(self, text):
-        tokens = self.tokenizer.encode(text)
-        token_count = len(tokens)
-        logger.note(f"Prompt Token Count: {token_count}")
-        return token_count
-
-    def chat_response(
+    def chat(
         self,
         prompt: str = None,
-        temperature: float = 0.5,
-        top_p: float = 0.95,
-        max_new_tokens: int = None,
-        api_key: str = None,
-        use_cache: bool = False,
+        temperature: float = 0.01,
+        max_new_tokens: int = 8192,
+        stream: bool = True,
+        yield_output: bool = False,
     ):
         # https://huggingface.co/docs/api-inference/detailed_parameters?code=curl
         # curl --proxy http://<server>:<port> https://api-inference.huggingface.co/models/<org>/<model_name> -X POST -d '{"inputs":"who are you?","parameters":{"max_new_token":64}}' -H 'Content-Type: application/json' -H 'Authorization: Bearer <HF_TOKEN>'
@@ -75,138 +47,51 @@ class MessageStreamer:
         self.request_headers = {
             "Content-Type": "application/json",
         }
-
-        if api_key:
-            logger.note(
-                f"Using API Key: {api_key[:3]}{(len(api_key)-7)*'*'}{api_key[-4:]}"
-            )
-            self.request_headers["Authorization"] = f"Bearer {api_key}"
-
-        if temperature is None or temperature < 0:
-            temperature = 0.0
-        # temperature must  0 < and < 1 for HF LLM models
-        temperature = max(temperature, 0.01)
-        temperature = min(temperature, 0.99)
-        top_p = max(top_p, 0.01)
-        top_p = min(top_p, 0.99)
-
-        token_limit = int(
-            self.TOKEN_LIMIT_MAP[self.model]
-            - self.TOKEN_RESERVED
-            - self.count_tokens(prompt) * 1.35
-        )
-        if token_limit <= 0:
-            raise ValueError("Prompt exceeded token limit!")
-
-        if max_new_tokens is None or max_new_tokens <= 0:
-            max_new_tokens = token_limit
-        else:
-            max_new_tokens = min(max_new_tokens, token_limit)
-
         # References:
         #   huggingface_hub/inference/_client.py:
         #     class InferenceClient > def text_generation()
         #   huggingface_hub/inference/_text_generation.py:
         #     class TextGenerationRequest > param `stream`
         # https://huggingface.co/docs/text-generation-inference/conceptual/streaming#streaming-with-curl
-        # https://huggingface.co/docs/api-inference/detailed_parameters#text-generation-task
         self.request_body = {
             "inputs": prompt,
             "parameters": {
-                "temperature": temperature,
-                "top_p": top_p,
+                "temperature": max(temperature, 0.01),  # must be positive
                 "max_new_tokens": max_new_tokens,
                 "return_full_text": False,
             },
-            "options": {
-                "use_cache": use_cache,
-            },
-            "stream": True,
+            "stream": stream,
         }
-
-        if self.model in self.STOP_SEQUENCES_MAP.keys():
-            self.stop_sequences = self.STOP_SEQUENCES_MAP[self.model]
-        #     self.request_body["parameters"]["stop_sequences"] = [
-        #         self.STOP_SEQUENCES[self.model]
-        #     ]
-
         logger.back(self.request_url)
         enver.set_envs(proxies=True)
-        stream_response = requests.post(
+        stream = requests.post(
             self.request_url,
             headers=self.request_headers,
             json=self.request_body,
             proxies=enver.requests_proxies,
-            stream=True,
+            stream=stream,
         )
-        status_code = stream_response.status_code
+        status_code = stream.status_code
         if status_code == 200:
             logger.success(status_code)
         else:
             logger.err(status_code)
 
-        return stream_response
-
-    def chat_return_dict(self, stream_response):
-        # https://platform.openai.com/docs/guides/text-generation/chat-completions-response-format
-        final_output = self.message_outputer.default_data.copy()
-        final_output["choices"] = [
-            {
-                "index": 0,
-                "finish_reason": "stop",
-                "message": {
-                    "role": "assistant",
-                    "content": "",
-                },
-            }
-        ]
-        logger.back(final_output)
-
-        final_content = ""
-        for line in stream_response.iter_lines():
+        for line in stream.iter_lines():
             if not line:
                 continue
-            content = self.parse_line(line)
-
-            if content.strip() == self.stop_sequences:
-                logger.success("\n[Finished]")
-                break
-            else:
-                logger.back(content, end="")
-                final_content += content
-
-        if self.model in self.STOP_SEQUENCES_MAP.keys():
-            final_content = final_content.replace(self.stop_sequences, "")
-
-        final_content = final_content.strip()
-        final_output["choices"][0]["message"]["content"] = final_content
-        return final_output
-
-    def chat_return_generator(self, stream_response):
-        is_finished = False
-        line_count = 0
-        for line in stream_response.iter_lines():
-            if line:
-                line_count += 1
-            else:
-                continue
 
             content = self.parse_line(line)
 
-            if content.strip() == self.stop_sequences:
+            if content.strip() == "</s>":
                 content_type = "Finished"
                 logger.success("\n[Finished]")
-                is_finished = True
             else:
                 content_type = "Completions"
-                if line_count == 1:
-                    content = content.lstrip()
                 logger.back(content, end="")
 
-            output = self.message_outputer.output(
-                content=content, content_type=content_type
-            )
-            yield output
-
-        if not is_finished:
-            yield self.message_outputer.output(content="", content_type="Finished")
+            if yield_output:
+                output = self.message_outputer.output(
+                    content=content, content_type=content_type
+                )
+                yield output

requirements.txt

@@ -1,12 +1,17 @@
 aiohttp
 fastapi
 httpx
-markdown2[all]
 openai
 pydantic
 requests
 sse_starlette
 termcolor
-tiktoken
 uvicorn
-websockets
+websockets
+googletrans==3.1.0a0
+gTTS
+torch
+transformers
+transformers[sentencepiece]
+python-multipart
+numpy

utils/enver.py

@@ -1,8 +1,10 @@
 import json
 import os
+
 from pathlib import Path
 from utils.logger import logger
 
+
 class OSEnver:
     def __init__(self):
         self.envs_stack = []