update

.gitignore

@@ -2,3 +2,5 @@
 chroma_db/
 __pycache__/
 speech_audio/
+video_cache/
+tts_folder/

api/openai_scripts_tai_gi/audio_processing.py

@@ -21,14 +21,14 @@ def transcribe(audio_file):
             file=audio_file,
             model="whisper-1",
             response_format="text", 
-            prompt=f"轉錄時對於「早餐、午餐、晚餐、幼兒園、國小、國中、高中、大學」等詞需特別注意。也需注意以下詞彙：{custom_vocab}"
+            prompt=f"""你是一位專業的閩南語轉錄為中文校對助理，專門處理有關長照/日照中心的台語對話轉錄. 
+            轉錄時對於「早餐、午餐、晚餐、幼兒園、國小、國中、高中、大學」等詞需特別注意。也需注意以下詞彙：{custom_vocab}"""
         )
         return transcript
     except Exception as e:
         print(f"轉錄時發生錯誤：{str(e)}")
         return None
 
-
 def post_process_transcript(transcript, temperature=0):
     corrected_transcript = fuzzy_correct_chinese(transcript)
     

api/openai_scripts_tai_gi/config.py

@@ -13,7 +13,7 @@ if not SUPABASE_URL or not SUPABASE_KEY:
 
 SYSTEM_PROMPT = """你是一位專業的閩南語轉錄為中文校對助理，專門處理有關長照/日照中心的台語對話轉錄。
 你的任務是：
-1. 確保以下專業術語在台語拼音轉換時的準確性：幫我、身體、什麼時候、現在、有點、冷、會不會、車、昨晚、血壓、需要、量血壓、早餐、午餐、晚餐、主任、早安、午安、晚安、聯絡簿、等車、交通車、國小、什麼、看什麼、可以、什麼意思、垃圾、冷氣、電風扇、喝水、倒垃圾、上廁所、上班、看孫子、走路、起床、幾點、毛筆、冰箱、哥哥、妹妹、菜市場、夜市、工作、老闆。
+1. 確保以下專業術語在台語拼音轉換時的準確性：廁所、幫我、身體、什麼時候、現在、有點、冷、會不會、車、昨晚、血壓、需要、量血壓、早餐、午餐、晚餐、主任、早安、午安、晚安、聯絡簿、等車、交通車、國小、什麼、看什麼、可以、什麼意思、垃圾、冷氣、電風扇、喝水、倒垃圾、上廁所、上班、看孫子、走路、起床、幾點、毛筆、冰箱、哥哥、妹妹、菜市場、夜市、工作、老闆。
 2. 在必要時添加適當的標點符號，如句號、逗號
 3. 使用台灣的繁體中文，確保語言表達符合台灣的用語習慣。
 4. 只更正明顯的錯誤或改善可讀性，不要改變原文的意思或結構。
@@ -22,12 +22,20 @@ SYSTEM_PROMPT = """你是一位專業的閩南語轉錄為中文校對助理，
 
 請只根據提供的原文進行必要的更正，不要添加或刪除任何實質性內容。在修正時，請特別注意上下文，確保修正後的詞語符合整句話的語境。"""
 
-CORRECT_TERMS = ["幫我", "什麼時候", "身體", "現在", "有點", "冷", "會不會", "車", "昨晚", "血壓", "需要", "量血壓", "早餐", "午餐", "晚餐", "主任", "早安", "午安", "晚安", "聯絡簿", "等車", "交通車", "幼兒園", "國小", "國中", "高中", "大學", "日照中心", "吃飯", "垃圾", "冷氣", "孫子", "電風扇", "喝水", "倒垃圾", "上廁所", "上班", "看孫子", "聯繫", "走路", "起床", "幾點", "毛筆", "電視", "冰箱", "廚房", "可以", "哥哥", "妹妹", "夜市", "工作", "老闆", "什麼"]
+CORRECT_TERMS = ["廁所", "幫我", "什麼時候", "身體", "現在", "有點", "冷", "會不會", "車", "昨晚", "血壓", "需要", "量血壓", "早餐", "午餐", "晚餐", "主任", "早安", "午安", "晚安", "聯絡簿", "等車", "交通車", "幼兒園", "國小", "國中", "高中", "大學", "日照中心", "吃飯", "垃圾", "冷氣", "孫子", "電風扇", "喝水", "倒垃圾", "上廁所", "上班", "看孫子", "聯繫", "走路", "起床", "幾點", "毛筆", "電視", "冰箱", "廚房", "可以", "哥哥", "妹妹", "夜市", "工作", "老闆", "什麼"]
 
 ERROR_CORRECTION = {
     '跨三回' : '看什麼', 
     '畫沙小' : '看三小', 
     '高通車' : '交通車', 
-    '的東西搞' : '什麼時候到'
+    '的東西搞' : '什麼時候到', 
+    '倒三餐' : '', 
+    '幫我看什麼嗎' : '幫我', 
+    '今天中午回來吃什麼?' : '今天午餐吃甚麼?', 
+    '本授抵多維' : '廁所在哪裡?', 
+    '本書的國語' : '廁所在哪裡?', 
+    '本週底得位' : '廁所在哪裡?', 
+    '本授地當位' : '廁所在哪裡?', 
+    '本說底都會' : '廁所在哪裡?', 
 }
 

app.py

@@ -1,3 +1,4 @@
+import openai
 import uvicorn
 
 from typing import List
@@ -47,7 +48,7 @@ async def chat(message, chat_history: List[ChatHistoryItem] = Body(...)):
         AIMessage(content="你好！很高興能和您聊天。今天您過得怎麼樣呢？有沒有什麼想分享的事情？")
     ]
     
-    for item in chat_history:
+    for item in chat_history[-10:]:
         if item.q == "string" or item.a == "string" : continue
         messages_list.append(HumanMessage(content=item.q))
         messages_list.append(AIMessage(content=item.a))
@@ -57,15 +58,24 @@ async def chat(message, chat_history: List[ChatHistoryItem] = Body(...)):
     prompt = ChatPromptTemplate(
         messages=messages_list
     )
-
-    with get_openai_callback() as cb:
-        cache_question, cache_answer = semantic_cache(supabase, message)
-        if cache_answer:
-            save_history(message, cache_answer)
-            return {"message": cache_answer}
-
-    AIMessage_ = chat_model.invoke(prompt.format_messages()).content
-    save_history(message, AIMessage_)
+    try:
+        with get_openai_callback() as cb:
+            # cache_question, cache_answer = semantic_cache(supabase, message)
+            cache_question, cache_answer, video_cache = semantic_cache(supabase, message, SIMILARITY_THRESHOLD=0.83)
+            if cache_answer:
+                save_history(message, cache_answer)
+                # return {"message": cache_answer}
+                if video_cache is not None:
+                    return {"message": cache_answer, "video_cache": video_cache}
+                else:
+                    return {"message": cache_answer}
+
+            AIMessage_ = chat_model.invoke(prompt.format_messages()).content
+            save_history(message, AIMessage_)
+            
+    except openai.RateLimitError as e:
+            print(f"Rate limit exceeded: {e}")
+            return {'message': "Current quota exceeded."}
     
     return {"message": AIMessage_}
 
@@ -75,9 +85,22 @@ def save_history(question, answer):
             .insert({"question": question, "answer": answer})
             .execute()
         )
-    
+
+from starlette.responses import JSONResponse
+@app.get("/health") # 使用網址給 kuma 監測
+async def health_check():
+    return JSONResponse(content={"status": "ok"}, status_code=200)
+
+
 if __name__ == "__main__":
     
-    uvicorn.run("app:app", reload=False, port=8087, host='cmm.ai', ssl_keyfile="/etc/letsencrypt/live/cmm.ai/privkey.pem", ssl_certfile="/etc/letsencrypt/live/cmm.ai/fullchain.pem")
+    try:
+        uvicorn.run("app:app", reload=False, port=8087, host='cmm.ai', ssl_keyfile="/etc/letsencrypt/live/cmm.ai/privkey.pem", ssl_certfile="/etc/letsencrypt/live/cmm.ai/fullchain.pem")
+    finally:
+        import shutil
+        cache_db = "chroma_db"
+        shutil.rmtree(cache_db)
+
+
 
 

semantic_search.py

@@ -51,10 +51,14 @@ def semantic_cache(supabase, q, SIMILARITY_THRESHOLD=0.83, k=1, vectordb_directo
     
     if score >= SIMILARITY_THRESHOLD:
         cache_question = doc.page_content
+        print(cache_question)
 
-        response = supabase.table("INNOLUX_cache").select("question, answer").eq("question", cache_question).execute()
+        # response = supabase.table("INNOLUX_cache").select("question, answer").eq("question", cache_question).execute()
+        response = supabase.table("INNOLUX_cache").select("question, answer, video_url").eq("question", cache_question).execute()
+        print(response.data)
 
         answer = response.data[0]["answer"]
-        return cache_question, answer
+        video_cache = response.data[0]["video_url"]
+        return cache_question, answer, video_cache
     else:
-        return None, None
+        return None, None, None

test2.py

@@ -0,0 +1,18 @@
+import os
+import shutil
+
+default_download_folder = os.path.join(os.path.expanduser('~'), 'Downloads')
+download_folder = os.path.normpath(os.path.join(os.path.dirname(__file__), 'downloads'))
+os.makedirs(download_folder, exist_ok=True)
+
+file_path = os.path.join(download_folder, 'output.wav')
+if os.path.exists(download_folder):
+    destination_path = os.path.join(download_folder, 'output.wav')
+    try:
+        # shutil.move(file_path, destination_path)
+        # print(f"文件已移動到: {destination_path}")
+        print(file_path)
+    except Exception as e:
+        print(f"移動文件時出錯: {e}")
+else:
+    print(f"文件不存在: {file_path}")

test3.py

@@ -0,0 +1,124 @@
+import os
+# import requests
+# from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+# import re 
+import time
+# from fake_useragent import UserAgent
+import undetected_chromedriver as uc
+from datetime import datetime
+import random
+import string
+# from selenium.webdriver.common.action_chains import ActionChains
+# from selenium.webdriver.common.keys import Keys
+# from selenium.webdriver.support.ui import WebDriverWait
+# from selenium.webdriver.support import expected_conditions as EC
+# from ga4mp import FirebaseMP
+from dotenv import load_dotenv
+import os
+# import shutil
+import logging
+from fastapi import APIRouter, FastAPI
+import uvicorn
+from fastapi.middleware.cors import CORSMiddleware
+from supabase import create_client, Client
+
+
+load_dotenv()
+
+logging.basicConfig(level=logging.INFO)
+
+SUPABASE_URL: str =  os.environ.get('SUPABASE_URL')
+SUPABASE_KEY: str = os.environ.get('SUPABASE_KEY')
+supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
+
+def download_blob(browser, blob_url, filename='output.mp3'):
+    # 使用 Selenium 獲取 Blob 內容
+    js_code = f"""
+    fetch('{blob_url}')
+        .then(response => response.blob())
+        .then(blob => {{
+            const url = URL.createObjectURL(blob);
+            const a = document.createElement('a');
+            a.href = url;
+            a.download = '{filename}';
+            document.body.appendChild(a);
+            a.click();
+            document.body.removeChild(a);
+        }})
+        .catch(error => console.error('Error downloading file:', error));
+    """
+    browser.execute_script(js_code)
+
+def tts_downloadfile(text):
+    start_time = time.time()
+    print(f'text長度: {len(text)}')
+    num = random.randint(3,5)
+    url = 'http://tts001.iptcloud.net:8804/'
+
+    # default_download_folder = os.path.join(os.path.expanduser('~'), 'Downloads')
+    download_folder = '/var/www/html/innolux/downloads'
+    file_name = datetime.now().strftime(f"%Y%m%d%H%M%S_{''.join(random.sample(string.ascii_lowercase, 3))}.wav")
+    # 替換為你希望的文件夾路徑
+    os.makedirs(download_folder, exist_ok=True)
+
+    options = uc.ChromeOptions()
+    options.add_argument('--ignore-certificate-errors')
+    prefs = {
+        "download.default_directory": download_folder,  # 設定預設下載文件夾
+        "download.prompt_for_download": False,
+        "safebrowsing.enabled": True,  # 確保安全瀏覽
+    }
+    options.add_experimental_option("prefs", prefs)
+    # options.add_argument('--incognito')
+    options.add_argument('--headless')  # 如果不想顯示瀏覽器可以啟用這行
+    options.add_argument("--disable-gpu")  # 禁用 GPU 加速
+
+    # 設置自定義 headers
+    with uc.Chrome(options=options, version_main=129) as browser:
+        try:           
+            browser.get(url)
+
+            time.sleep(num)
+            # 轉中文成台語拼音
+            browser.find_element(By.XPATH, '//*[@id="js-input"]').send_keys(text)
+            time.sleep(0.1)
+            browser.find_element(By.XPATH, '//*[@id="js-translate"]').click()
+            time.sleep(0.1 + len(text)*0.01)
+
+            browser.execute_script('window.scrollBy(0, 200);')
+
+            # 轉語音
+            browser.find_element(By.XPATH, '//*[@id="button1"]').click()
+            time.sleep(len(text)*0.5)
+
+            audio_element = browser.find_element(By.XPATH, '//*[@id="audio1"]')
+            time.sleep(0.2)
+            # # 取得 <audio> 標籤的屬性（例如 src）
+            audio_src = audio_element.get_attribute('src')
+            print("音頻來源:", audio_src)
+
+            # 下載音檔
+            download_blob(browser, audio_src, file_name)
+
+            # check 是否下載完成
+            file_path = os.path.normpath(os.path.join(download_folder, file_name))
+            while not os.path.exists(file_path):
+                print('...')
+                time.sleep(0.01)
+            print(file_path)
+            # destination_path = os.path.join(download_folder, datetime.now().strftime(f"%Y%m%d%H%M%S_{''.join(random.sample(string.ascii_lowercase, 3))}.wav"))
+            # shutil.move(file_path, destination_path)
+            print(f"下載完成: {file_path}")
+
+            print(time.time() - start_time)
+
+            return file_path
+
+        except Exception as e:
+            print(f'Error: {e}')
+            return e
+            
+            
+tts_downloadfile('沒問題，可以試著從包包自己拿出來放桌上，也可以請老師協助你唷')
+

台語tts.py

@@ -0,0 +1,238 @@
+import os
+# import requests
+# from bs4 import BeautifulSoup
+from selenium.webdriver.common.by import By
+# import re 
+import time
+# from fake_useragent import UserAgent
+import undetected_chromedriver as uc
+from datetime import datetime
+import random
+import string
+# from selenium.webdriver.common.action_chains import ActionChains
+# from selenium.webdriver.common.keys import Keys
+# from selenium.webdriver.support.ui import WebDriverWait
+# from selenium.webdriver.support import expected_conditions as EC
+# from ga4mp import FirebaseMP
+from dotenv import load_dotenv
+import os
+import shutil
+import logging
+from fastapi import APIRouter, FastAPI
+import uvicorn
+from fastapi.middleware.cors import CORSMiddleware
+from supabase import create_client, Client
+
+load_dotenv()
+
+logging.basicConfig(level=logging.INFO)
+
+SUPABASE_URL: str =  os.environ.get('SUPABASE_URL')
+SUPABASE_KEY: str = os.environ.get('SUPABASE_KEY')
+supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
+
+def download_blob(browser, blob_url, filename='output.mp3'):
+    # 使用 Selenium 獲取 Blob 內容
+    js_code = f"""
+    fetch('{blob_url}')
+        .then(response => response.blob())
+        .then(blob => {{
+            const url = URL.createObjectURL(blob);
+            const a = document.createElement('a');
+            a.href = url;
+            a.download = '{filename}';
+            document.body.appendChild(a);
+            a.click();
+            document.body.removeChild(a);
+        }})
+        .catch(error => console.error('Error downloading file:', error));
+    """
+    browser.execute_script(js_code)
+
+def tts_downloadfile(text):
+    start_time = time.time()
+    print(f'text長度: {len(text)}')
+    num = random.randint(3,5)
+    url = 'http://tts001.iptcloud.net:8804/'
+
+    default_download_folder = os.path.join(os.path.expanduser('~'), 'Downloads')
+    download_folder = '/var/www/html/innolux/tts_folder'
+    file_name = datetime.now().strftime(f"%Y%m%d%H%M%S_{''.join(random.sample(string.ascii_lowercase, 3))}.mp3")
+    # 替換為你希望的文件夾路徑
+    os.makedirs(download_folder, exist_ok=True)
+
+    options = uc.ChromeOptions()
+    options.add_argument('--ignore-certificate-errors')
+    prefs = {
+        "download.default_directory": download_folder,  # 設定預設下載文件夾
+        "download.prompt_for_download": False,
+        "safebrowsing.enabled": True,  # 確保安全瀏覽
+    }
+    options.add_experimental_option("prefs", prefs)
+    # options.add_argument('--incognito')
+    options.add_argument('--headless')  # 如果不想顯示瀏覽器可以啟用這行
+    options.add_argument("--disable-gpu")  # 禁用 GPU 加速
+
+    # 設置自定義 headers
+    with uc.Chrome(options=options, version_main=129) as browser:
+        try:           
+            browser.get(url)
+
+            time.sleep(num)
+            # 轉中文成台語拼音
+            browser.find_element(By.XPATH, '//*[@id="js-input"]').send_keys(text)
+            time.sleep(0.1)
+            browser.find_element(By.XPATH, '//*[@id="js-translate"]').click()
+            time.sleep(0.1 + len(text)*0.01)
+
+            browser.execute_script('window.scrollBy(0, 200);')
+
+            # 轉語音
+            browser.find_element(By.XPATH, '//*[@id="button1"]').click()
+            # time.sleep(len(text)*0.6)
+
+            # audio_element = browser.find_element(By.XPATH, '//*[@id="audio1"]')
+            # time.sleep(0.2)
+            # # 取得 <audio> 標籤的屬性（例如 src）
+            audio_src = None
+            while not audio_src:
+                if time.time() - start_time > 45:
+                    return 'Time exceeded'
+            
+                audio_element = browser.find_element(By.XPATH, '//*[@id="audio1"]')
+                audio_src = audio_element.get_attribute('src')
+                
+                if audio_src:
+                    print("音頻來源:", audio_src)
+                    download_blob(browser, audio_src, file_name)
+                else:
+                    print("尚未檢測到音頻，繼續等待...")
+                    time.sleep(0.3)  # 每隔 0.3 秒檢測一次
+
+
+
+            # 下載音檔
+            # download_blob(browser, audio_src, file_name)
+            is_default = False
+            # check 是否下載完成
+            file_path = '/var/www/html/innolux/tts_folder' + '/' + file_name
+            default_file_path = '/root/Downloads' + '/' + file_name
+            while not os.path.exists(file_path):
+                print('...')
+                # if default_file_path:
+                    # is_default = True
+                    # break
+                time.sleep(0.001)
+            # destination_path = os.path.join(download_folder, datetime.now().strftime(f"%Y%m%d%H%M%S_{''.join(random.sample(string.ascii_lowercase, 3))}.wav"))
+            # shutil.move(file_path, destination_path)
+            # if is_default:
+                # while not os.path.exists(default_file_path):
+                    # print('...')
+                    # time.sleep(0.001)
+                # shutil.move(default_file_path, file_path)
+                # print("檔案移動完成")
+            print(f"下載完成: {file_path}")
+            file_path = file_path.split('html/')[1]
+            print(file_path)
+            print(time.time() - start_time)
+
+            return file_path
+
+        except Exception as e:
+            print(f'Error: {e}')
+            return e
+
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.post('/tts')
+async def tts(answer: str):
+    file_path = tts_downloadfile(answer)
+    # if '.mp3' not in file_path:
+        # return {"message": file_path}
+    if file_path:
+        return {'message': {'mp3_url': file_path}}
+    else:
+        return {"message": "tts processing failed."}
+
+
+
+# from apscheduler.schedulers.background import BackgroundScheduler
+# import asyncio
+# import requests
+
+# scheduler = BackgroundScheduler()
+# loop = asyncio.get_event_loop()
+
+# # 巴巴群組
+# def notify_line(id, question, message):
+#     # url = 'http://cmm.ai:3001/api/push/PLSDC1vOG9'
+#     url = 'https://notify-api.line.me/api/notify'
+#     token = 'OtAC4mBxi1tHjFT5RDcCiA8NwNKuxHVOAlZU5iO04XB' # 巴巴工程師群組
+#     headers = {
+#         'Authorization': 'Bearer ' + token
+#     }
+#     # 構造請求的數據
+#     data = {
+#         'message': f"\n群創\nid:【{id}】\n問題: {question}\nmessage: {message}"
+#     }
+    
+#     try:
+#         response = requests.post(url, headers=headers, data=data)          
+#         if response.status_code == 200:
+#             print("Notification sent successfully.")
+#         else:
+#             print(f"Failed to send notification. Status code: {response.status_code}")
+#     except Exception as e:
+#         print(f"An error occurred: {e}")
+
+# def sub_1_minute():
+#     time.sleep(1)
+#     cursor = supabase.table('INNOLUX_cache').select('*').filter('mp3_url', 'is', 'null').order('id', desc=False).execute()
+#     if cursor.data:
+#         data = cursor.data[0]
+#         if data['is_run']:
+#             return
+#         try:
+#             supabase.table('INNOLUX_cache').update({'is_run':True}).eq('question', data['question']).eq('answer', data['answer']).execute()
+#             file_path = tts_downloadfile(data['answer'])
+#             if '.mp3' in file_path:
+#                 supabase.table('INNOLUX_cache').update({'mp3_url':file_path}).eq('question', data['question']).eq('answer', data['answer']).eq('is_run', True).execute()
+#                 print(f'{file_path} 已存入資料庫')
+#                 supabase.table('INNOLUX_cache').update({'is_run':None}).eq('question', data['question']).eq('answer', data['answer']).eq('is_run', True).execute()
+#                 return
+#             else:
+#                 notify_line(data['id'], data['question'], file_path)
+#                 supabase.table('INNOLUX_cache').update({'is_run':None}).eq('question', data['question']).eq('answer', data['answer']).execute()
+#                 print(file_path)
+#                 return
+#         except Exception as e:
+#             print(f'Error: {e}')
+#             supabase.table('INNOLUX_cache').update({'is_run':None}).eq('question', data['question']).eq('answer', data['answer']).execute()
+#     else:
+#         return
+
+
+
+# # 添加定时任务
+# scheduler.add_job(sub_1_minute, 'interval', minutes=0.8)
+# scheduler.start()
+
+# @app.on_event("shutdown")
+# def shutdown_event():
+#     scheduler.shutdown()
+
+
+
+
+if __name__ == '__main__':
+    uvicorn.run("台語tts:app", reload=False, port=8093, host='cmm.ai', ssl_keyfile="/etc/letsencrypt/live/cmm.ai/privkey.pem", ssl_certfile="/etc/letsencrypt/live/cmm.ai/fullchain.pem")