from datetime import date, datetime import pymysql pymysql.install_as_MySQLdb() import dataset import pandas as pd import os import random import re import json import requests from summarizer import Summarizer from transformers import AutoConfig, AutoTokenizer, AutoModel from summarizer.sentence_handler import SentenceHandler from spacy.lang.zh import Chinese from random import sample import csv from gn3 import GoogleNews3 googlenews = GoogleNews3(lang="zh-tw", period="7d", encode="utf-8", region="tw") # from fastapi import FastAPI # app = FastAPI() os.environ["TOKENIZERS_PARALLELISM"] = "false" db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4') result = db.query('SELECT * FROM gnews.weather_detail2') data = pd.DataFrame(result, columns=next(iter(result)).keys()) custom_config = AutoConfig.from_pretrained('bert-base-chinese') custom_config.output_hidden_states=True custom_tokenizer = AutoTokenizer.from_pretrained('bert-base-chinese') custom_model = AutoModel.from_pretrained('bert-base-chinese', config=custom_config) summarizer = Summarizer(custom_model=custom_model, custom_tokenizer=custom_tokenizer, sentence_handler = SentenceHandler(language=Chinese)) df_test = pd.read_csv('./news_text.txt') df = pd.read_csv('./image.csv') def sentence2img(news_text: str): tags = [] for i in range(len(df['tags'])): tags.extend(df['tags'][i].split(',')) images = [] for i in list(set(tags)): if i in news_text: images.extend(list(df[df['tags'].str.contains(i,regex=False)==True]['img_url'])) if images: return images[random.randint(0, len(images)-1)] else: return list(df['img_url'])[random.randint(0, len(list(df['img_url']))-1)] def remove_redundancy(text, words): for w in words: if w in text: text = text.split(w)[0] text = text.replace('\n','').replace('\r','').replace('（','(').replace('）',')') return text def news_clean(text): text = text.replace('u3000','') text = text.replace('u200b','') text = remove_redundancy(text, ['原始連結','更多內容','看更多','參考資料','本文獲','Source','註','點擊瀏覽', '【延伸閱讀】','更多三立','延伸閱讀','新聞來源','更多 TVBS 報導','【更多中天快點TV報導】', '今日各地紫外線預報概況。 【','國際都市氣象','亞洲、大洋洲美洲歐洲、非洲中國各地體感溫度預報']) text = re.sub("[\(\[].*?[\)\]]", "", text) text = re.sub(r'〔.*?／.*?〕', '', text) text = re.sub(r'【 第.*?號.*?地震報告 】', '後面都不要了', text) text = re.sub(r'\w\w\w接受《.*?》訪問指出，', '', text) text = re.sub(r'\w\w\w「.*?」.*?，', '', text) text = re.sub(r'對此，.*?指出，', '', text) text = re.sub(r'臉書.*?｜.*?表示，', '', text) text = re.sub(r'氣象粉專.*?提醒，', '', text) text = re.sub(r'請繼續往下閱讀...', '', text) text = re.sub(r'\w\w\w說，', '', text) text = re.sub(r'\w\w\w也提醒，', '', text) text = re.sub(r'\w\w\w分析，', '', text) text = re.sub(r'圖：.*?/攝', '', text) text = re.sub(r'圖：.*?／提供', '', text) text = re.sub(r'圖／.*?中心', '', text) text = re.sub(r'三立氣象主播\w\w\w', '', text) text = re.sub(r'記者.*?／攝影', '', text) text = re.sub(r'三立.*?／.*?報導', '', text) text = re.sub(r'生活中心／.*?報導', '', text) text = re.sub(r'\w\w\w / .*?報導', '', text) text = re.sub(r'記者.*?／.*?報導', '', text) text = re.sub(r'（.*?提供）', '', text) text = re.sub(r'（圖／.*?）', '', text) text = re.sub(r'\w在《.*?》.*?撰文，', '', text) text = re.sub(r'氣象專家.*?認為，', '', text) text = re.sub(r'\w\w\w表示，', '', text) text = re.sub(r'※點圖放大', '', text) text = re.sub(r'綜合報導 / \w\w\w', '', text) text = re.sub(r'【.*?-.*?】', '', text) text = re.sub(r'不用抽.*?活動辦法', '', text) text = re.sub(r'◤.*?◢?.*?酒店', '', text) text = re.sub(r'\d+/\d+/\d+ \d+:\d+', '', text) text = re.sub(r'▼(\w*\w*。)', '', text) text = re.sub(r'▲(\w*\w*。)', '', text).replace('▲','').replace('▼','') text = text.replace('xa0','') text = remove_redundancy(text, ['後面都不要了']) return text def gen_daily_news(data, date, summarizer): text_content_list = [] image_urls_list = [] news_date = data.groupby("news_date") print("-"*100) print(date) news_content_list = [] news_imgs_list = [] for idx, news in enumerate(news_date.get_group(date).iterrows()): # clean news content news_content = news_clean(news[1]['news_content']) # compress content if len(news_content)>1000: news_content = summarizer(news_content, ratio=0.8) if len(news_content) > 180 and len(news_content) < 800: news_content_list.append(news_content) news_imgs_list.append(news[1]['news_imgs']) error_count = 0 zero_count = 0 news = '' while True: try: text = ''.join(sample(news_content_list,min(4,len(news_content_list)))) text = text.replace('.','·') # print(text) ratio = 400 / len(text) # print(ratio) news = news_clean(summarizer(text, ratio=ratio)) print('字數:',len(news)) if zero_count > 5: print('So sad! Something wrong!') print('-'*80) break if len(news) != 0: news_split = [n.strip()+'。' for n in news.split('。')] image_urls_list = sample(image_urls_list, min(len(image_urls_list),len(news_split[:-1]))) text_content_list = news_split[:-1] break if len(news) <120: print('Regenerating...') zero_count = zero_count+1 except: print('Regenerating...') error_count = error_count+1 if error_count>5: print('So sad! Something wrong!') print('-'*80) break image_urls_list = [] for t in text_content_list: # print(t) image_urls_list.append(sentence2img(t)) return text_content_list, image_urls_list # name_date = '2021年11月01日' # t, i = gen_daily_news(data, name_date, summarizer) kw = input("關鍵字：") googlenews.clear() googlenews.get_news(kw) trdresult = googlenews.result() """ for i in range(25,26): if i <10: name_date = '2021年11月0' + str(i) + '日' else: name_date = '2021年11月' + str(i) + '日' """ #t, i = gen_daily_news(data, '2021年11月25日', summarizer) t, i = [trdresult[0]['title']], [trdresult[0]['img']] url = 'https://www.choozmo.com:8887/make_anchor_video' headers = {'Authorization':"Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ5dWt5byIsImlhdCI6MTYzNzkwNjc0NCwibmJmIjoxNjM3OTA2NzQ0LCJqdGkiOiIwZmM2MjQ5MS1kNzFmLTRjNzktYTk1MC1iZmVjOGYyNGI1ODUiLCJleHAiOjE2MzgzMzg3NDQsInR5cGUiOiJhY2Nlc3MiLCJmcmVzaCI6ZmFsc2UsImNzcmYiOiIxYmEwYjU4MC03OTU1LTQ4MmMtYjNiYi0yZjhmYmMyZjg5YjYifQ.MUT8tXk2v_M2nI5fxJewSh2kIi2-uh-vWAj9Y3N2niE", 'Content-Type': 'application/json'} payload = { "name": "Demo_1_" + kw + '_' + datetime.now().strftime('%Y%m%d%H%M%S'), "text_content": t, "image_urls": i, "avatar": "7", "client_id": "-1", "multiLang": 0 } #print(t) #print(i) r = requests.request("POST", url, data=json.dumps(payload), headers=headers) print(r.text) # @app.get('/') # async def genlist(date:date, flag: int): # return { # "text" : [ # "中央氣象局指出，今天天氣與昨天類似，降雨範圍以基隆北海岸與東半部為主，颱風消息方面，第20號輕位於台灣東方約1910公里的海面上，未來朝日本東方海面前進，對台灣天氣無直接影響，但基隆北海岸、東半部及恆春半島沿海易有長浪發生。", # "氣象局預估，週五至週六清晨東北季風影響，北部、東北部早晚較涼，降雨方面在迎風面的基隆北海岸、東半部及大台北地區有局部短暫雨，其他地區為多雲到晴，午後南部地區有局部短暫陣雨。", # "天氣風險公司總經理彭啟未來7-10天都維持類似的天氣，下一波變化有些預測落在11月中上旬，約11/10-11/12，北方冷高壓有機會再度南下，是否如某些預測大降溫，但這變數仍很多，保持關注就可以了。", # "氣象局說明，今東北季風稍微增強，然而水氣沒有顯著增加，因此天氣表現與昨日類似，只是早晚稍微又涼些，清晨中部以北及東北部約20、21度，花東及南部地區約23度，降雨範圍依舊以迎風面的基隆北海岸及東半部的局部短暫雨為主，且大台北地區雨較零星，其他地方則為多雲到晴，午後南部地區有局部短暫陣雨的機率，但範圍跟程度都影響不大。" # ], # "imgurl" : [ # "https://cdn2.ettoday.net/images/5930/d5930739.jpg", # "https://cdn2.ettoday.net/images/5917/d5917494.jpg", # "https://cdn2.ettoday.net/images/5915/d5915515.jpg", # "https://cdn2.ettoday.net/images/5931/d5931569.jpg" # ] # }