Merge branch 'master' of http://git.choozmo.com:3000/choozmo/kw_tools

.gitignore

@@ -0,0 +1 @@
+userdata.*

SEO/feature_snippets.py

@@ -0,0 +1,69 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from setting import rua
+import urllib
+import time
+import csv
+from random import randint
+import dataset
+import pandas as pd
+import pymysql
+
+
+def restart_browser():
+    s = Service('/Users/zooeytsai/Downloads/chromedriver 4')
+    options = webdriver.ChromeOptions()
+    options.add_argument("user-agent=%s" % rua())
+    options.add_argument('--headless')
+    options.add_argument("--incognito")
+    driver = webdriver.Chrome(options=options, service=s)
+    str1 = driver.capabilities['chrome']['chromedriverVersion'].split(' ')[0]
+    print('這裡',str1)
+    driver.delete_all_cookies()
+    driver.set_window_size(950, 20000)
+    return driver
+
+
+
+def read_csv():
+    lst = []
+    df = pd.read_csv('/Users/zooeytsai/Documents/幸福空間FAQ關鍵字.csv')
+    for i, row in df.iterrows():
+        lst.append(row['熱門查詢項目'])
+    return lst
+
+lst = read_csv()
+for term in lst:
+    print(term)
+    driver = restart_browser()
+    escaped_search_term = urllib.parse.quote(term)
+    url = 'https://mops.twse.com.tw/mops/web/t135sb03'
+    driver.get(url)
+    time.sleep(6)
+
+    df = pd.DataFrame()
+    
+    elmts = driver.find_elements(By.XPATH, "/html/body/div[7]/div/div[10]/div/div[2]/div[2]/div/div/div[1]/block-component/div/div[1]/div/div/div/div/div[1]/div/div/div/div/div/div[2]/div/div/div[1]/a")
+
+    datalist = []
+    print(len(elmts))
+    for elmt in elmts:
+        href = elmt.get_attribute('href')
+        txt = elmt.text
+        print(txt)
+        
+        datalist.append([term,elmt.text,href])
+
+
+    with open('/Users/zooeytsai/Documents/幸福空間FAQ統計.csv','a') as f:
+        writer = csv.writer(f)
+        for i in datalist:
+            print(i)
+            writer.writerow(i)
+
+    
+    driver.quit()
+    
+    print('中場休息')
+    time.sleep(randint(45, 50))

SEO/seo_monitor.py

@@ -15,6 +15,10 @@ db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb
 
 data = pd.DataFrame()
 
+scope = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive']
+credentials = ServiceAccountCredentials.from_json_keyfile_name("choozmointernal-2e314f3d4e12.json", scope)
+gc = gspread.authorize(credentials)
+
 def notify_group(msg):
     gid='WekCRfnAirSiSxALiD6gcm0B56EejsoK89zFbIaiZQD'
     headers = {"Authorization": "Bearer " + gid,"Content-Type": "application/x-www-form-urlencoded"}
@@ -84,28 +88,55 @@ def arrange_date(date):
         date = date[:2]+'/'+date[2:]
     return date
 
-def checkrank(data):
+def checkrank(data, targetcust, targetkw):
+    '''
     msg = "以下是排名下降的關鍵字：\n"
     msg2 = "以下是目前排名於百名外的關鍵字：\n"
+    '''
+    msg = "以下是亮點字的績效：\n"
     flag1=0
     flag2=0
     for row in data.index:
             #print(row)
             delta = int(data['starting_rank'][row])-int(data['current_rank'][row])
+
+            for t in targetkw:
+                try:
+                    w = data['kw'][row]
+                    if ((t in w) and data['client'][row] in targetcust):
+                        msg += "[" + data['client'][row] + "] " + data['kw'][row] + ": " + str(int(data['starting_rank'][row])) + " -> " + str(int(data['current_rank'][row])) + " (" + str(int(delta)) + ")\n"
+                except:
+                    pass
+
+            '''
             if (delta < 0):
                 msg += "[" + data['client'][row] + "] " + data['kw'][row] + ": " + str(int(data['starting_rank'][row])) + " -> " + str(int(data['current_rank'][row])) + " (" + str(int(delta)) + ")\n"
                 flag1=1
             if (int(data['current_rank'][row]) == 101):  #out of range...
                 msg2 += "[" + data['client'][row] + "] " + data['kw'][row] + "\n"
                 flag2=1
+            '''
     print(msg)
-    print(msg2)
-    return msg, msg2, flag1, flag2
+    #print(msg2)
+    return msg
+    #return msg, msg2, flag1, flag2
 
 if __name__ == "__main__":
     date1 = '0707'
     date2 = '0717'
-    lst = ['清原', '神助物流', '班尼斯', '有夠讚', '仁本', '火柴星人', '呷茶']
+    lst = ['清原', '神助物流', '班尼斯', '有夠讚', '仁本', '火柴星人', '呷茶', 'ChoozMo']
+
+    sheet = gc.open('SEO 亮點字')
+    sheet_instance = sheet.get_worksheet(0)
+    records_data = sheet_instance.get_all_records()
+    #print(records_data[0])
+
+    targetkw = []
+    for rd in records_data:
+        targetkw.append(rd["亮點字"])
+
+    print(targetkw)
+
     for client in lst:
         date1 = '0707'
         date2 = '0718'
@@ -115,9 +146,7 @@ if __name__ == "__main__":
         data = data.append(res, ignore_index=True)
     data.to_csv(f"./排名CSV/combine/__排名變化.csv", index=False, encoding='utf_8_sig')
 
-    scope = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive']
-    credentials = ServiceAccountCredentials.from_json_keyfile_name("choozmointernal-2e314f3d4e12.json", scope)
-    gc = gspread.authorize(credentials)
+    
     spread = Spread("kw_ranking_tracker",creds=credentials)
     spread.df_to_sheet(data, index=False, sheet="main", start="A1", replace=False)
     time_stamp = datetime.now()
@@ -125,11 +154,18 @@ if __name__ == "__main__":
     
     spread.sheet.update('G1', '最後更新:' + time_stamp)
     
-    msg, msg2, flag1, flag2=checkrank(data)
+    #msg, msg2, flag1, flag2=checkrank(data, lst, targetkw)
+    msg = checkrank(data, lst, targetkw)
 
     msg += "\n完整資料請至https://cmm.ai/seo1"
-    msg2 += "\n完整資料請至https://cmm.ai/seo1"
+    #msg2 += "\n完整資料請至https://cmm.ai/seo1"
+
+    msg01 = [msg[i:i+1000] for i in range(0, len(msg), 1000)]
+    for m1 in msg01:
+        notify_group(m1)
+        print(m1)
 
+    '''
     if flag1 == 1:
         msg01 = [msg[i:i+1000] for i in range(0, len(msg), 1000)]
         for m1 in msg01:
@@ -140,4 +176,5 @@ if __name__ == "__main__":
         msg02 = [msg2[i:i+1000] for i in range(0, len(msg2), 1000)]
         for m2 in msg02:
             notify_group(m2)
-            print(m2)
+            print(m2)
+    '''

choozmo/bpm_gen.py

@@ -0,0 +1,100 @@
+import requests
+import datetime
+from dateutil import parser
+import gspread
+from oauth2client.service_account import ServiceAccountCredentials
+import time
+import sys
+import dataset
+import codecs
+import networkx as nx
+
+db = dataset.connect('sqlite:///:memory:')
+table=db['tmp']
+# drop this at PUBLIC2
+
+# TASK LIST HERE
+# https://docs.google.com/spreadsheets/d/153w046qHhpnZTYU7FJ-S2xyGIH_BXbycLYfLI2fFsZc/edit#gid=0
+
+scope = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive']
+creds = ServiceAccountCredentials.from_json_keyfile_name('c:/keys/choozmointernal-2e314f3d4e12.json', scope)
+client = gspread.authorize(creds)
+message = "提醒近期預定執行的作業如下：\n\n"
+message2 = "提醒近期需完成的作業如下：\n\n"
+
+def notify_group(msg):
+    gid='WekCRfnAirSiSxALiD6gcm0B56EejsoK89zFbIaiZQD'
+    headers = {"Authorization": "Bearer " + gid,"Content-Type": "application/x-www-form-urlencoded"}
+    print(msg)
+#    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params={"message": msg})
+
+sheet = client.open('ChoozMo 專案執行表單_20220720')
+sheet_instance = sheet.get_worksheet(0)
+
+records_data = sheet_instance.get_all_records()
+#print(records_data[0])
+
+rowidx=1
+for data in records_data:
+    data['rownum']=rowidx
+    print(data['完成日'].strip())
+    print(len(data['完成日'].strip()))
+
+    if len(data['完成日'].strip())<=2:
+        dtstr=None
+        data['完成日']=None
+    elif len(data['完成日'].strip())<8:
+        dtstr='2022/'+data['完成日']
+    if dtstr is not None:
+        data['完成日']=datetime.datetime.strptime(dtstr, '%Y/%m/%d')
+#        print(datetime.datetime.strptime(dtstr, '%Y/%m/%d'))
+#    data['完成日']=
+
+    table.insert(data)
+    rowidx+=1
+#cursor=db.query("select *, Cast (julianday('now') - julianday(完成日) as integer) as dt from tmp order by 專案來源,專案")
+cursor=db.query("select *, julianday('2022-08-01') - julianday(完成日)  as dt from tmp order by 專案來源,專案")
+
+#[#red]
+
+fw=codecs.open('C:\\gitlab\\worklog\\docs\\test.md','w','utf-8')
+fw.write('# example \n\n')
+fw.write('```plantuml\n')
+fw.write('@startmindmap\n')
+fw.write('+ 專案管理\n')
+
+G = nx.DiGraph()
+prev=None
+for c in cursor:
+    G.add_edge('root', c['專案來源'])
+    G.add_edge(c['專案來源'], c['專案'])
+    if c['專案項目'] is not None:
+        if len(c['專案項目'].strip())>0:
+            if c['dt'] is not None and c['dt']>=0:
+                G.add_edge(c['專案'], '過期:'+c['專案項目'])
+            else:
+                G.add_edge(c['專案'], c['專案項目'])
+
+
+l1=list(G.successors('root'))
+print(l1)
+for l2 in l1:
+    fw.write('++ '+l2+'\n')
+    print(l2)
+    l3=list(G.successors(l2))
+    for l4 in l3:
+        print(l4)
+        fw.write('+++ '+l4+'\n')
+        l5=list(G.successors(l4))
+        for l6 in l5:
+            print(l6)
+            if '過期:' in l6:
+                fw.write('++++[#red] '+l6+'\n')
+            else:
+                fw.write('++++_ '+l6+'\n')
+
+fw.write('@endmindmap\n')
+fw.write('```\n')
+fw.close()
+
+sys.exit()

hhh/SEO/local_1777.py

@@ -47,7 +47,7 @@ def re_get_webdriver():
 
         options.add_argument("--no-sandbox")
         options.add_argument("--disable-dev-shm-usage")
-        options.add_argument("--headless")
+#        options.add_argument("--headless")
 
 #        print(user_agent)
 #        options.add_argument("--user-agent=" +user_agent)
@@ -61,7 +61,7 @@ def re_get_webdriver():
             os.system('kill %d' % os.getpid())
             sys.exit()
             return
-        driver.set_window_size(1400,1000)
+#        driver.set_window_size(1400,1000)
         return
     except:
         import traceback

joblist/jobcheck.py

@@ -4,6 +4,9 @@ from dateutil import parser
 import gspread
 from oauth2client.service_account import ServiceAccountCredentials
 import time
+import smtplib
+from userdata import *
+from email.message import EmailMessage
 
 # drop this at PUBLIC2
 
@@ -21,60 +24,122 @@ def notify_group(msg):
     headers = {"Authorization": "Bearer " + gid,"Content-Type": "application/x-www-form-urlencoded"}
     r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params={"message": msg})
 
+def send_line(messages):
+    for message in messages:
+        msg = [message[i:i+1000] for i in range(0, len(message), 1000)]
+        for m in msg:
+            notify_group(m)
+            print(m)
+
+def send_email(emails, content, has_overdue):
+    server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
+    server.ehlo()
+    server.login(gmail_username, gmail_password)
+
+    for name in content:
+        msg = EmailMessage()
+        msg['Subject'] = '執行表單過期作業通知' + str(datetime.date.today())
+        msg['From'] = "service@choozmo.com"
+        msg['To'] = emails[name]
+
+        if has_overdue[name] == 1:
+            msg.set_content(content[name])
+        else:
+            msg.set_content(name + "您好，\n\n您目前沒有過期的作業。\n更新執行表單請至 https://cmm.ai/pm ，謝謝您。\n\n")
+        
+        print ('Sending email to ' + msg['To'])
+        server.send_message(msg)
+        print ('Email sent to ' + msg['To'])
+        fname = name+'.txt'
+        with open(fname, 'w', encoding="UTF+8") as f:
+            if has_overdue[name] == 1:
+                f.write(content[name])
+            else:
+                f.write(name + "您好，\n\n您目前沒有過期的作業。\n更新執行表單請至 https://cmm.ai/pm ，謝謝您。\n\n")
+
+    server.close()
+
 sheet = client.open('ChoozMo 專案執行表單_20220720')
 sheet_instance = sheet.get_worksheet(0)
 
 records_data = sheet_instance.get_all_records()
 #print(records_data[0])
 
+emails = {"Jared": "jared@choozmo.com", "Morrison": "morrison@choozmo.com", "Ginia": "ginia@choozmo.com", "Doris": "doris@choozmo.com", "Nina": "nina.huang@choozmo.com", "Yukyo": "yukyo@choozmo.com", "Andy": "andy@choozmo.com", "Oak": "oak@choozmo.com", "Zooey": "zooey@choozmo.com", "Syuanyuri": "syuanyuri@choozmo.com", "Wizer": "wizer@choozmo.com", "Jason": "jason@choozmo.com", "Tomoya": "tomoya@choozmo.com", "詠心": "noodlesloves@gmail.com", "献竤": "fxp87257@gmail.com"}
+overdue_email_content = {}
+has_overdue = {}
+
+for e in emails.keys():
+    overdue_email_content[e] = e + "您好，\n\n提醒您有已過期的作業，請儘快完成以下作業。\n更新執行表單請至 https://cmm.ai/pm ，謝謝您。\n\n過期作業如下：\n\n"
+    has_overdue[e] = 0
+
 for data in records_data:
     try:
-        print(parser.parse(data["完成日"]))
-        print(datetime.date.today())
+        print(parser.parse(data["起始日"]), parser.parse(data["完成日"]))
         deltastart = (parser.parse(data["起始日"]).date()-datetime.date.today()).days
         deltaend = (parser.parse(data["完成日"]).date()-datetime.date.today()).days
         print(deltastart, deltaend)
+        info = ""
 
         if(deltastart <= 7 and data["完成度"] == "預定執行"):
-            message += "專案：" + data["專案"] + "\n"
-            message += "專案項目：" + data["專案項目"] + "\n"
-            message += "作業：" + data["執行項目"] + "\n"
-            message += "執行人員：" + data["指派對象"] + "\n"
-            message += "PM：" + data["PM"] + "\n"
-            message += "執行期間：" + data["起始日"] + "-" + data["完成日"]
+            info += "專案：" + data["專案"] + " / " + data["專案項目"] + "\n"
+            info += "作業：" + data["執行項目"] + "\n"
+            info += "執行人員：" + data["指派對象"] + "\n"
+            info += "PM：" + data["PM"] + "\n"
+            info += "預定執行期間：" + data["起始日"] + "-" + data["完成日"]
             if(deltastart>0):
-                message += " (尚有"+ str(deltastart) + "天)\n\n"
+                info += " (尚有"+ str(deltastart) + "天)\n\n"
             elif(deltastart==0):
-                message += " (當天開始執行)\n\n"
+                info += " (當天開始執行)\n\n"
             else:
-                message += " (過期"+ str(deltastart).replace('-','') + "天)\n\n"
+                info += " (過期"+ str(deltastart).replace('-','') + "天)\n\n"
+                overdue_email_content[data["PM"]] += info
+                has_overdue[data["PM"]] = 1
+                if [data["PM"]]!=[data["指派對象"]]: #sometimes you have to do the task and you are the PM
+                    overdue_email_content[data["指派對象"]] += info
+                    has_overdue[data["指派對象"]] = 1
+                if data["PM"] != "Jared": #sometimes Jared is the PM
+                    overdue_email_content["Jared"] += info
+                    has_overdue["Jared"] = 1
+            message += info
+
 
         elif(deltaend <= 7 and data["完成度"] == "執行中"):
-            message2 += "專案：" + data["專案"] + "\n"
-            message2 += "專案項目：" + data["專案項目"] + "\n"
-            message2 += "作業：" + data["執行項目"] + "\n"
-            message2 += "執行人員：" + data["指派對象"] + "\n"
-            message2 += "PM：" + data["PM"] + "\n"
-            message2 += "完成日：" + data["完成日"]
+            info += "專案：" + data["專案"] + " / " + data["專案項目"] + "\n"
+            info += "作業：" + data["執行項目"] + "\n"
+            info += "執行人員：" + data["指派對象"] + "\n"
+            info += "PM：" + data["PM"] + "\n"
+            info += "完成日：" + data["完成日"]
             if(deltaend>0):
-                message2 += " (尚有"+ str(deltaend) + "天)\n\n"
+                info += " (尚有"+ str(deltaend) + "天)\n\n"
             elif(deltaend==0):
-                message2 += " (當天完成)\n\n"
+                info += " (當天完成)\n\n"
             else:
-                message2 += " (過期"+ str(deltaend).replace('-','') + "天)\n\n"
+                info += " (過期"+ str(deltaend).replace('-','') + "天)\n\n"
+                overdue_email_content[data["PM"]] += info
+                has_overdue[data["PM"]] = 1
+                if [data["PM"]]!=[data["指派對象"]]: #sometimes you have to do the task and you are the PM
+                    overdue_email_content[data["指派對象"]] += info
+                    has_overdue[data["指派對象"]] = 1
+                if data["PM"] != "Jared": #sometimes Jared is the PM
+                    overdue_email_content["Jared"] += info
+                    has_overdue["Jared"] = 1
+            message2 += info
 
     except:
         print("PASS")
 
-msg1 = [message[i:i+1000] for i in range(0, len(message), 1000)]
-for m1 in msg1:
-    #notify_group(m1) #annotated to prevent unintentional trigger
-    print(m1)
+message += "更新執行表單請至 https://cmm.ai/pm"
+message2 += "更新執行表單請至 https://cmm.ai/pm"
+
+messages = []
+messages.append(message)
+messages.append(message2)
+
+print(has_overdue)
 
-msg2 = [message2[i:i+1000] for i in range(0, len(message2), 1000)]
-for m2 in msg2:
-    #notify_group(m2)
-    print(m2)
+send_line(messages)
+send_email(emails, overdue_email_content, has_overdue)
 
 '''notify_group(message)
 time.sleep(5)