add SEO monitoring

SEO/choozmointernal-2e314f3d4e12.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "choozmointernal",
+  "private_key_id": "2e314f3d4e1249f255cbc0668a8ce4ccf1cbd6b9",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCro3/iYoVWz5E3\nFeO9VtjJo5d9tRDr+prS9ebcIKiVYVD8GgxYKetdzKdnCMhzPGKR4I4iGCC4OU2W\n8xOk1xXVneOouXhYESMB/GpJEspXQCgGHoRrioBASDxtC4BIpfXvyFpr0rl/Nk2F\ni8ihQM0Sa2qZDbBnJIMAqHDQMje6tZs5A3B8svtdXg60N33JU+bPshG5EqbJ0ttq\nPa+66wtsxgzdyTx287/4dgEZCc/CB7Oh8bRhdqAgBT8ZQjW6KEXa4jDs2ny8TTfZ\njpeZaZWE5qpeConhmjNjiL5IKnjjEkQvIYKxDAMEmSRVb4DI3OlE+nZLtg0ETebb\nJ+scoa8hAgMBAAECggEAVI7tLvu4kycZdJj9JMeHb/d/Z7fviUJiSjJJNQOF6moS\nOtEeacTlJsXvZwxlrJgAO7v0agmW/QuGFSrV76zjHeV2RiZxinmO1KilODDgrJF3\n5mPcexq1ynWMB3DWF3RjsGj1Ww9qwahgEr3l7sN9BGkVECg1Uusi+w0A3yEOYZXM\ndDUUx3k4x+9UTHXz0VJ/uL0SZ7yp/aG47erilgpNuZ39J2/sPv5nBy8xMP2ME4QY\nc+tW/JQVB4eFStmcBPS63WZwbegWKMPfdCgdTJGT1A4lwcyeShmnjkFfytLA4O+y\n8pdz8UQIjgR1ZBuQ29VCA26dTE38jb4xQoAgMI7fPQKBgQDvRSUTktcETUbp/hUn\nTIJIlQDlwsowULQ8SS09r69TfPkwkkMnssZ9YFhKII+FpI+8JmwHSe0RomD0hSDc\nO9fb/D1pYq2yplnA+NeP41oLbXD9nxzHqLpd02hahvOPmjwjn02sEDYjH8j42/2g\nnEsMbGLFXTCuXRhF4U3C8X+czwKBgQC3o8ZQkl8zt7TKqCVE8lqpHyYVAGJICgnF\nqJIF9z7UF8Ob/QTsC1FL+Cw8yv3iffIqwCX5p2gNXzbZrR92qtAiLPiChiKTk2jZ\njXUGxSiFK7PKiwHluNbQWrvkAMCVOZOg54ie6AFdr2l4vKaCBfiVJ9Ahj+1Vncq/\nVIgkmFZRDwKBgQCmw011F/nukKGlQqahrkx7pAZtHNdo10bXYbU7cjBDgy32sWpG\nx+5ZwgWnHIyLe8eGPk06w0aM7NygcA0YEkl/lI2E3vbrNdAGlClBqxa+Mjf3PKeF\nbM7/x/HKbpZg9wGFXml3TBE/Z87dzA44m9YewYgRwMLGNDJPuhlFjqX/twKBgDvS\nURcMJ5EsZeuijMmB/YCcUgaDYoC/LGHEQRVKbqAewaz95m2SXkBWLur64Wu3xXyh\nrX0/SIkanD6aw+YfTL/lUGsMkRTr7zr3dsjmzwVasC3pzItuBCRJem2SxSmXYP0I\nj5vcTh2bt8sKAV/X3BuSpZ1ZSmIajeJ4AgWRzoSHAoGBAIeq13kqPm1Nfr6qFi8b\nDBHDv/4VvyxXLyXlRQHEWDDzrdn3gCHbzPjryUdxU8U3u8OTSra6vAyhFEmIk32/\nTq0vx7XH2AdN8iIwo59Qhmp4g1wPoiXKSjXVxAgK7eCfkDi1p+0CzQulZGz3KRIV\nhucDeA0a767QtKpGDi9hi2iZ\n-----END PRIVATE KEY-----\n",
+  "client_email": "gdrive@choozmointernal.iam.gserviceaccount.com",
+  "client_id": "111207848091817318160",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/gdrive%40choozmointernal.iam.gserviceaccount.com"
+}

SEO/seo_monitor.py

@@ -0,0 +1,140 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+import pandas as pd
+
+from datetime import datetime
+import pymysql
+pymysql.install_as_MySQLdb()
+import dataset
+import requests
+import gspread
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+
+data = pd.DataFrame()
+
+def notify_group(msg):
+    gid='WekCRfnAirSiSxALiD6gcm0B56EejsoK89zFbIaiZQD'
+    headers = {"Authorization": "Bearer " + gid,"Content-Type": "application/x-www-form-urlencoded"}
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params={"message": msg})
+
+def download(client, date, newest=True):
+    """關鍵字清單"""
+    lst_kw = []
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    cursor_kw = db.query(f"SELECT term FROM seo.selected_kw where client='{client}'")
+    for c in cursor_kw:
+        for i in c.values():
+            lst_kw.append(i)
+    print(len(lst_kw))
+    
+    """抓取general_log關鍵字"""
+    last_check_lst = []
+    for i in lst_kw:
+        if newest:
+            cursor_last_check = db.query(f"select kw,ranking from seo.general_log where kw='{i}' order by dt desc limit 1")
+        else:
+            cursor_last_check = db.query(f"select kw,ranking from seo.general_log where kw='{i}' order by dt limit 1")
+        group = []
+        n = 0
+        for c in cursor_last_check:
+            for j in c.values():
+                if 'site' in str(j):
+                    n+=1
+                if n==1 and j==1:
+                    group.append(101)
+                else:
+                    group.append(j)
+            n = 0
+        last_check_lst.append(group)
+    # df2 = pd.DataFrame(not_match_kw_lst,columns=['關鍵字組','搜尋排名'])
+    df3 = pd.DataFrame(last_check_lst, columns=['關鍵字組', '搜尋排名'])
+    # df_all = pd.concat([df, df3])
+    df3.to_csv(f"./排名CSV/{client}當日排名{date}.csv", index=False, encoding='utf_8_sig')
+    db.close()
+
+
+def combine_df(client, date1, date2):
+    df = pd.read_csv(f'./排名CSV/{client}當日排名{date1}.csv')
+    df2 = pd.read_csv(f'./排名CSV/{client}當日排名{date2}.csv')
+    df_result = pd.merge(df, df2, on='關鍵字組', how='outer').fillna(0)
+    date1 = arrange_date(date1)
+    date2 = arrange_date(date2)
+    df_result.columns = ['kw', 'starting_rank', 'current_rank']
+        
+    p = df_result['starting_rank'].tolist()
+    l = df_result['current_rank'].tolist()
+
+    c = [u"\u2197 "+str(abs(int(i)-int(j))) if int(i)-int(j)>0 \
+        else u"\u2198 "+str(abs(int(i)-int(j))) if int(i)-int(j)<0 \
+        else u"\u2192 "+str(abs(int(i)-int(j))) for (i, j) in zip(p, l)]
+
+    df_result['client'] = client
+    df_result['change'] = c
+    df_result['kw'] = df_result['kw'].str.split('site',0).str[0]
+    df_result.to_csv(f"./排名CSV/combine/{client}排名變化.csv", index=False, encoding='utf_8_sig')
+    return df_result, date1, date2
+
+def arrange_date(date):
+    if date[0] == 0 :
+        date = date[1]+'/'+date[2:]
+    else:
+        date = date[:2]+'/'+date[2:]
+    return date
+
+def checkrank(data):
+    msg = "以下是排名下降的關鍵字：\n"
+    msg2 = "以下是目前排名於百名外的關鍵字：\n"
+    flag1=0
+    flag2=0
+    for row in data.index:
+            #print(row)
+            delta = int(data['starting_rank'][row])-int(data['current_rank'][row])
+            if (delta < 0):
+                msg += "[" + data['client'][row] + "] " + data['kw'][row] + ": " + str(int(data['starting_rank'][row])) + " -> " + str(int(data['current_rank'][row])) + " (" + str(int(delta)) + ")\n"
+                flag1=1
+            if (int(data['current_rank'][row]) == 101):  #out of range...
+                msg2 += "[" + data['client'][row] + "] " + data['kw'][row] + "\n"
+                flag2=1
+    print(msg)
+    print(msg2)
+    return msg, msg2, flag1, flag2
+
+if __name__ == "__main__":
+    date1 = '0707'
+    date2 = '0717'
+    lst = ['清原', '神助物流', '班尼斯', '有夠讚', '仁本', '火柴星人', '呷茶']
+    for client in lst:
+        date1 = '0707'
+        date2 = '0718'
+        download(client, date1, newest=False)
+        download(client, date2, newest=True)
+        res, d1, d2 = combine_df(client, date1, date2)
+        data = data.append(res, ignore_index=True)
+    data.to_csv(f"./排名CSV/combine/__排名變化.csv", index=False, encoding='utf_8_sig')
+
+    scope = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive']
+    credentials = ServiceAccountCredentials.from_json_keyfile_name("choozmointernal-2e314f3d4e12.json", scope)
+    gc = gspread.authorize(credentials)
+    spread = Spread("kw_ranking_tracker",creds=credentials)
+    spread.df_to_sheet(data, index=False, sheet="main", start="A1", replace=False)
+    time_stamp = datetime.now()
+    time_stamp = time_stamp.strftime("%Y-%m-%d %H:%M:%S")
+    
+    spread.sheet.update('G1', '最後更新:' + time_stamp)
+    
+    msg, msg2, flag1, flag2=checkrank(data)
+
+    if flag1 == 1:
+        msg01 = [msg[i:i+1000] for i in range(0, len(msg), 1000)]
+        for m1 in msg01:
+            notify_group(m1)
+            print(m1)
+    
+    if flag2 == 1:
+        msg02 = [msg2[i:i+1000] for i in range(0, len(msg2), 1000)]
+        for m2 in msg02:
+            notify_group(m2)
+            print(m2)