update stuff and add clickjob type firefox

SEO/seo_monitor.py

@@ -10,6 +10,9 @@ import requests
 import gspread
 from gspread_pandas import Spread, Client
 from oauth2client.service_account import ServiceAccountCredentials
+import smtplib
+from userdata import *
+from email.message import EmailMessage
 
 db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
 
@@ -24,6 +27,31 @@ def notify_group(msg):
     headers = {"Authorization": "Bearer " + gid,"Content-Type": "application/x-www-form-urlencoded"}
     r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params={"message": msg})
 
+def send_email(message):
+    server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
+    server.ehlo()
+    server.login(gmail_username, gmail_password)
+
+    emails = {"Jared": "jared@choozmo.com", "Morrison": "morrison@choozmo.com", "Ginia": "ginia@choozmo.com", "Doris": "doris@choozmo.com"}
+    
+    for name in emails:
+        msg = EmailMessage()
+        msg['Subject'] = 'SEO亮點字績效通知' + str(datetime.date.today())
+        msg['From'] = "service@choozmo.com"
+        msg['To'] = emails[name]
+
+        
+        msg.set_content(name + "您好，\n\n" + message)
+        
+        print ('Sending email to ' + msg['To'])
+        server.send_message(msg)
+        print ('Email sent to ' + msg['To'])
+        fname = name+'.txt'
+        with open(fname, 'w', encoding="UTF+8") as f:
+            f.write(name + "您好，\n\n" + message)
+
+    server.close()
+
 def download(client, date, newest=True):
     """關鍵字清單"""
     lst_kw = []
@@ -165,6 +193,7 @@ if __name__ == "__main__":
         notify_group(m1)
         print(m1)
 
+    send_email(msg)
     '''
     if flag1 == 1:
         msg01 = [msg[i:i+1000] for i in range(0, len(msg), 1000)]

joblist/jobcheck.py

@@ -18,6 +18,7 @@ creds = ServiceAccountCredentials.from_json_keyfile_name('choozmointernal-2e314f
 client = gspread.authorize(creds)
 message = "提醒近期預定執行的作業如下：\n\n"
 message2 = "提醒近期需完成的作業如下：\n\n"
+message3 = "提醒以下已取消的作業需填寫取消原因：\n\n"
 
 def notify_group(msg):
     gid='WekCRfnAirSiSxALiD6gcm0B56EejsoK89zFbIaiZQD'
@@ -26,36 +27,44 @@ def notify_group(msg):
 
 def send_line(messages):
     for message in messages:
-        msg = [message[i:i+1000] for i in range(0, len(message), 1000)]
-        for m in msg:
-            notify_group(m)
-            print(m)
-
-def send_email(emails, content, has_overdue):
+        print(len(message))
+        if(len(message)>55):
+            msg = [message[i:i+1000] for i in range(0, len(message), 1000)]
+            for m in msg:
+                notify_group(m)
+                #print(m)
+
+def send_email(emails, unstarted, incomplete, canceled, has_unstarted, has_incomplete, has_canceled):
     server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
     server.ehlo()
     server.login(gmail_username, gmail_password)
 
-    for name in content:
+    for name in emails:
         msg = EmailMessage()
         msg['Subject'] = '執行表單過期作業通知' + str(datetime.date.today())
         msg['From'] = "service@choozmo.com"
         msg['To'] = emails[name]
 
-        if has_overdue[name] == 1:
-            msg.set_content(content[name])
+        message = name + "您好，\n\n以下通知您目前作業現況，更新執行表單請至 https://cmm.ai/pm ，謝謝您。\n\n"
+
+        if has_unstarted[name] == 0 and has_overdue[name] == 0:
+            message += "您目前沒有過期的作業。\n\n"
         else:
-            msg.set_content(name + "您好，\n\n您目前沒有過期的作業。\n更新執行表單請至 https://cmm.ai/pm ，謝謝您。\n\n")
+            if has_unstarted[name] == 1:
+                message += unstarted[name]
+            if has_incomplete[name] == 1:
+                message += incomplete[name]
         
+        if has_canceled[name] == 1:
+            message += canceled[name]
+
+        msg.set_content(message)
         print ('Sending email to ' + msg['To'])
         server.send_message(msg)
         print ('Email sent to ' + msg['To'])
         fname = name+'.txt'
         with open(fname, 'w', encoding="UTF+8") as f:
-            if has_overdue[name] == 1:
-                f.write(content[name])
-            else:
-                f.write(name + "您好，\n\n您目前沒有過期的作業。\n更新執行表單請至 https://cmm.ai/pm ，謝謝您。\n\n")
+            f.write(message)
 
     server.close()
 
@@ -65,81 +74,126 @@ sheet_instance = sheet.get_worksheet(0)
 records_data = sheet_instance.get_all_records()
 #print(records_data[0])
 
-emails = {"Jared": "jared@choozmo.com", "Morrison": "morrison@choozmo.com", "Ginia": "ginia@choozmo.com", "Doris": "doris@choozmo.com", "Nina": "nina.huang@choozmo.com", "Yukyo": "yukyo@choozmo.com", "Andy": "andy@choozmo.com", "Oak": "oak@choozmo.com", "Zooey": "zooey@choozmo.com", "Syuanyuri": "syuanyuri@choozmo.com", "Wizer": "wizer@choozmo.com", "Jason": "jason@choozmo.com", "Tomoya": "tomoya@choozmo.com", "詠心": "noodlesloves@gmail.com", "献竤": "fxp87257@gmail.com"}
-overdue_email_content = {}
+emails = {"Jared": "jared@choozmo.com", "Morrison": "morrison@choozmo.com", "Doris": "doris@choozmo.com", "Nina": "nina.huang@choozmo.com", "Yukyo": "yukyo@choozmo.com", "Andy": "andy@choozmo.com", "Oak": "oak@choozmo.com", "Zooey": "zooey@choozmo.com", "Yuri": "syuanyuri@choozmo.com", "Wizer": "wizer@choozmo.com", "Jason": "jason@choozmo.com", "Tomoya": "tomoya@choozmo.com", "詠心": "noodlesloves@gmail.com", "献竤": "fxp87257@gmail.com"}
+
+overdue_notstarted_content = {}
+overdue_incomplete_content = {}
+canceled_content = {}
+has_unstarted = {}
 has_overdue = {}
+has_canceled = {}
+status = [0,0,0]
 
 for e in emails.keys():
-    overdue_email_content[e] = e + "您好，\n\n提醒您有已過期的作業，請儘快完成以下作業。\n更新執行表單請至 https://cmm.ai/pm ，謝謝您。\n\n過期作業如下：\n\n"
+    overdue_notstarted_content[e] = "您有已過期而尚未執行的作業，請儘快執行並完成這些作業：\n\n"
+    overdue_incomplete_content[e] = "您有已過期而未完成的作業，請儘快完成這些作業：\n\n"
+    canceled_content[e] = "此外，以下已取消執行的作業請在備註欄位填寫取消原因：\n\n"
+    has_unstarted[e] = 0
     has_overdue[e] = 0
+    has_canceled[e] = 0
 
 for data in records_data:
     try:
-        print(parser.parse(data["起始日"]), parser.parse(data["完成日"]))
-        deltastart = (parser.parse(data["起始日"]).date()-datetime.date.today()).days
-        deltaend = (parser.parse(data["完成日"]).date()-datetime.date.today()).days
-        print(deltastart, deltaend)
-        info = ""
-
-        if(deltastart <= 7 and data["完成度"] == "預定執行"):
-            info += "專案：" + data["專案"] + " / " + data["專案項目"] + "\n"
-            info += "作業：" + data["執行項目"] + "\n"
-            info += "執行人員：" + data["指派對象"] + "\n"
-            info += "PM：" + data["PM"] + "\n"
-            info += "預定執行期間：" + data["起始日"] + "-" + data["完成日"]
-            if(deltastart>0):
-                info += " (尚有"+ str(deltastart) + "天)\n\n"
-            elif(deltastart==0):
-                info += " (當天開始執行)\n\n"
-            else:
-                info += " (過期"+ str(deltastart).replace('-','') + "天)\n\n"
-                overdue_email_content[data["PM"]] += info
-                has_overdue[data["PM"]] = 1
-                if [data["PM"]]!=[data["指派對象"]]: #sometimes you have to do the task and you are the PM
-                    overdue_email_content[data["指派對象"]] += info
-                    has_overdue[data["指派對象"]] = 1
-                if data["PM"] != "Jared": #sometimes Jared is the PM
-                    overdue_email_content["Jared"] += info
-                    has_overdue["Jared"] = 1
-            message += info
-
-
-        elif(deltaend <= 7 and data["完成度"] == "執行中"):
+        if(data["完成度"] == "取消"):
+            status[2] = 1
             info += "專案：" + data["專案"] + " / " + data["專案項目"] + "\n"
             info += "作業：" + data["執行項目"] + "\n"
             info += "執行人員：" + data["指派對象"] + "\n"
-            info += "PM：" + data["PM"] + "\n"
-            info += "完成日：" + data["完成日"]
-            if(deltaend>0):
-                info += " (尚有"+ str(deltaend) + "天)\n\n"
-            elif(deltaend==0):
-                info += " (當天完成)\n\n"
-            else:
-                info += " (過期"+ str(deltaend).replace('-','') + "天)\n\n"
-                overdue_email_content[data["PM"]] += info
-                has_overdue[data["PM"]] = 1
-                if [data["PM"]]!=[data["指派對象"]]: #sometimes you have to do the task and you are the PM
-                    overdue_email_content[data["指派對象"]] += info
-                    has_overdue[data["指派對象"]] = 1
-                if data["PM"] != "Jared": #sometimes Jared is the PM
-                    overdue_email_content["Jared"] += info
-                    has_overdue["Jared"] = 1
-            message2 += info
+            info += "PM：" + data["PM"] + "\n\n"
+            canceled_content[data["PM"]] += info
+            has_canceled[data["PM"]] = 1
+            if [data["PM"]]!=[data["指派對象"]]: #sometimes you have to do the task and you are the PM
+                canceled_content[data["指派對象"]] += info
+                has_canceled[data["指派對象"]] = 1
+            if data["PM"] != "Jared": #sometimes Jared is the PM
+                canceled_content["Jared"] += info
+                has_canceled["Jared"] = 1
+            message3 += info
+        else:
+            print(parser.parse(data["起始日"]), parser.parse(data["完成日"]))
+            deltastart = (parser.parse(data["起始日"]).date()-datetime.date.today()).days
+            deltaend = (parser.parse(data["完成日"]).date()-datetime.date.today()).days
+            print(deltastart, deltaend)
+            info = ""
+
+            print("Project: " + data["專案"] + "Note: " + data["備註"])
+
+            if(deltastart <= 7 and data["完成度"] == "預定執行"):
+                status[0] = 1
+                info += "專案：" + data["專案"] + " / " + data["專案項目"] + "\n"
+                info += "作業：" + data["執行項目"] + "\n"
+                info += "執行人員：" + data["指派對象"] + "\n"
+                info += "PM：" + data["PM"] + "\n"
+                info += "預定執行期間：" + data["起始日"] + "-" + data["完成日"]
+                if(deltastart>0):
+                    info += " (尚有"+ str(deltastart) + "天)\n\n"
+                elif(deltastart==0):
+                    info += " (當天開始執行)\n\n"
+                else:
+                    info += " (過期"+ str(deltastart).replace('-','') + "天)\n\n"
+                    overdue_notstarted_content[data["PM"]] += info
+                    has_unstarted[data["PM"]] = 1
+                    if [data["PM"]]!=[data["指派對象"]]: #sometimes you have to do the task and you are the PM
+                        overdue_notstarted_content[data["指派對象"]] += info
+                        has_unstarted[data["指派對象"]] = 1
+                    if data["PM"] != "Jared": #sometimes Jared is the PM
+                        overdue_notstarted_content["Jared"] += info
+                        has_unstarted["Jared"] = 1
+                message += info
+            elif(deltaend <= 7 and data["完成度"] == "執行中"):
+                status[1] = 1
+                info += "專案：" + data["專案"] + " / " + data["專案項目"] + "\n"
+                info += "作業：" + data["執行項目"] + "\n"
+                info += "執行人員：" + data["指派對象"] + "\n"
+                info += "PM：" + data["PM"] + "\n"
+                info += "完成日：" + data["完成日"]
+                if(deltaend>0):
+                    info += " (尚有"+ str(deltaend) + "天)\n\n"
+                elif(deltaend==0):
+                    info += " (當天完成)\n\n"
+                else:
+                    info += " (過期"+ str(deltaend).replace('-','') + "天)\n\n"
+                    overdue_incomplete_content[data["PM"]] += info
+                    has_overdue[data["PM"]] = 1
+                    if [data["PM"]]!=[data["指派對象"]]: #sometimes you have to do the task and you are the PM
+                        overdue_incomplete_content[data["指派對象"]] += info
+                        has_overdue[data["指派對象"]] = 1
+                    if data["PM"] != "Jared": #sometimes Jared is the PM
+                        overdue_incomplete_content["Jared"] += info
+                        has_overdue["Jared"] = 1
+                message2 += info
 
     except:
         print("PASS")
 
 message += "更新執行表單請至 https://cmm.ai/pm"
 message2 += "更新執行表單請至 https://cmm.ai/pm"
+message3 += "更新執行表單請至 https://cmm.ai/pm"
 
 messages = []
 messages.append(message)
 messages.append(message2)
+messages.append(message3)
 
+print(has_unstarted)
 print(has_overdue)
+print(has_canceled)
+print(status)
+
+fname="msg1.txt"
+with open(fname, 'w', encoding="UTF+8") as f:
+    f.write(message)
+
+fname="msg2.txt"
+with open(fname, 'w', encoding="UTF+8") as f:
+    f.write(message2)
+
+fname="msg3.txt"
+with open(fname, 'w', encoding="UTF+8") as f:
+    f.write(message3)
 
 send_line(messages)
-send_email(emails, overdue_email_content, has_overdue)
+send_email(emails, overdue_notstarted_content, overdue_incomplete_content, canceled_content, has_unstarted, has_overdue, has_canceled)
 
 '''notify_group(message)
 time.sleep(5)

website_clickjobs/type-F/_clickjob.py

@@ -0,0 +1,148 @@
+import time
+from datetime import datetime
+import json
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.firefox.options import Options
+import codecs
+import random
+import requests
+import dataset
+import traceback
+import sys
+from selenium.webdriver.common.keys import Keys
+
+add_tabs = [7,9,11,13,15,7,9,11,13,15,7,9,11,13,15,7,9,11,13,15]
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+driver=None
+headers = {
+        "Authorization": "Bearer " + "t35vhZtWNgvDNWHc3DJh0OKll3mcB9GvC8K2EAkBug2",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+sleepoffset = 0
+
+def send_msg(kw):
+    params = {"message": "處理關鍵字: "+kw}  
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+def scroll_shim(passed_in_driver, object):
+        x = object.location['x']
+        y = object.location['y']
+        scroll_by_coord = 'window.scrollTo(%s,%s);' % (
+            x,
+            y
+        )
+        scroll_nav_out_of_way = 'window.scrollBy(0, -120);'
+        passed_in_driver.execute_script(scroll_by_coord)
+        passed_in_driver.execute_script(scroll_nav_out_of_way)
+
+def process_query(domain, target_domain, brands, query):
+    print(query)
+    sleepoffset = 0
+    global driver
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}&gl={}'.format(urllib.parse.quote(query), 100, 'zh-TW', 'TW')
+    driver.get(googleurl) 
+    print(driver.current_url)
+
+    elmts=driver.find_elements("xpath","//div[@class='yuRUbf']/a")
+
+    print (len(elmts))
+    # driver.save_screenshot('c:/tmp/test.png')
+
+    domain_in_link=0
+
+    for el in elmts:
+        href=el.get_attribute('href')
+        txt=el.text
+        if len(txt)>10:
+            if domain in href:
+                domain_in_link += 1
+                print('clicked....')
+                print(href)
+                print(txt)
+                scroll_shim(driver, el)
+                webdriver.ActionChains(driver).move_to_element(el).perform()
+                webdriver.ActionChains(driver).move_to_element(el).click().perform()
+                time.sleep(15)
+
+                new_windows_count = add_tabs[random.randint(0,19)]
+                print(str(new_windows_count) + " new tabs")
+                for i in range (0,new_windows_count):
+                    print("Tab" + str(i+1))
+                    #original_window = driver.current_window_handle
+                    #driver.switch_to.new_window('window')
+                    #driver.get(href)
+                    sleepoffset += 12
+                    driver.execute_script('window.open("'+href+'","_blank");')
+                    driver.execute_script("window.scrollTo(0, 600)")
+                    time.sleep(15)
+                    #driver.close()
+                    #driver.switch_to.window(original_window)
+
+                if domain in target_domain:
+                    print("Target link found")
+                    time_stamp = datetime.fromtimestamp(time.time())
+                    time_stamp = time_stamp.strftime("%Y-%m-%d %H:%M:%S")
+                    db['click_results'].insert({"time_stamp": time_stamp, "brand": brands[domain], "domain": domain, "query": query, "url": href, "content": txt, "extra_windows": new_windows_count})
+                break
+
+    if domain in target_domain:
+        print("Target domain found")
+        time_stamp = datetime.fromtimestamp(time.time())
+        time_stamp = time_stamp.strftime("%Y-%m-%d %H:%M:%S")
+        db['query_results'].insert({"time_stamp": time_stamp, "brand": brands[domain], "domain": domain, "query": query, "googleurl": googleurl, "element_count": len(elmts), "domain_in_link_count": domain_in_link})
+            
+
+    print(domain_in_link)
+    
+
+def run_once(domain, target_domain, brands, query):
+    global driver
+    result=[]
+    options = Options()
+    options.headless = True
+#    options.add_argument("--user-agent=" +user_agent)
+#    options.add_argument('-no-sandbox')
+#    options.add_argument('-disable-dev-shm-usage')
+
+    firefox_profile = webdriver.FirefoxProfile()
+    firefox_profile.set_preference("browser.privatebrowsing.autostart", True)
+
+    driver = webdriver.Firefox(
+    options=options, firefox_profile=firefox_profile)
+
+    driver.delete_all_cookies()
+    driver.set_window_size(1400,1000)
+
+    process_query(domain, target_domain, brands, query)
+    time.sleep(3)
+    driver.quit()
+
+#execution starts here
+
+def execute(domain, target_domain, brands, query_list):
+    while True:
+        try:
+            run_once(domain, target_domain, brands, random.choice(query_list))
+        except:
+            traceback.print_exc()
+        sleepint = random.randint(75,90)
+        sleepint = sleepint - sleepoffset
+        print("Completed (" + str(sleepint) + ")")
+        if sleepint > 0:
+            time.sleep(sleepint)

website_clickjobs/type-F/bennis.py

@@ -0,0 +1,8 @@
+from _clickjob import *
+
+domain = 'https://www.bennis.com.tw'
+target_domain = ['bennis.com.tw']
+brands={domain:'班尼斯'}
+query_list = ('班尼斯', '乳膠 墊 推薦', '床 墊 訂 做 價格', '乳膠 床 推薦')
+
+execute(domain, target_domain, brands, query_list)

website_clickjobs/type-F/yogoclean.py

@@ -0,0 +1,8 @@
+from _clickjob import *
+
+domain = 'https://www.yogoclean.com'
+target_domain = ['yogoclean.com']
+brands={domain:'有夠讚'}
+query_list = ('冷氣 清洗','有夠讚',)
+
+execute(domain, target_domain, brands, query_list)