Merge branch 'master' of http://git.choozmo.com:3000/choozmo/kw_tools

INNNews/run_sheet_2.py

@@ -21,20 +21,40 @@ import dataset
 def run_once(pport, dockername):
     db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
     lst = []
-
+    
     cursor = db.query('select term, domain from seo.selected_kw')
     for c in cursor:
+        lst.append([c['term'], c['domain']])
+    
+    obj = random.choice(lst)
+    kw = obj[0]
+    domain = obj[1]
+    print(kw, domain)
+    
+    intval = os.system(f'python3 general_clickbot.py get --kw="{kw}" --domain="{domain}" --port="{str(pport)}"')
+    
+    print('執行完成genetal_clickbot')
+    
+    if intval == -1:
+        print('-1')
+        sys.exit()
 
-        lst.append([c['term'],c['domain']])
 
+def run_specific(pport, dockername):
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    lst = []
+    
+    cursor = db.query('select term, domain from seo.selected_kw where client="神助物流"')
+    for c in cursor:
+        lst.append([c['term'], c['domain']])
+    
     obj = random.choice(lst)
     kw = obj[0]
     domain = obj[1]
     print(kw,domain)
 
-    intval = os.system(f'python3 general_clickbot.py get --kw="{kw}" --domain="{domain}" --port="{str(pport)}"')
-    
-    print('執行完成genetal_clickbot')
+    intval = os.system(f'python3 general_clickbot.py get --kw="{kw}" --domain="{domain}" --port="{str(pport)}"')    
+    print('執行完成神助genetal_clickbot')
 
     if intval == -1:
         print('-1')
@@ -49,18 +69,18 @@ class JParams(object):
                 os.system('docker container restart tiny1')
                 os.system('docker container restart tiny2')
                 os.system('docker container restart tiny3')
+                os.system('docker container restart tiny8')
                 time.sleep(1)
                 run_once(9922, 'tiny1')
                 run_once(9923, 'tiny2')
                 run_once(9924, 'tiny3')
+                run_specific(9929, 'tiny8')
                 time.sleep(20)
                 break           
             except:
                 os.system('docker container restart tiny1')
                 time.sleep(15)
-    def run(self):
-        run_once(9922,'tiny1')
+
 
 if __name__ == '__main__':
     fire.Fire(JParams)
-

SEO/seo_notify.py

@@ -20,7 +20,7 @@ import sys
 import docker
 import pymysql
 pymysql.install_as_MySQLdb()
-from monitor import GA_innnews
+from monitor.ga_click import main
 
 headers = {
     "Authorization": "Bearer " + "OZDcq7sVKwr3F6YNLtBF3LuIgpa4Ql9eAnBWeD7sHTJ",
@@ -32,19 +32,21 @@ name = {'beastparadise.net':'毛怪','taroboba-yuan.com':'清原','match-planet.
 def send_msg(kw):
     params = {"message": kw}
     print('通知結果', params)
-    # r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
 
 
 db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
 cursor = db.query(
     'SELECT * FROM seo.general_log where domain != "innews.com.tw" and datediff(now(),dt)=0 order by dt desc')
+cursor_rank = db.query(
+    'SELECT * FROM seo.google_rank where datediff(now(),dt)=0 order by dt desc')
 result = "\n"
 idx = 0
 d = {'beastparadise.net':0,'match-planet.com':0,'taroboba-yuan.com':0,'hhh.com.tw':0,'sctt.com.tw':0,'hhh.com.tw/designer':0}
 for c in cursor:
     custom_name = c['domain']
     if custom_name in d:
-        if custom_name == 'hhh.com.tw' and c['num']=='s':
+        if custom_name == 'hhh.com.tw' and c['num']==1:
             d['hhh.com.tw/designer']+=1
         d[custom_name]+=1
     idx += 1
@@ -58,7 +60,9 @@ per_total_click = f"毛怪:{d['beastparadise.net']}\n清原:{d['taroboba-yuan.co
 print(per_total_click)
 msg_per_total_click = '\n點擊次數總計:\n' + per_total_click
 msg = "\n機器人執行次數:" + str(idx)
-send_msg(msg_per_total_click + result)
+ga_sessions = '\n幸福空間的GA Sessions'+main()['ga:sessions']
+send_msg(msg_per_total_click + result + ga_sessions)
+
 
 
 # schtasks /create /sc minute /mo 30 /sd 2022/05/050 /st 9:00 /et 23:00 /tn "linebat" /tr "C:\tmp\inn_line.bat"

hhh/SEO/local_1777.py

@@ -87,7 +87,10 @@ db = dataset.connect('mysql://choozmo:pAssw0rd@172.105.194.225:3306/seo?charset=
 
 cursor=db.query('SELECT * FROM columnids order by rand()')
 for c in cursor:
-    lst.append('https://www.hhh.com.tw/columns/detail/'+str(c['cid'])+'/')
+#    lst.append('https://www.hhh.com.tw/columns/detail/'+str(c['cid'])+'/')
+    lst.append('https://m.hhh.com.tw/columns/detail/'+str(c['cid'])+'/')
+
+    
 
 
 

monitor/ga_click.py

@@ -0,0 +1,115 @@
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+from googleapiclient.discovery import build
+from oauth2client.service_account import ServiceAccountCredentials
+import dataset
+import pymysql
+pymysql.install_as_MySQLdb()
+from datetime import datetime
+import platform
+
+SCOPES = ['https://www.googleapis.com/auth/analytics.readonly']
+KEY_FILE_LOCATION_bennis = '/Users/zooeytsai/Downloads/corded-velocity-301807-a3e3d5420aba.json'
+KEY_FILE_LOCATION_inn = '/Users/zooeytsai/Downloads/corded-velocity-301807-9262189e4dd5.json'
+KEY_FILE_LOCATION_hhh = '/Users/zooeytsai/Downloads/choozmo-ga-58715a47b676.json'
+VIEW_ID_inn = '213054263'
+VIEW_ID_hhh = '188916214'
+
+def initialize_analyticsreporting():
+    """Initializes an Analytics Reporting API V4 service object.
+  
+    Returns:
+      An authorized Analytics Reporting API V4 service object.
+    """
+    credentials = ServiceAccountCredentials.from_json_keyfile_name(
+        KEY_FILE_LOCATION_hhh, SCOPES)
+    
+    # Build the service object.
+    analytics = build('analyticsreporting', 'v4', credentials=credentials)
+    
+    return analytics
+
+
+def get_report(analytics, body):
+    """Queries the Analytics Reporting API V4.
+  
+    Args:
+      analytics: An authorized Analytics Reporting API V4 service object.
+    Returns:
+      The Analytics Reporting API V4 response.
+    """
+    return analytics.reports().batchGet(
+        body={
+            'reportRequests': body
+        }
+    ).execute()
+
+
+def print_response(response):
+    """Parses and prints the Analytics Reporting API V4 response.
+  
+    Args:
+      response: An Analytics Reporting API V4 response.
+    """
+    result = []
+    for report in response.get('reports', []):
+        columnHeader = report.get('columnHeader', {})
+        dimensionHeaders = columnHeader.get('dimensions', [])
+        metricHeaders = columnHeader.get('metricHeader', {}).get('metricHeaderEntries', [])
+        
+        for row in report.get('data', {}).get('rows', []):
+            dimensions = row.get('dimensions', [])
+            dateRangeValues = row.get('metrics', [])
+            ga_dict = {}
+            
+            for header, dimension in zip(dimensionHeaders, dimensions):
+                #        print(header + ': ', dimension)
+                ga_dict[header] = dimension
+            
+            for i, values in enumerate(dateRangeValues):
+                #        print('Date range:', str(i))
+                for metricHeader, value in zip(metricHeaders, values.get('values')):
+                    ga_dict[metricHeader.get('name')] = value
+                #          print(metricHeader.get('name') + ':', value)
+                result.append(ga_dict)
+        return result
+
+
+def main():
+    analytics = initialize_analyticsreporting()
+    current_time = datetime.now().strftime('%Y-%m-%d')
+    body = [{ 'viewId': VIEW_ID_hhh,
+      'dateRanges': [{'startDate': current_time, 'endDate': current_time}],
+    
+      'metrics': [{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'}],
+      'dimensions': [{'name': 'ga:sourceMedium'}],
+    #  'orderBys':[{"fieldName": "ga:pageviews", "sortOrder": "DESCENDING"}],
+      'pageSize': '100'
+      }]
+
+    response = get_report(analytics, body)
+    ga_dict = print_response(response)
+    result = None
+    for elmt in ga_dict:
+        hour = datetime.now().strftime('%H')
+        if 'google' in elmt['ga:sourceMedium'] and 'organic' in elmt['ga:sourceMedium']:
+            print(elmt)
+            result = elmt
+    return result
+
+
+if __name__ == '__main__':
+  main()

monitor/general_click_proxy.py

@@ -0,0 +1,205 @@
+import time
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.keys import Keys
+
+import codecs
+import random
+import datetime
+import dataset
+import time
+import traceback
+import sys
+import random
+import socket
+import requests
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['general_log']
+
+
+driver = None
+headers = {
+        "Authorization": "Bearer " + "6SDULL1Ebklduc6TFxa97AFto5Sj21kyJ30CxiLiSoi",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+def scrolling(driver,pgnum):
+    ub = driver.find_element_by_css_selector('body')
+    for i in range(pgnum):
+        ub.send_keys(Keys.PAGE_DOWN)
+        if pgnum>1:
+            time.sleep(0.3)
+
+
+
+def rua():
+    pool = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 OPR/68.0.3618.125",
+    ]
+    return random.choice(pool)
+
+
+
+
+def send_msg(kw):
+    hname=socket.gethostname()
+    params = {"message": hname+": "+kw}  
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+
+def process_query(qs):
+    q=qs[0]
+    domain=qs[1]
+    global driver
+#    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(urllib.parse.quote(q), 100,'zh-TW')
+    googleurl = 'https://www.google.com/'
+
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(6)
+    if 'sorry' in driver.current_url:
+        print('sorry...............')
+        return None
+    try:
+        elmt = driver.find_element(By.XPATH, "//button[@id='L2AGLb']")
+        if elmt:
+            elmt.click()           
+    except:
+        print('exception')
+
+
+    try:
+#        elmt = driver.find_element(By.XPATH, "//input[@aria-label='搜尋']")
+        elmt = driver.find_element(By.XPATH, "//input[@class='gLFyf gsfi']")
+
+
+        if elmt:
+            time.sleep(1)
+            elmt.send_keys(q)
+            time.sleep(6)
+
+            elmt.send_keys(Keys.ENTER)
+    except:
+        print(elmt)
+
+    time.sleep(6)
+
+    while True:
+        try:
+            elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']/a")
+            print('尋找')
+            time.sleep(2)
+            break
+        except:
+            pass
+
+    idx=1
+    ranking=-1
+    print('搜尋結果數量',len(elmts))
+    if len(elmts) <=0:
+        send_msg('network failed...')
+    for elmt in elmts:
+
+        href=elmt.get_attribute('href')
+        txt=elmt.text
+        if len(txt)>10:
+            if domain in href:
+                print('clicked....')
+                print(href)
+                print(txt)
+                print("ranking", idx)
+                driver.execute_script("return document.body.scrollHeight")
+                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+                elmt.click()
+                table.insert({'kw':q,'domain':domain,'ranking':idx,'title':txt,'url':href,'dt':datetime.datetime.now(),'num':1})
+#                webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+                print('clicked')
+                time.sleep(12)
+                break
+        idx+=1
+    return "ok"
+
+
+def run_once(q):
+    global driver
+    result=[]
+    os.system('docker container restart proxy1')
+    time.sleep(12)
+#    s = Service('/root/driver/chromedriver')
+    # s = Service('/Users/zooeytsai/Downloads/chromedriver 2')
+    user_agent = rua()
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless')
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+###    options.add_argument("--proxy-server=socks5://192.168.192.156:1080")
+    options.add_argument("--proxy-server=socks5://127.0.0.1:9050")
+
+#    options.add_argument("start-maximized")
+#    options.add_argument('--remote-debugging-port='+str(q[2]))
+#    options.add_argument('--remote-debugging-port=9222')
+#    options.add_argument("--user-agent=" +user_agent)
+    options.add_argument("--incognito")
+
+#    driver = webdriver.Chrome(options=options,service=s)
+    print('before init')
+#    driver = webdriver.Chrome(options=options)
+
+
+
+    profile = webdriver.FirefoxProfile() 
+
+    profile.set_preference("network.proxy.type", 1)
+    profile.set_preference("network.proxy.socks", "127.0.0.1")
+    profile.set_preference("network.proxy.socks_port", 9050)
+    profile.set_preference("network.proxy.socks_version", 5)
+    profile.update_preferences()
+
+    options = webdriver.FirefoxOptions()
+    options.add_argument('--headless')
+    driver = webdriver.Firefox(firefox_profile=profile,options=options)
+
+
+
+    print('after init')
+ 
+    driver.delete_all_cookies()
+    driver.set_window_size(1400,1000)
+#    driver.set_window_size(900, 3000)
+    print('到此')
+    data=process_query(q)
+    if data is not None:
+        time.sleep(3)
+    driver.quit()
+    sys.exit()
+
+cursor = db.query('SELECT query FROM seo.hhh_gsc_imp where position >=2.5 and position <=8.5 order by rand() limit 1')
+query=None
+for c in cursor:
+    query=c['query']
+    print(query)
+    break
+
+run_once((query,'hhh.com.tw'))
+

monitor/general_clickbot_hhh.py

@@ -101,8 +101,22 @@ def run_once(q):
     user_agent = rua()
     options = webdriver.ChromeOptions()
     options.add_argument('--headless')
+<<<<<<< HEAD
     options.add_argument('--remote-debugging-port=9222')
     options.add_experimental_option("debuggerAddress", f"127.0.0.1:{q[2]}")
+=======
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+###    options.add_argument("--proxy-server=socks5://192.168.192.156:1080")
+    options.add_argument("--proxy-server=socks5://192.168.192.119:1080")
+
+#    options.add_argument("start-maximized")
+#    options.add_argument('--remote-debugging-port='+str(q[2]))
+#    options.add_argument('--remote-debugging-port=9222')
+#    options.add_argument("--user-agent=" +user_agent)
+#    options.add_argument('--remote-debugging-port=9222')
+#    options.add_experimental_option("debuggerAddress", "127.0.0.1:{q[2]}")
+>>>>>>> d38b39b4e07b1ed66eb4980b06240a2e31f3ac76
     options.add_argument("--user-agent=" +user_agent)
     options.add_argument("--incognito")
 

monitor/test_wd.py

@@ -0,0 +1,162 @@
+import time
+import json
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+import requests
+import datetime
+import dataset
+import time
+import traceback
+import sys
+import fire
+import random
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['general_log']
+
+
+driver = None
+headers = {
+        "Authorization": "Bearer " + "6SDULL1Ebklduc6TFxa97AFto5Sj21kyJ30CxiLiSoi",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+
+def rua():
+    pool = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 OPR/68.0.3618.125",
+    ]
+    return random.choice(pool)
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+
+def process_query(qs):
+    q=qs[0]
+    domain=qs[1]
+    global driver
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(urllib.parse.quote(q), 100,'zh-TW')
+    print(googleurl)
+
+#    stealth(driver,
+#        languages=["en-US", "en"],
+#        vendor="Google Inc.",
+#        platform="Win32",
+#        webgl_vendor="Intel Inc.",
+#        renderer="Intel Iris OpenGL Engine",
+#        fix_hairline=True,
+#        )
+
+    driver.get(googleurl)
+    time.sleep(6)
+
+    while True:
+        try:
+            elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']/a")
+            print('尋找')
+            break
+        except:
+            pass
+
+    idx=1
+    ranking=-1
+    lenelmts=len(elmts)
+    print('搜尋結果數量',lenelmts)
+    if lenelmts <=0:
+        os.system('python3 tplink_reboot.py')
+        time.sleep(200)
+        os.system('netsh wlan connect TP-Link_78E0')
+
+#        os.system('docker container restart p4444')
+        return
+
+
+    # if len(elmts) <=0:
+    #     send_msg('network failed...')
+#    driver.save_screenshot('c:/tmp/test.png')
+
+    for elmt in elmts:
+
+        href=elmt.get_attribute('href')
+        txt=elmt.text
+        if len(txt)>10:
+            if domain in href:
+                print('clicked....')
+                print(href)
+                print(txt)
+                print("ranking", idx)
+                table.insert({'kw':q,'domain':domain,'ranking':idx,'title':txt,'url':href,'dt':datetime.datetime.now(),'num':1})
+                webdriver.ActionChains(driver).move_to_element(elmt).perform()
+                webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+                break
+        idx+=1
+
+def run_once(q):
+    global driver
+    result=[]
+    os.system('docker container restart p4444')
+    time.sleep(12)
+    user_agent = rua()
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless')
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+    options.add_argument('--disable-blink-features=AutomationControlled') 
+
+#    options.add_argument("--user-agent=" +user_agent)
+#    options.add_argument("--incognito")
+    driver = webdriver.Remote(
+            command_executor='http://127.0.0.1:4444/wd/hub',
+            options=options
+            )
+    driver.delete_all_cookies()
+    driver.set_window_size(1400,1000)
+    print('到此')
+    process_query(q)
+    time.sleep(3)
+    driver.quit()
+#    sys.exit()
+
+
+cursor = db.query('SELECT query FROM seo.hhh_gsc_imp where position >=2.5 and position <=8.5 order by rand() limit 500;')
+lst=[]
+for c in cursor:
+    lst.append(c['query'])
+
+
+while True:
+    kw=random.choice(lst)
+    print(kw)
+    run_once((kw, 'hhh.com.tw', 4444))
+
+#class JParams(object):
+#    
+#    def get(self, kw, domain, port):
+#        print(kw)
+#        print(domain)
+#        run_once((kw, domain, port))
+
+
+#if __name__ == '__main__':
+#    fire.Fire(JParams)
+