Merge branch 'master' of http://git.choozmo.com:3000/choozmo/kw_tools

INNNews/general_clickbot.py

@@ -18,18 +18,27 @@ import time
 import traceback
 import sys
 import fire
-import pymysql
-pymysql.install_as_MySQLdb()
-from ig_tags import setting
+#import pymysql
+#pymysql.install_as_MySQLdb()
+
 
 db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
 table=db['general_log']
-
-
-  
 driver = None
 
 
+def rua():
+    pool = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 OPR/68.0.3618.125",
+    ]
+    return random.choice(pool)
+
 
 def empty_query(q):
     global driver
@@ -47,7 +56,7 @@ def process_query(qs):
     driver.get(googleurl)
     time.sleep(6)
 
-    elmts=driver.find_elements(By.T,"//div[@class='yuRUbf']/a")
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
 
     idx=1
     ranking=-1
@@ -74,11 +83,11 @@ def run_once(q):
     global driver
     result=[]
     s = Service('/root/driver/chromedriver')
-    user_agent = setting.rua()
+    user_agent = rua()
     options = webdriver.ChromeOptions()
     options.add_argument('--headless')
     options.add_argument('--remote-debugging-port=9222')
-    options.add_experimental_option("debuggerAddress", "127.0.0.1:9922")
+    options.add_experimental_option("debuggerAddress", f"127.0.0.1:{q[2]}")
     options.add_argument("--user-agent=" +user_agent)
     options.add_argument("--incognito")
 
@@ -105,10 +114,9 @@ def run_once(q):
 
 class JParams(object):
 
-  def get(self, kw,domain):
-    print('kw')
-    print(domain)
-    run_once( (kw,domain)   )
+  def get(self, kw,domain,port):
+    print('關鍵字',kw)
+    run_once( (kw,domain,port)   )
 
 
 if __name__ == '__main__':

INNNews/run_sheet_2.py

@@ -21,20 +21,40 @@ import dataset
 def run_once(pport, dockername):
     db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
     lst = []
-
+    
     cursor = db.query('select term, domain from seo.selected_kw')
     for c in cursor:
+        lst.append([c['term'], c['domain']])
+    
+    obj = random.choice(lst)
+    kw = obj[0]
+    domain = obj[1]
+    print(kw, domain)
+    
+    intval = os.system(f'python3 general_clickbot.py get --kw="{kw}" --domain="{domain}" --port="{str(pport)}"')
+    
+    print('執行完成genetal_clickbot')
+    
+    if intval == -1:
+        print('-1')
+        sys.exit()
 
-        lst.append([c['term'],c['domain']])
 
+def run_specific(pport, dockername):
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    lst = []
+    
+    cursor = db.query('select term, domain from seo.selected_kw where client="神助物流"')
+    for c in cursor:
+        lst.append([c['term'], c['domain']])
+    
     obj = random.choice(lst)
     kw = obj[0]
     domain = obj[1]
     print(kw,domain)
 
-    intval = os.system(f'python general_clickbot.py get --kw={kw} --domain={domain} --port={str(pport)}')
-    
-    print('執行genetal_clickbot')
+    intval = os.system(f'python3 general_clickbot.py get --kw="{kw}" --domain="{domain}" --port="{str(pport)}"')    
+    print('執行完成神助genetal_clickbot')
 
     if intval == -1:
         print('-1')
@@ -44,19 +64,19 @@ def run_once(pport, dockername):
 class JParams(object):
     
     def get(self, port=9222):
-        print(port)
         while True:
             try:
                 os.system('docker container restart tiny1')
-                # os.system('docker container restart tiny2')
-                # os.system('docker container restart tiny3')
+                os.system('docker container restart tiny2')
+                os.system('docker container restart tiny3')
+                os.system('docker container restart tiny8')
                 time.sleep(1)
                 run_once(9922, 'tiny1')
-                # run_once(9923, 'tiny2')
-                # run_once(9924, 'tiny3')
-                
+                run_once(9923, 'tiny2')
+                run_once(9924, 'tiny3')
+                run_specific(9929, 'tiny8')
                 time.sleep(20)
-            
+                break           
             except:
                 os.system('docker container restart tiny1')
                 time.sleep(15)
@@ -64,4 +84,3 @@ class JParams(object):
 
 if __name__ == '__main__':
     fire.Fire(JParams)
-

SEO/ranking_day.py

@@ -13,6 +13,7 @@ from logging.handlers import SysLogHandler
 import socket
 import pandas as pd
 import random
+from selenium.webdriver.common.by import By
 from selenium.webdriver.chrome.service import Service
 import os
 from random import randint
@@ -57,7 +58,7 @@ def process_one(item):
     # driver.save_screenshot('c:/tmp/seo/'+fname+'.png')
     # df=pd.DataFrame()
     
-    elmts = driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+    elmts = driver.find_elements(By.XPATH,"//div[@class='yuRUbf']/a")
     cnt = 1
     datadict = {'搜尋詞': [], '結果標題': [], '結果網址': [], '結果名次': []}
     

SEO/ranking_day2.py

@@ -13,11 +13,12 @@ import sys
 from logging.handlers import SysLogHandler
 import socket
 import pandas as pd
-import pymysql
-
-pymysql.install_as_MySQLdb()
+#import pymysql
+#pymysql.install_as_MySQLdb()
 import random
+from selenium.webdriver.common.by import By
 from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.support.ui import WebDriverWait
 import os
 import fire
 
@@ -57,10 +58,11 @@ def process_one(item):
     # driver.save_screenshot('c:/tmp/seo/'+fname+'.png')
     # df=pd.DataFrame()
     
-    elmts = driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+    elmts = driver.find_elements(By.XPATH,"//div[@class='yuRUbf']/a")
     cnt = 1
+    clickelmt=None
     datadict = {'搜尋詞': [], '結果標題': [], '結果網址': [], '結果名次': []}
-    
+    print('搜尋結果',len(elmts))
     for elmt in elmts:
         try:
             href = elmt.get_attribute('href')
@@ -70,14 +72,22 @@ def process_one(item):
             datadict['結果網址'].append(href)
             datadict['結果名次'].append(str(cnt))
             if domain in href:
+                clickelmt = elmt
                 print(href)
                 print(elmt.text)
                 table.insert(
-                    {'title': elmt.text, 'url': href, 'keyword': term, 'dt': datetime.datetime.now(), 'num': cnt})
+                    {'kw': term, 'domain': domain, 'ranking': cnt, 'title': elmt.text, 'url': href,'dt': datetime.datetime.now()})
+#                webdriver.ActionChains(driver).move_to_element(elmt).perform()
+#                webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
             cnt += 1
         except:
             print('href2 exception')
             traceback.print_exc()
+    if clickelmt:
+        webdriver.ActionChains(driver).move_to_element(clickelmt).perform()
+        webdriver.ActionChains(driver).move_to_element(clickelmt).click().perform()
+        time.sleep(5)
+        print('點擊完成')
     if len(datadict['結果標題']) <= 0:
         print('None')
         driver.quit()
@@ -90,8 +100,8 @@ def process_one(item):
     # df.to_excel('/Users/zooeytsai/'+fname+".xls")
     
     driver.quit()
-    print('中場休息')
-    time.sleep(randint(90, 120))
+    print('結束')
+
 
 
 def run_once(q):

SEO/run_ranking_day.py

@@ -5,8 +5,8 @@ import time
 import fire
 import dataset
 from random import randint
-import pymysql
-pymysql.install_as_MySQLdb()
+#import pymysql
+#pymysql.install_as_MySQLdb()
 
 def run_once(pport, dockername):
     db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
@@ -26,17 +26,17 @@ def run_once(pport, dockername):
     if now_day_len == 0:
         print('首位')
         intval = os.system(
-            f'python3 ranking_day.py get --kw="清原 中央" --domain="taroboba-yuan.com" --port="{str(pport)}"')
+            f'python3 ranking_day2.py get --kw="清原 中央" --domain="taroboba-yuan.com" --port="{str(pport)}"')
     
     for i in lst:
         if i[0] == kw:
             id = lst.index(i)
-            intval = os.system(f'python3 ranking_day.py get --kw="{lst[id+1][0]}" --domain="{lst[id+1][1]}" --port="{str(pport)}"')
+    intval = os.system(f'python3 ranking_day2.py get --kw="{lst[id+1][0]}" --domain="{lst[id+1][1]}" --port="{str(pport)}"')
 
-        print('執行ranking_day.py')
-        if intval == -1:
-            print('-1')
-            sys.exit()
+    if intval == -1:
+        print('-1')
+        sys.exit()
+    print('執行完成ranking_day2.py')
 
 
 class JParams(object):
@@ -44,16 +44,15 @@ class JParams(object):
     def get(self, port=9222):
         while True:
             try:
-                os.system('docker container restart tiny9')
+                os.system('docker container restart tiny8')
                 time.sleep(1)
-                run_once(9928,'tiny9')
+                run_once(9929,'tiny8')
                 time.sleep(20)
                 break
             except:
                 os.system('docker container restart tiny9')
                 time.sleep(15)
 
-
-# if __name__ == '__main__':
-#     run_once()
+if __name__ == '__main__':
+    fire.Fire(JParams)
 

monitor/GA_all.py

@@ -0,0 +1,164 @@
+#!/usr/bin/python3
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+from googleapiclient.discovery import build
+from oauth2client.service_account import ServiceAccountCredentials
+import dataset
+import pymysql
+pymysql.install_as_MySQLdb()
+from datetime import datetime
+import platform
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+db.query('delete from ga_pagepath')
+db.begin()
+
+table=db['ga_pagepath']
+SCOPES = ['https://www.googleapis.com/auth/analytics.readonly']
+platform_now = platform.system().lower()
+
+# KEY_FILE_LOCATION = 'c:\gitlab\kw_tools\monitor\corded-velocity-301807-a3e3d5420aba.json'
+KEY_FILE_LOCATION = '/Users/zooeytsai/Downloads/corded-velocity-301807-a3e3d5420aba.json'
+
+
+# line notify header
+headers = {
+        "Authorization": "Bearer " + "QCAM5upFjeBVp54PGqT4eMZSXPU0y4vYk1e1CoASa2P",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+com_table = []
+def creat_table():
+  for i in range(0,24):
+    com_table.append([i,6000])
+  # com_table.append([24,70000])
+
+
+def send_msg_pg(pv):
+  # line notify send message
+    current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') #現在時間
+    hour = datetime.now().strftime('%H') 
+    # 判斷是否達標
+    complet="否"
+    # for i in range(0,25):
+    #   if int(hour)+1==com_table[i][0]:
+    #     print(i)
+    #     if int(kw) > com_table[i][1] :
+    #       complet="是"
+    #   elif int(hour) == 24:
+    #     if int(kw) > 70000 :
+    #       complet="是"
+    print('網頁瀏覽量',pv)
+    if int(pv) > 6000:
+      complet = "是"
+    params = {"message": "\n現在時間: " + current_time + "\n當前pageViews: "+pv + "\n是否達標: " + complet}
+    print(params)
+    return params
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+    print(r)
+
+
+def initialize_analyticsreporting(key_file):
+  """Initializes an Analytics Reporting API V4 service object.
+
+  Returns:
+    An authorized Analytics Reporting API V4 service object.
+  """
+  credentials = ServiceAccountCredentials.from_json_keyfile_name(
+      key_file, SCOPES)
+
+  # Build the service object.
+  analytics = build('analyticsreporting', 'v4', credentials=credentials)
+
+  return analytics
+
+
+def get_report(analytics,body):
+  """Queries the Analytics Reporting API V4.
+
+  Args:
+    analytics: An authorized Analytics Reporting API V4 service object.
+  Returns:
+    The Analytics Reporting API V4 response.
+  """
+  return analytics.reports().batchGet(
+      body={
+        'reportRequests':body
+      }
+  ).execute()
+
+
+def print_response(response):
+  """Parses and prints the Analytics Reporting API V4 response.
+
+  Args:
+    response: An Analytics Reporting API V4 response.
+  """
+  result=[]
+  for report in response.get('reports', []):
+    columnHeader = report.get('columnHeader', {})
+    dimensionHeaders = columnHeader.get('dimensions', [])
+    metricHeaders = columnHeader.get('metricHeader', {}).get('metricHeaderEntries', [])
+
+    for row in report.get('data', {}).get('rows', []):
+      dimensions = row.get('dimensions', [])
+      dateRangeValues = row.get('metrics', [])
+      ga_dict={}
+
+      for header, dimension in zip(dimensionHeaders, dimensions):
+#        print(header + ': ', dimension)
+        ga_dict[header]=dimension
+      
+      for i, values in enumerate(dateRangeValues):
+#        print('Date range:', str(i))
+        for metricHeader, value in zip(metricHeaders, values.get('values')):
+          ga_dict[metricHeader.get('name')]=value
+#          print(metricHeader.get('name') + ':', value)
+        result.append(ga_dict)
+    return result
+#        print(ga_dict)
+
+
+def main(viewid,key_file):
+  creat_table()
+  analytics = initialize_analyticsreporting(key_file)
+  current_time = datetime.now().strftime('%Y-%m-%d') #現在時間
+  body=[{ 'viewId': viewid,
+  'dateRanges': [{'startDate': current_time, 'endDate': current_time}],
+
+  'metrics': [{'expression': 'ga:users'},{'expression': 'ga:newusers'},{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'},{'expression': 'ga:bounceRate'},{'expression': 'ga:pageviewsPerSession'}],
+#  'dimensions': [{'name': 'ga:pagePath'}],
+#  'orderBys':[{"fieldName": "ga:pageviews", "sortOrder": "DESCENDING"}],
+  'pageSize': '100'
+  }]
+
+  response = get_report(analytics,body)
+  ga_dict=print_response(response)
+
+  result=[]
+  for elmt in ga_dict:
+    print(elmt)
+    hour = datetime.now().strftime('%H')
+    # if int(hour)+1 > 8 :
+    message = send_msg_pg(elmt['ga:pageviews'])
+#      result.append(elmt)
+  print('inserting.....')
+  return message
+
+
+if __name__ == '__main__':
+  creat_table()
+  main('123')

monitor/general_clickbot_hhh.py

@@ -19,8 +19,8 @@ import traceback
 import sys
 import fire
 import random
-import pymysql
-pymysql.install_as_MySQLdb()
+#import pymysql
+#pymysql.install_as_MySQLdb()
 
 db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
 table=db['general_log']
@@ -101,6 +101,10 @@ def run_once(q):
     user_agent = rua()
     options = webdriver.ChromeOptions()
     options.add_argument('--headless')
+<<<<<<< HEAD
+    options.add_argument('--remote-debugging-port=9222')
+    options.add_experimental_option("debuggerAddress", f"127.0.0.1:{q[2]}")
+=======
     options.add_argument("--no-sandbox")
     options.add_argument("--disable-dev-shm-usage")
 ###    options.add_argument("--proxy-server=socks5://192.168.192.156:1080")
@@ -112,6 +116,7 @@ def run_once(q):
 #    options.add_argument("--user-agent=" +user_agent)
 #    options.add_argument('--remote-debugging-port=9222')
 #    options.add_experimental_option("debuggerAddress", "127.0.0.1:{q[2]}")
+>>>>>>> d38b39b4e07b1ed66eb4980b06240a2e31f3ac76
     options.add_argument("--user-agent=" +user_agent)
     options.add_argument("--incognito")
 

monitor/notifyall.py

@@ -0,0 +1,52 @@
+import redis
+import time
+import json
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import dataset
+import time
+import rpyc
+import sys
+import docker
+import pymysql
+pymysql.install_as_MySQLdb()
+from monitor import GA_all
+
+
+ga_info = {'班尼斯':['154114800',{
+        "Authorization": "Bearer " + "QCAM5upFjeBVp54PGqT4eMZSXPU0y4vYk1e1CoASa2P",
+        "Content-Type": "application/x-www-form-urlencoded"
+},'/Users/zooeytsai/Downloads/corded-velocity-301807-a3e3d5420aba.json'
+],'有夠讚':['208868237',{
+        "Authorization": "Bearer " + "Alur8iUgifYsh9CMiyUdlrTWK4Y9Xys8RZD52YkBMGl",
+        "Content-Type": "application/x-www-form-urlencoded"
+},'/Users/zooeytsai/Downloads/corded-velocity-301807-a0d690cd59d9.json'],
+          '神助物流':['265874206',{
+        "Authorization": "Bearer " + "50Qu8O3XUhOhIOkz5WHy6mhftRVBe8CtL3DXS05UXRC",
+        "Content-Type": "application/x-www-form-urlencoded"},'/Users/zooeytsai/Downloads/corded-velocity-301807-42d10dfc21a4.json']}
+
+def send_msg(viewid,header,key_file):
+    send_message = GA_all.main(viewid, key_file)
+    params = {"message": send_message['message']}
+    print('通知結果',params)
+    r = requests.post("https://notify-api.line.me/api/notify",headers=header, params=params)
+
+
+for k,v in ga_info.items():
+    send_msg(v[0],v[1],v[2])
+
+
+# schtasks /create /sc minute /mo 30 /sd 2022/05/050 /st 9:00 /et 23:00 /tn "linebat" /tr "C:\tmp\inn_line.bat"
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.dirname(SCRIPT_DIR))

monitor/run_sheet_hhh.py

@@ -16,8 +16,8 @@ import redis
 import time
 import fire
 import dataset
-import pymysql
-pymysql.install_as_MySQLdb()
+#import pymysql
+#pymysql.install_as_MySQLdb()
 
 def run_once(pport, dockername):
     db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
@@ -59,7 +59,7 @@ class JParams(object):
                 run_once(9927, 'tiny6')
 
                 time.sleep(20)
-
+                break
             except:
                 os.system('docker container restart tiny4')
                 time.sleep(15)