Merge branch 'master' of http://git.choozmo.com:3000/choozmo/kw_tools

SEO/news_clickbot.py

@@ -0,0 +1,116 @@
+from random import randint
+import sys
+import os
+import dataset
+from selenium import webdriver
+import traceback
+import datetime
+import codecs
+import time
+import urllib
+import argparse
+import schedule
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+import logging
+import sys
+from logging.handlers import SysLogHandler
+import socket
+import pandas as pd
+import pymysql
+pymysql.install_as_MySQLdb()
+
+_LOG_SERVER = ('hhh.ptt.cx', 514)
+logger = logging.getLogger('clickbot_100')
+handler1 = SysLogHandler(address=_LOG_SERVER, socktype=socket.SOCK_DGRAM)
+logger.addHandler(handler1)
+logger.debug('[clickbot_100][火柴星人]begin')
+
+
+path = 'C:\portable\chromedriver'
+path_z = '/Users/zooeytsai/Downloads/chromedriver 4'
+
+
+def restart_browser(pport):
+    s = Service('/root/driver/chromedriver')
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless')
+    options.add_argument('--remote-debugging-port=9222')
+    options.add_experimental_option("debuggerAddress", f"127.0.0.1:{pport}")
+    options.add_argument("--user-agent=" + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0")
+    options.add_argument("--incognito")
+    driver = webdriver.Chrome(options=options, service=s)
+    driver.delete_all_cookies()
+    driver.set_window_size(950, 20000)
+    return driver
+
+
+def process_one(pport):
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    lst = ['好睡王 引新聞','好睡王 Yahoo','好睡王 HiNet','好睡王 PCHOME','好睡王 蕃新聞','好睡王 新浪','好睡王 台北郵報','好睡王 LIFE','好睡王 match生活網','好睡王 POPDAILY','好睡王 LINE TODAY']
+    table = db['news_log']
+
+    for term in lst:
+        print(term)
+        logger.debug('[clickbot_100][' + term + ']')
+        driver = restart_browser(pport)
+        googleurl = 'https://www.google.com/?num=30'
+        driver.get(googleurl)
+        time.sleep(6)
+        send_kw_elmt = driver.find_element(By.XPATH,'/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input')
+        send_kw_elmt.send_keys(term)
+        time.sleep(3)
+        send_kw_elmt.send_keys(Keys.ENTER)
+        time.sleep(6)
+        elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']/a")
+        idx = 1
+        print(len(elmts))
+        for elmt in elmts:
+            href = elmt.get_attribute('href')
+            txt = elmt.text
+            print(txt)
+            if len(txt) > 10:
+                if '炎炎夏日易輾轉難眠' in txt:
+                    print("ranking", idx)
+                    table.insert({'kw': term, 'ranking': idx, 'title': txt, 'url': href,
+                                  'dt': datetime.datetime.now()})
+                    print('clicked....')
+                    webdriver.ActionChains(driver).move_to_element(elmt).perform()
+                    webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+                    time.sleep(5)
+                    break
+            idx += 1
+        
+        
+        driver.quit()
+        print('中場休息')
+        time.sleep(randint(90, 120))
+    db.close()
+
+
+while True:
+    try:
+        os.system('docker container restart tiny4')
+        time.sleep(1)
+        break
+    except:
+        os.system('docker container restart tiny4')
+        time.sleep(15)
+
+process_one('9925')
+
+# parser = argparse.ArgumentParser()
+# parser.add_argument('--loop', action="store_true")
+# args = parser.parse_args()
+
+# if args.loop:
+#     schedule.every(0.4).minutes.do(process_one)
+#     # print('今天開始')
+#     # schedule.every().day.at('9:30').do(process_one)
+#
+#     while True:
+#         schedule.run_pending()
+#         time.sleep(1)
+
+# >> C:\tmp\seo_line.txt 2>&1

SEO/seo_notify_gs.py

@@ -20,14 +20,14 @@ import sys
 import docker
 import pymysql
 pymysql.install_as_MySQLdb()
-from ga_click import main
+# from ga_click import main
 
 headers = {
     "Authorization": "Bearer " + "OZDcq7sVKwr3F6YNLtBF3LuIgpa4Ql9eAnBWeD7sHTJ",
     "Content-Type": "application/x-www-form-urlencoded"
 }
 
-name = {'beastparadise.net':'毛怪','taroboba-yuan.com':'清原','match-planet.com':'火柴星人','hhh.com.tw':['幸福空間','幸福空間設計師'],'sctt.com.tw':'仁本','gs-rack.com':'神助物流','bennis.com.tw':'班尼斯'}
+name = {'beastparadise.net':'毛怪','taroboba-yuan.com':'清原','match-planet.com':'火柴星人','hhh.com.tw':['幸福空間','幸福空間設計師'],'sctt.com.tw':'仁本','gs-rack.com':'神助物流','bennis.com.tw':'班尼斯','jiatetea.tw':'呷茶'}
 
 def send_msg(kw):
     params = {"message": kw}
@@ -40,9 +40,13 @@ cursor = db.query(
     'SELECT * FROM seo.general_log where domain != "innews.com.tw" and datediff(now(),dt)=0 order by dt desc')
 cursor_rank = db.query(
     'SELECT * FROM seo.google_rank where datediff(now(),dt)=0 order by dt desc')
+cursor_rank = db.query(
+    'SELECT * FROM seo.google_rank where datediff(now(),dt)=0 order by dt desc')
+cursor_sns = db.query(
+    'SELECT * FROM seo.sns_log where datediff(now(),dt)=0 order by dt desc')
 result = "\n"
 idx = 0
-d = {'beastparadise.net':0,'match-planet.com':0,'taroboba-yuan.com':0,'hhh.com.tw':0,'sctt.com.tw':0,'hhh.com.tw/designer':0,'gs-rack.com':0,'bennis.com.tw':0}
+d = {'beastparadise.net':0,'match-planet.com':0,'taroboba-yuan.com':0,'hhh.com.tw':0,'sctt.com.tw':0,'hhh.com.tw/designer':0,'gs-rack.com':0,'bennis.com.tw':0,'jiatetea.tw':0}
 for c in cursor:
     custom_name = c['domain']
     if custom_name in d:
@@ -55,13 +59,25 @@ for c in cursor:
         domain = c['domain']
         custom_name = name[domain]
         result += "[" + str(c['ranking']) + "]:" + elmt + " "+ f"({custom_name})\n"
-
-per_total_click = f"毛怪:{d['beastparadise.net']}\n清原:{d['taroboba-yuan.com']}\n火柴星人:{d['match-planet.com']}\n幸福空間:{d['hhh.com.tw']}\n幸福空間設計師:{d['hhh.com.tw/designer']}\n仁本:{d['sctt.com.tw']}\n神助物流:{d['gs-rack.com']}\n班尼斯:{d['bennis.com.tw']}\n"
+sns_result = ""
+sns_d = {'班尼斯':0}
+for c in cursor_sns:
+    elmt = c['kw']
+    title = c['title']
+    custom_name = c['client']
+    if custom_name in sns_d:
+        sns_d[custom_name]+=1
+    sns_result = f"\n[{c['ranking']}] {elmt} {title}"
+per_total_click = f"毛怪:{d['beastparadise.net']}\n清原:{d['taroboba-yuan.com']}\n火柴星人:{d['match-planet.com']}\n幸福空間:{d['hhh.com.tw']}\n幸福空間設計師:{d['hhh.com.tw/designer']}\n仁本:{d['sctt.com.tw']}\n神助物流:{d['gs-rack.com']}\n班尼斯:{d['bennis.com.tw']}\n呷茶:{d['jiatetea.tw']}\n"
+sns_per_total_click = f"(口碑)班尼斯:{sns_d['班尼斯']}\n"
 print(per_total_click)
+print(sns_per_total_click)
+print(sns_result)
 msg_per_total_click = '\n點擊次數總計:\n' + per_total_click
 msg = "\n機器人執行次數:" + str(idx)
-ga_sessions = '\n幸福空間的GA Sessions'+main()['ga:sessions']
-send_msg(msg_per_total_click + result + ga_sessions)
+# ga_sessions = '\n幸福空間的GA Sessions'+main()['ga:sessions']
+sns_per_total_click = '\n點擊次數總計:\n' + sns_per_total_click
+# send_msg(msg_per_total_click + result + sns_per_total_click + sns_result + ga_sessions )
 
 
 

choozmo/ads_csv.py

@@ -0,0 +1,78 @@
+import csv
+import sys
+import codecs
+with codecs.open('C:\\Users\\jared\\Downloads\\search.csv', 'r','utf-16') as csvfile:
+    spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|')
+
+    kwdict={}
+    addict={}
+    head=True
+    for row in spamreader:
+        if head:
+            head=False
+            continue
+        ll=len(row)
+        campaign=row[0]
+        adgroup=row[19]
+        kw=row[47]
+        if len(kw)>0:
+            print(campaign)
+            print(adgroup)
+            if kwdict.get(adgroup) is None:
+                kwdict[adgroup]=[]
+            if addict.get(adgroup) is None:
+                addict[adgroup]=[]
+
+            kwdict[adgroup].append(kw)
+        for i in range(66,104,2):
+            hline=row[i]
+            if len(hline)>0:
+                addict[adgroup].append(hline)
+
+
+    for k,v in kwdict.items():
+        print('rectangle '+k+'_kw'+' {')
+        print('card '+k+'關鍵字'+' [')
+        for itm in v:
+            print(itm)
+        print(']')
+        print('}')
+
+    for k,v in addict.items():
+        print('rectangle '+k+'_ad'+' {')
+        print('card '+k+'廣告'+' [')
+
+        for itm in v:
+            print(itm)
+        print(']')
+        print('}')
+
+    for k,v in kwdict.items():
+        print(k+' -[#black]-- '+k+'_ad')
+        print(k+' -[#black]-- '+k+'_kw')
+
+    for k,v in kwdict.items():
+        print(campaign+' -[#black]-> '+k)
+
+
+
+#        print(campaign+' -[#black]-> '+k+'_ad')
+
+
+#    print(kwdict)
+#    print(addict)
+
+
+
+
+
+#    for row in spamreader:
+#        ll=len(row)
+#
+#        for i in range(ll):
+#            print(i)
+#            print(row[i])
+#        break
+    
+#
+#    sys.exit()

choozmo/ads_csv2_csv.py

@@ -0,0 +1,84 @@
+import csv
+import sys
+import codecs
+with codecs.open('C:\\Users\\jared\\Downloads\\search.csv', 'r','utf-16') as csvfile:
+    spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|')
+
+    kwdict={}
+    addict={}
+    head=True
+    for row in spamreader:
+        if head:
+            head=False
+            continue
+        ll=len(row)
+        campaign=row[0]
+        adgroup=row[19]
+        kw=row[47]
+        if len(kw)>0:
+            print(campaign)
+            print(adgroup)
+            if kwdict.get(adgroup) is None:
+                kwdict[adgroup]=[]
+            if addict.get(adgroup) is None:
+                addict[adgroup]=[]
+
+            kwdict[adgroup].append(kw)
+        for i in range(66,104,2):
+            hline=row[i]
+            if len(hline)>0:
+                addict[adgroup].append(hline)
+
+fw=codecs.open('c:/tmp/gads.csv','w','utf-8')
+fw.write("活動,群組,關鍵字,廣告")
+for k,v in kwdict.items():
+    kwlen=len(v)
+    adlen=len(addict[k])
+    totlen=max(kwlen,adlen)
+    for i in range(totlen):
+        try:
+            kw=v[i]
+        except:
+            kw=' '
+        try:
+            ad=addict[k][i]
+        except:
+            ad=' '
+        data=campaign+"\t"+k+"\t"+kw+"\t"+ad+"\n"
+        fw.write(data)
+
+
+fw.close()
+
+#    for k,v in addict.items():
+#        print('rectangle '+k+'_ad'+' {')
+#        print('card '+k+'廣告'+' [')
+
+#        for itm in v:
+#            print(itm)
+#        print(']')
+#        print('}')
+
+
+
+
+#        print(campaign+' -[#black]-> '+k+'_ad')
+
+
+#    print(kwdict)
+#    print(addict)
+
+
+
+
+
+#    for row in spamreader:
+#        ll=len(row)
+#
+#        for i in range(ll):
+#            print(i)
+#            print(row[i])
+#        break
+    
+#
+#    sys.exit()

choozmo/ads_selenium.py

@@ -0,0 +1,99 @@
+import time
+import json
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+import requests
+import datetime
+import dataset
+import time
+import traceback
+import sys
+import fire
+import pymysql
+pymysql.install_as_MySQLdb()
+
+driver = None
+
+
+def process_query(qs):
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    table = db['ads']
+    q = qs[0]
+    client = qs[1]
+    global driver
+    googleurl = 'https://www.google.com/webhp?hl=zh-TW&sa=X&ved=0ahUKEwj84vXliMX4AhUYqFYBHcUMAlgQPAgI'
+    driver.get(googleurl)
+    time.sleep(6)
+    send_kw_elmt = driver.find_element(By.XPATH,'/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input')
+    send_kw_elmt.send_keys(q)
+    time.sleep(3)
+    send_kw_elmt.send_keys(Keys.ENTER)
+    time.sleep(6)
+    elmts_title = driver.find_elements(By.XPATH, "//div[@class='CCgQ5 vCa9Yd QfkTvb MUxGbd v0nnCb']/span")
+    elmts_content = driver.find_elements(By.XPATH, "//div[@class='MUxGbd yDYNvb lyLwlc']")
+    print(len(elmts_title))
+    print(len(elmts_content))
+    
+    for i, j in zip(elmts_title,elmts_content):
+        title = i.text
+        content = j.text
+        
+        table.insert({'kw': q, 'client': client, 'title': title, 'content': content,
+                              'dt': datetime.datetime.now()})
+
+    db.close()
+
+
+def run_once(q):
+    global driver
+    result = []
+    user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 OPR/68.0.3618.125'
+    s = Service('/Users/zooeytsai/Downloads/chromedriver 4')
+    op = webdriver.ChromeOptions()
+    # options.add_argument('--headless')
+    # options.add_argument('--remote-debugging-port=9222')
+    # options.add_experimental_option("debuggerAddress", "192.168.192.45:9922")
+    # options.add_argument("--user-agent=" + user_agent)
+    # options.add_argument("--incognito")
+    op.add_argument('--disable-dev-shm-usage')
+    op.add_argument('--no-sandbox')
+    op.add_argument(r'user-data-dir=C:\Users\Administrator\AppData\Local\Google\Chrome\User Data')
+    op.add_argument(f"profile-directory=Profile 25")
+    op.add_experimental_option("excludeSwitches", ["enable-automation"])
+    op.add_experimental_option('useAutomationExtension', False)
+    driver = webdriver.Chrome(options=op,
+                              executable_path=r'C:\Users\Administrator\Downloads\chromedriver_win32 (4)\chromedriver')
+    
+    driver = webdriver.Chrome(
+        options=op, service=s)
+    str1 = driver.capabilities['browserVersion']
+    print('版本', str1)
+    driver.delete_all_cookies()
+    driver.set_window_size(1400, 1000)
+    
+    print(q)
+    process_query(q)
+    time.sleep(3)
+    driver.quit()
+
+
+class JParams(object):
+    
+    def get(self, kw, domain):
+        print(kw)
+        print(domain)
+        run_once((kw, domain))
+
+
+if __name__ == '__main__':
+    # fire.Fire(JParams)
+    run_once(('新北 家具 推薦','班尼斯'))

choozmo/fetch_content.py

@@ -5,14 +5,15 @@ import html2text
 import jieba
 import dataset
 
+
 jieba.load_userdict("c:/tmp/userdict.txt")
-stopwords='， 的/-。*.|)(][_!、」「:：jpgmenu有了也gif%stylespnghttpsimagesicogovRSSscript'
+stopwords='， 的/-。*.|)(][_!、」「:：｜）』『（xmlimgursvgbase64jpgmenuMenu有了也gif%stylespnghttpsimagesicogovRSSscript'
 db = dataset.connect('sqlite:///c:/tmp/jieba.db')
 db.query('delete from tmp')
 #db.query('drop table tmp')
 
-urls=['https://www.nightnight.tw/%E5%BA%8A%E5%A2%8A%E6%8E%A8%E8%96%A6/']
-
+#urls=['https://www.dcard.tw/f/house/p/232318765?cid=BBDFB720-BAE4-406E-8449-D2F12EA11241']
+urls=['https://www.gold-kirin.com.tw/about']
 
 #db = dataset.connect('sqlite:///:memory:')
 table=db['tmp']
@@ -36,7 +37,7 @@ for url in urls:
             table.insert({'word':word})
 
 
-cursor=db.query('select word,count(word) as cnt from tmp group by word having count(word) >2 order by count(word) desc')
+cursor=db.query('select word,count(word) as cnt from tmp group by word having ( count(word) >1 or length(word)>2) order by count(word) desc')
 for c in cursor:
     print(c['word'])
     print(c['cnt'])

choozmo/gsc_tree.py

@@ -1,110 +1,84 @@
-import traceback
+import searchconsole
 import dataset
-import codecs
-import sys
-import pickle
 import os
-import searchconsole
-import pandas as pd
 import networkx as nx
-#import pysftp
-import codecs
-import pyvis
-import sys
-import pickle
-import os
-import searchconsole
 from pyvis.network import Network
-import jieba
-
-
-#db = dataset.connect('mysql://choozmo:pAssw0rd@127.0.0.1:3306/hhh?charset=utf8mb4')
-#db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+import sqlite3
 
-#db.begin()
 db = dataset.connect('sqlite:///:memory:')
-table=db['tmp']
-#table=db['gsc_page_query_year']
-#pname='korea'
-rid=0
+table = db['tmp']
+# table=db['gsc_page_query_year']
+# pname='korea'
+rid = 0
+
 
 def checkig():
     global instl
     global table
     global pname
     global rid
-    lst=[]
-    cntdict={}
-    codelist={}
-    idx=0
-    flag_break=False
-
-    fname=os.path.abspath(__file__)
-    elmts=fname.split(os.path.sep)
-    path2=os.path.sep.join(elmts[0:-1])
-    keysdir=path2+os.path.sep+'../keys'+os.path.sep
-
-    account = searchconsole.authenticate(client_config='c:/keys/client_secret.json',credentials='c:/keys/credentials.json')
-#    account = searchconsole.authenticate(client_config='C:\\gitlab\\kw_tools\\kw_tools\\hhh\\keys\\client_secret.json',credentials='C:\\gitlab\\kw_tools\\kw_tools\\hhh\\keys\\credentials.json')
+    lst = []
+    cntdict = {}
+    codelist = {}
+    idx = 0
+    flag_break = False
+    
+    fname = os.path.abspath(__file__)
+    elmts = fname.split(os.path.sep)
+    path2 = os.path.sep.join(elmts[0:-1])
+    keysdir = path2 + os.path.sep + '../keys' + os.path.sep
+    
+    account = searchconsole.authenticate(client_config='c:/keys/client_secret.json',
+                                         credentials='c:/keys/credentials.json')
+    # webproperty = account['sc-domain:www.yogoclean.com']
+    webproperty = account['https://www.yogoclean.com/']
+    print(webproperty)
+    report = webproperty.query.range('today', months=-6).dimension('page', 'query').get()
+    print(report)
+    # report = webproperty.query.range('2022-05-28', '2022-05-29').dimension('page', 'query').get()
+    # account = searchconsole.authenticate(client_config='C:\\gitlab\\kw_tools\\kw_tools\\hhh\\keys\\client_secret.json',credentials='C:\\gitlab\\kw_tools\\kw_tools\\hhh\\keys\\credentials.json')
     G = nx.Graph()
-
-#    webproperty = account['https://ipromise.com.tw/']
-#    webproperty = account['sc-domain:face8ook.org']
-#    webproperty = account['sc-domain:hhh.com.tw']
-#   webproperty = account['sc-domain:hhh.com.tw']
-
-#    webproperty = account['https://www.damanwoo.com/']
-    webproperty = account['https://innews.com.tw/']
-
-#    report=webproperty.query.range('2021-03-01', '2021-06-17').dimension('page','query').get()
-#    report=webproperty.query.range('2021-06-01', '2021-06-17').dimension('page','query').get()
-#    report=webproperty.query.range('2020-06-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/(491|31|293|278|31|24|594|356|307|491|33|385)', 'equals').get()
-#    report=webproperty.query.range('2020-03-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/'+pgnum, 'contains').get()
-#    report=webproperty.query.range('2020-03-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/'+pgnum, 'contains').get()
-    report=webproperty.query.range('2022-01-01', '2022-04-16').dimension('page','query').get()
-
-    result=[]
-    rdict={}
-    total_idx=0
-
+    
+    result = []
+    rdict = {}
+    total_idx = 0
     for r in report.rows:
-        if 'hhh.com.tw/designers/cases/' not in r[0]:
-            continue
+        print(r)
+        print(rdict)
         if rdict.get(r[0]) is None:
-            total_idx+=1
-            rid=total_idx
-            rdict[r[0]]=rid
+            total_idx += 1
+            rid = total_idx
+            rdict[r[0]] = rid
         else:
-            rid=rdict[r[0]]
-        entry={'page':r[0],'query':r[1],'rid':rid}
+            rid = rdict[r[0]]
+        entry = {'page': r[0], 'query': r[1], 'rid': rid}
         result.append(entry)
-
+    
     print('list done')
-
     for r in result:
         table.insert(r)
     db.commit()
     print('db done')
-
-#    cursor=db.query('select query as q,page as url,rid from tmp where rid in (select rid from (select rid,count(*) from tmp group by rid having count(*) > 2 and count(*) < 6) as tbl1) order by rid ')
-    cursor=db.query('select query as q,page as url,rid from tmp order by rid ')
-
-    riddict={}
-    prev=''
-    curnode=''
-    cururl=''
-
-    total_idx=0
-    for c in cursor:
-        G.add_edge(c['q'],c['rid'],weight=3,width=3,borderwidth=3)
-    remove=[]
+    
+    cursor = db.query(
+        'select query as q,page as url,rid from tmp where rid in (select rid from (select rid,count(*) from tmp group by rid having count(*) > 2 and count(*) < 6) as tbl1) order by rid ')
+    # select_data = cur.execute('select q,url,rid from tmp order by rid ')
+    
+    riddict = {}
+    prev = ''
+    curnode = ''
+    cururl = ''
+    
+    total_idx = 0
+    for data in cursor:
+        G.add_edge(data[0], data[2], weight=3, width=3, borderwidth=3)
+    remove = []
     G.remove_edges_from(nx.selfloop_edges(G))
-    G2=G
-    pyG = Network(height="600px", width="100%",bgcolor="#444444",font_color="white")
-
+    G2 = G
+    pyG = Network(height="600px", width="100%", bgcolor="#444444", font_color="white")
+    
     pyG.from_nx(G2)
-    pyG.show('news.html')
-
+    pyG.show('news15.html')
 
-r=checkig()
 
+r = checkig()