Merge branch 'master' of http://git.choozmo.com:3000/choozmo/kw_tools

INNNews/redis_1777.py

@@ -0,0 +1,104 @@
+#import redis
+import time
+import traceback
+#import json
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import dataset
+import json
+import random
+import time
+import sys
+import codecs
+import redis
+import random
+import os
+import time
+from userAgentRandomizer import userAgents
+
+driver=None
+
+def re_get_webdriver():
+    global driver
+    result=[]
+    if driver is not None:
+        print('closing....')
+        driver.quit()
+        os.system('killall chrome')
+        print('quit....')
+        driver=None
+    try:
+        ua = userAgents()
+
+        user_agent = ua.random()        
+
+        options = webdriver.ChromeOptions()
+        options.add_argument("--no-sandbox")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("--headless")
+
+        print(user_agent)
+        options.add_argument("--user-agent=" +user_agent)
+        options.add_argument("--incognito")
+        driver=None
+        try:
+            driver = webdriver.Chrome(options=options)
+        except:
+            traceback.print_exc()
+#            driver.quit()
+#            os.system('pkill -f ')
+            os.system('kill %d' % os.getpid())
+            sys.exit()
+            return
+        driver.set_window_size(1400,1000)
+        return
+    except:
+        import traceback
+        traceback.print_exc()
+        driver=None
+        return None
+
+def run_once(url):
+    global driver
+    i=random.randint(0,7)
+    if i==0 or driver is None:
+        time.sleep(8)
+        re_get_webdriver()
+    if driver is None:
+        return
+    try:
+        driver.execute_script('window.open("'+url+'","_blank");')
+        driver.execute_script("window.scrollTo(0, window.scrollY + 400)")
+        time.sleep(0.5)
+    except:
+        print('exception')
+
+lst=[]
+#lst.append('https://innews.com.tw/62183/')
+#lst.append('https://innews.com.tw/48338/')
+#lst.append('https://innews.com.tw/62326/')
+#lst.append('https://innews.com.tw/38246/')
+#lst.append('https://innews.com.tw/24843/')
+
+r = redis.Redis(host='db.ptt.cx', port=6379, db=1,password='choozmo9')
+data=r.get('innews_five')
+js=json.loads(data)
+for j in js:
+    lst.append(j)
+
+
+
+
+#lst=['https://www.hhh.com.tw/columns/detail/3427/index.php']
+    #for i in range(20):
+#while True:
+for i in range(500):
+    l=random.choice(lst)
+    print(l)
+    run_once(l)
+

INNNews/sns_clickbot.py

@@ -8,6 +8,7 @@ import urllib.parse
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.common.by import By
 from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as EC
 import codecs
 import random
@@ -53,16 +54,17 @@ def process_query(qs):
     client=qs[2]
     domain=qs[3]
     global driver
-    escaped_search_term = urllib.parse.quote(q)
-    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(escaped_search_term, 100, 'zh-TW')
-    print(googleurl)
+    googleurl = 'https://www.google.com/?num=100'
     driver.get(googleurl)
+    time.sleep(6)
+    send_kw_elmt = driver.find_element(By.XPATH, '/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input')
+    send_kw_elmt.send_keys(q)
+    time.sleep(3)
+    send_kw_elmt.send_keys(Keys.ENTER)
+    time.sleep(6)
     time.sleep(10)
-    # fname=term.replace(' ','_')
-    # driver.save_screenshot('c:/tmp/seo/'+fname+'.png')
-    # df=pd.DataFrame()
 
-    elmts = driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+    elmts = driver.find_elements(By.XPATH,"//div[@class='yuRUbf']/a")
     print('網頁數量',len(elmts))
     idx = 1
     for elmt in elmts:

choozmo/kw_plan_tree.py

@@ -0,0 +1,101 @@
+import searchconsole
+import dataset
+import os
+import networkx as nx
+from pyvis.network import Network
+import sqlite3
+import csv
+import sys
+import codecs
+import difflib
+
+
+
+kwdict={}
+G = nx.Graph()
+
+
+
+def gcm0(strings):
+    clusters = {}
+    for string in (x.strip() for x in strings):
+        match = difflib.get_close_matches(string, clusters.keys(), 8, 0.65)
+        if match:
+            clusters[match[0]].append(string)
+        else:
+            clusters[string] = [ string ]
+    return clusters
+
+
+def proc_row(row):
+    elmts=row.split(' ')
+    for elmt in elmts:
+        if kwdict.get(elmt) is None:
+            kwdict[elmt]=1
+        else:
+            kwdict[elmt]+=1
+
+with codecs.open('C:\\tmp\\test.csv', 'r','utf-16') as csvfile:
+    spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|')
+    kwdict={}
+    addict={}
+    head=True
+    rowlst=[]
+    for row in spamreader:
+        if head:
+            head=False
+            continue
+        ll=len(row)
+        proc_row(row[0])
+        rowlst.append(row[0])
+
+
+clusters=gcm0(rowlst)
+keys=[]
+for k,v in clusters.items():
+    keys.append(k)
+    for x in v:
+        G.add_edge(k,x,weight=1,label='')
+
+
+already_dict={}
+from strsimpy.qgram import QGram
+qgram = QGram(2)
+for k1 in keys:
+    for k2 in keys:
+        if k1!=k2:
+            if qgram.distance(k1, k2)<=12:
+                if already_dict.get(k1) is None and already_dict.get(k2) is None:
+                    already_dict[k1]=1
+                    already_dict[k2]=1
+                    G.add_edge(k1,k2,weight=1,label='')
+
+
+pyG = Network(height="600px", width="100%", bgcolor="#444444", font_color="white")
+pyG.set_options("""
+const options = {
+"nodes" : {
+        "font" : {
+            "size" : "30",
+            "color" : "#ffffff"
+        }
+    },
+  "physics": {
+    "forceAtlas2Based": {
+      "springLength": 100
+    },
+    "maxVelocity": 150,
+    "minVelocity": 0.28,
+    "solver": "forceAtlas2Based"
+  }
+}
+""")
+G.remove_edges_from(nx.selfloop_edges(G))
+pyG.from_nx(G)
+
+#pyG.show_buttons(filter_=['physics'])
+pyG.show('news17.html')
+
+#print(clusters)
+
+sys.exit()

website_clickjobs/clickjob.py

@@ -0,0 +1,120 @@
+import time
+from datetime import datetime
+import json
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+import requests
+import dataset
+import traceback
+import sys
+
+target_domain=['bennis.com.tw']
+brands={'bennis.com.tw':'班尼斯'}
+
+driver=None
+headers = {
+        "Authorization": "Bearer " + "t35vhZtWNgvDNWHc3DJh0OKll3mcB9GvC8K2EAkBug2",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+
+
+def send_msg(kw):
+    params = {"message": "處理關鍵字: "+kw}  
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+
+def process_query(qs):
+    q=qs[0]
+    domain=qs[1]
+    global driver
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(urllib.parse.quote(q), 100,'zh-TW')
+    print(googleurl)
+    driver.get(googleurl)
+    #time.sleep(6)
+
+    # elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+    # ABOVE METHOD IS DEPRECATED STARTING SELENIUM 4.3.0, USE THIS
+    elmts=driver.find_elements("xpath","//div[@class='yuRUbf']/a")
+
+    idx=1
+    ranking=-1
+    domain_in_link = 0
+
+    print(len(elmts))
+    # driver.save_screenshot('c:/tmp/test.png')
+
+    for elmt in elmts:
+        href=elmt.get_attribute('href')
+        txt=elmt.text
+        if len(txt)>10:
+            if domain in href:
+                domain_in_link += 1
+                print('clicked....')
+                print(href)
+                print(txt)
+                webdriver.ActionChains(driver).move_to_element(elmt).perform()
+                webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+                break
+    
+    if domain in target_domain:
+        print("Target domain found")
+        time_stamp = datetime.fromtimestamp(time.time())
+        time_stamp = time_stamp.strftime("%Y-%m-%d %H:%M:%S")
+        db['query_results'].insert({"time_stamp": time_stamp, "brand": brands[domain], "domain": domain, "query": q, "googleurl": googleurl, "element_count": len(elmts), "domain_in_link_count": domain_in_link})
+
+def run_once(q):
+    global driver
+    result=[]
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless')
+#    options.add_argument("--user-agent=" +user_agent)
+    options.add_argument("--incognito")
+    options.add_argument('--no-sandbox')
+    options.add_argument('--disable-dev-shm-usage')
+
+    driver = webdriver.Chrome(
+    options=options)
+
+    driver.delete_all_cookies()
+    driver.set_window_size(1400,1000)
+
+    print(q)
+    process_query(q)
+    time.sleep(3)
+    driver.quit()
+
+#lst=[{'kw':'幸福空間','domain':'hhh.com.tw','page':0}]
+lst=[]
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+
+cursor=db.query('select term,domain from selected_kw order by rand()')
+
+for c in cursor:
+    lst.append(c)
+
+
+#for c in lst:
+while True:
+    try:
+        c=random.choice(lst)
+        run_once( (c['term'],c['domain'])   )
+    except:
+        traceback.print_exc()
+    sleepint=random.randint(20,40)
+    time.sleep(sleepint)