Merge branch 'master' of http://git.choozmo.com:3000/choozmo/kw_tools

INNNews/redis_1777.py

@@ -58,7 +58,6 @@ def re_get_webdriver():
         driver.set_window_size(1400,1000)
         return
     except:
-        import traceback
         traceback.print_exc()
         driver=None
         return None
@@ -76,6 +75,7 @@ def run_once(url):
         driver.execute_script("window.scrollTo(0, window.scrollY + 400)")
         time.sleep(0.5)
     except:
+        traceback.print_exc()
         print('exception')
 
 lst=[]

similar_web/main.py

@@ -17,7 +17,8 @@ import time
 from google.ads.googleads.client import GoogleAdsClient
 sys.path.append('.')
 import kw_planner
- 
+import uvicorn
+
 fake_rank_plus = 700
 fake_traffic_weighted = 1.3
 
@@ -240,6 +241,8 @@ async def get_domain_data(req:q_req):
 
 
 
+if __name__ == "__main__":
+    uvicorn.run("main:app", host="0.0.0.0", port=8081)
 
 
 

web/clickbot_100.py

@@ -0,0 +1,61 @@
+
+import dataset
+from selenium import webdriver
+import traceback
+import time
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.by import By
+import pymysql
+pymysql.install_as_MySQLdb()
+
+
+path = 'C:\portable\chromedriver'
+path_z = '/Users/zooeytsai/Downloads/chromedriver 4'
+def restart_browser():
+    options = webdriver.ChromeOptions()
+    # options.add_argument("user-agent=%s" % rua())
+    options.add_argument('--headless')
+    driver=webdriver.Chrome(options=options,executable_path=path_z)
+    driver.set_window_size(950, 20000)
+    return driver
+
+
+def process_one(kw,domain):
+    # db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    db_local = dataset.connect('mysql://root:jondae350@localhost/ig_tags')
+    table=db_local['google_rank']
+
+    driver=restart_browser()
+    # escaped_search_term=urllib.parse.quote(term)
+    googleurl = 'https://www.google.com/?num=100'
+    driver.get(googleurl)
+    time.sleep(6)
+    send_kw_elmt = driver.find_element(By.XPATH,'/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input')
+    send_kw_elmt.send_keys(kw)
+    time.sleep(3)
+    send_kw_elmt.send_keys(Keys.ENTER)
+    time.sleep(6)
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+
+    cnt=1
+    datadict={'搜尋詞':[],'結果標題':[],'結果網址':[],'結果名次':[]}
+    
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            if domain in href:
+                datadict['搜尋詞'].append(kw)
+                datadict['結果標題'].append(elmt.text)
+                datadict['結果網址'].append(href)
+                datadict['結果名次'].append(str(cnt))
+            cnt+=1
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+    
+    driver.quit()
+    time.sleep(60)
+    # db.close()
+    db_local.close()
+    return datadict
+

web/main.py

@@ -7,7 +7,7 @@ import time
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel
-from fastapi import FastAPI, Form
+from fastapi import FastAPI, Form, Response
 import subprocess
 import suggests
 from typing import Optional
@@ -18,11 +18,21 @@ from pyvis.network import Network
 import pickle
 import logging
 import threading
-import time
 import random
 import string
 from fastapi.responses import HTMLResponse
 from fastapi.responses import RedirectResponse
+import dataset
+from selenium import webdriver
+import traceback
+import time
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.by import By
+# import pymysql
+# pymysql.install_as_MySQLdb()
+
+driver = None
+
 def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
     return ''.join(random.choice(chars) for _ in range(size))
 
@@ -39,7 +49,8 @@ app.add_middleware(
 
 db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/gtrends?charset=utf8mb4')
 
-app.mount("/web", StaticFiles(directory="static"), name="static")
+# app.mount("/web", StaticFiles(directory="static"), name="static")
+app.mount("/web", StaticFiles(directory="/Users/zooeytsai/kw_tools/web/static"), name="static")
 
 def thread_function(kw):
     global db
@@ -116,3 +127,62 @@ async def func_expand(kw: str = Form(...),kw2:Optional[str] = Form(None),kw3:Opt
     return RedirectResponse(url="/tree_list",status_code=302)
 #    return HTMLResponse('<html><head><meta http-equiv="refresh" content="0; URL="/tree_list" /></head></html>')
 
+    
+@app.post("/ranking/")
+async def ranking(kw: str = Form(...), domain:str = Form(...)):
+    global driver
+    options = webdriver.ChromeOptions()
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+    options.add_argument('--headless')
+    driver = webdriver.Chrome(options=options)
+    driver.set_window_size(950, 20000)
+
+    # db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    db_local = dataset.connect('mysql://root:jondae350@localhost/ig_tags')
+    table = db_local['google_rank']
+
+    # escaped_search_term=urllib.parse.quote(term)
+    googleurl = 'https://www.google.com/?num=100'
+    driver.get(googleurl)
+    time.sleep(6)
+    send_kw_elmt = driver.find_element(By.XPATH,
+                                       '/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input')
+    send_kw_elmt.send_keys(kw)
+    time.sleep(3)
+    send_kw_elmt.send_keys(Keys.ENTER)
+    time.sleep(6)
+    elmts = driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+
+    cnt = 1
+    datadict = {'搜尋詞': [], '結果標題': [], '結果網址': [], '結果名次': []}
+
+    for elmt in elmts:
+        try:
+            href = elmt.get_attribute('href')
+            if domain in href:
+                datadict['搜尋詞'].append(kw)
+                datadict['結果標題'].append(elmt.text)
+                datadict['結果網址'].append(href)
+                datadict['結果名次'].append(str(cnt))
+            cnt += 1
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+
+    driver.quit()
+    time.sleep(60)
+    # db.close()
+    db_local.close()
+    # return "ok"
+    # return RedirectResponse(url="/ranking_result",)
+    html = "<html><body><h2>"+str(datadict)+"</br></br>"
+    return html
+
+@app.get("/ranking_result/")
+async def tree_list():
+    html = "<table border='1'>"
+
+    # html += "<tr><td>" + c['kw'] + "</td>"
+    
+    return html

web/static/ranking.html

@@ -0,0 +1,14 @@
+<html>
+    <body>
+        <form action="/ranking" method="post">
+            KW: <input type="text" id="kw" name="kw" value=""></br>
+            domain: <input type="text" id="domain" name="domain" value=""></br>
+            <input type="submit" value="開始">
+        </form>
+    </br>
+</br>
+
+
+
+</body>
+</html>

website_clickjobs/type-1/_clickjob.py

@@ -17,7 +17,7 @@ import traceback
 import sys
 from selenium.webdriver.common.keys import Keys
 
-add_tabs = [4,5,6,7,8,4,5,6,7,8,4,5,6,7,8,4,5,6,7,8]
+add_tabs = [7,9,11,13,15,7,9,11,13,15,7,9,11,13,15,7,9,11,13,15]
 
 db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
 driver=None

website_clickjobs/type-2/hhh.py

@@ -0,0 +1,5 @@
+from _clickjob import *
+
+target = 'https://www.hhh.com.tw'
+
+execute(target)

website_clickjobs/type-FD/_clickjob.py

@@ -0,0 +1,88 @@
+import time
+from datetime import datetime
+import json
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+import requests
+import dataset
+import traceback
+import sys
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.firefox.options import *
+
+add_tabs = [4,5,6,7,8,4,5,6,7,8,4,5,6,7,8,4,5,6,7,8]
+sleepint=0
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+driver=None
+headers = {
+        "Authorization": "Bearer " + "t35vhZtWNgvDNWHc3DJh0OKll3mcB9GvC8K2EAkBug2",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+sleepoffset = 0
+
+def send_msg(kw):
+    params = {"message": "處理關鍵字: "+kw}  
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+
+def process_query(domain, target_domain, brands, query):
+    print(query)
+    sleepoffset = 0
+    global driver
+    driver.get(domain)
+    time.sleep(6)
+    print(driver.current_url)
+    
+
+def run_once(domain, target_domain, brands, query):
+    global driver
+    result=[]
+    options = Options()
+    options.headless = True
+#    options.add_argument("--user-agent=" +user_agent)
+#    options.add_argument('-no-sandbox')
+#    options.add_argument('-disable-dev-shm-usage')
+
+    firefox_profile = webdriver.FirefoxProfile()
+    firefox_profile.set_preference("browser.privatebrowsing.autostart", True)
+
+    driver = webdriver.Firefox(
+    options=options, firefox_profile=firefox_profile)
+
+    driver.delete_all_cookies()
+    driver.set_window_size(1400,1000)
+
+    process_query(domain, target_domain, brands, query)
+    driver.quit()
+
+#execution starts here
+
+def execute(domain, target_domain, brands, query_list):
+    i=1
+    while True:
+        print("Run " + str(i))
+        i+=1
+        try:
+            run_once(domain, target_domain, brands, random.choice(query_list))
+        except:
+            traceback.print_exc()
+        #sleepint=random.randint(75,90) - sleepoffset
+        print("Completed (")
+        time.sleep(8)

website_clickjobs/type-FD/bennis.py

@@ -0,0 +1,8 @@
+from _clickjob import *
+
+domain = 'https://www.bennis.com.tw'
+target_domain = ['bennis.com.tw']
+brands={domain:'班尼斯'}
+query_list = ('班尼斯', '乳膠 墊 推薦', '床 墊 訂 做 價格', '乳膠 床 推薦')
+
+execute(domain, target_domain, brands, query_list)

website_clickjobs/type-FD/gsrack.py

@@ -0,0 +1,8 @@
+from _clickjob import *
+
+domain = 'https://gs-rack.com/'
+target_domain = ['gs-rack.com']
+brands={domain:'神助'}
+query_list = ('神助','神助 物流')
+
+execute(domain, target_domain, brands, query_list)

website_clickjobs/type-FD/yogoclean.py

@@ -0,0 +1,8 @@
+from _clickjob import *
+
+domain = 'https://www.yogoclean.com'
+target_domain = ['yogoclean.com']
+brands={domain:'有夠讚'}
+query_list = ('有夠讚','有夠讚')
+
+execute(domain, target_domain, brands, query_list)