Merge remote-tracking branch 'origin/master'

choozmo/get_newterms.py

@@ -0,0 +1,43 @@
+
+from bs4 import BeautifulSoup
+import requests
+import html2text
+import jieba
+import dataset
+
+
+jieba.load_userdict("c:/tmp/userdict.txt")
+stopwords='， 的/-。*.|)(][_!、」「:：｜）』『（xmlimgursvgbase64jpgmenuMenu有了也gif%stylespnghttpsimagesicogovRSSscript'
+db = dataset.connect('sqlite:///c:/tmp/jieba.db')
+db.query('delete from tmp')
+#db.query('drop table tmp')
+
+#urls=['https://www.dcard.tw/f/house/p/232318765?cid=BBDFB720-BAE4-406E-8449-D2F12EA11241']
+#urls=['https://www.gold-kirin.com.tw/about']
+urls=['https://www.xingcai-beauty.com/service.html']
+#db = dataset.connect('sqlite:///:memory:')
+table=db['tmp']
+
+# request web page
+#resp = requests.get("https://casino543.com/2021%E5%B9%B4%E5%8D%81%E5%A4%A7%E7%B7%9A%E4%B8%8A%E5%A8%9B%E6%A8%82%E5%9F%8E%E6%8E%92%E5%90%8D%E6%8E%A8%E8%96%A6-%E5%A8%9B%E6%A8%82%E5%9F%8E%E5%89%8D100%E5%90%8D%E5%A8%9B%E6%A8%82%E5%9F%8E%E9%82%84/")
+#resp = requests.get("https://mort.moi.gov.tw/frontsite/cms/newsAction.do?method=viewContentDetail&iscancel=true&contentId=MjU3NA==")
+#resp = requests.get("https://www.memory.com.tw/funeral_ceremony-in.php?i=5&c=3")
+for url in urls:
+    resp = requests.get(url)
+    html = resp.content
+    html=html.decode('utf-8')
+    h = html2text.HTML2Text()
+
+    h.ignore_links = True
+
+    docs=h.handle(html )
+    words = jieba.cut(docs, cut_all=False)
+    for word in words:
+        if word not in stopwords:
+            table.insert({'word':word})
+
+
+cursor=db.query('select word,count(word) as cnt from tmp group by word having ( count(word) >1 or length(word)>2) order by count(word) desc')
+for c in cursor:
+    print(c['word'])
+    print(c['cnt'])

choozmo/kw_plan_tree.py

@@ -35,7 +35,7 @@ def proc_row(row):
         else:
             kwdict[elmt]+=1
 
-with codecs.open('C:\\tmp\\test.csv', 'r','utf-16') as csvfile:
+with codecs.open('C:\\tmp\\test9.csv', 'r','utf-16') as csvfile:
     spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|')
     kwdict={}
     addict={}
@@ -71,7 +71,7 @@ for k1 in keys:
                     G.add_edge(k1,k2,weight=1,label='')
 
 
-pyG = Network(height="600px", width="100%", bgcolor="#444444", font_color="white")
+pyG = Network(height="900px", width="100%", bgcolor="#444444", font_color="white")
 pyG.set_options("""
 const options = {
 "nodes" : {

choozmo/m_hhh.py

@@ -0,0 +1,188 @@
+#import redis
+import time
+import traceback
+#import json
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import dataset
+from selenium.webdriver.common.keys import Keys
+import json
+import random
+import time
+import redis
+import sys
+import codecs
+import random
+import os
+import time
+from userAgentRandomizer import userAgents
+import requests
+driver=None
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+
+headers = {
+        "Authorization": "Bearer " + "6SDULL1Ebklduc6TFxa97AFto5Sj21kyJ30CxiLiSoi",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+def send_msg(kw):
+    params = {"message":kw}  
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+
+
+
+
+def re_get_webdriver():
+    global port
+    global driver
+    result=[]
+    if driver is not None:
+        print('closing....')
+        driver.quit()
+        os.system('killall chrome')
+        print('quit....')
+        driver=None
+    try:
+        ua = userAgents()
+
+        user_agent = ua.random()        
+
+        options = webdriver.ChromeOptions()
+        mobile_emulation = {
+            "deviceMetrics": { "width": 360, "height": 640, "pixelRatio": 3.0 },
+            "userAgent": "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19" }
+        options.add_experimental_option("mobileEmulation", mobile_emulation)
+
+        options.add_argument("--no-sandbox")
+        options.add_argument("--disable-dev-shm-usage")
+#        options.add_argument("--headless")
+#        options.add_argument('--remote-debugging-port='+port)
+#        options.add_experimental_option("debuggerAddress", '127.0.0.1:9922')
+
+        print(user_agent)
+#        options.add_argument("--user-agent=" +user_agent)
+        options.add_argument("--incognito")
+        driver=None
+        try:
+#            driver = webdriver.Chrome(executable_path='C:\\portable\\webdriver\\chrome101\\chromedriver.exe',options=options)
+            driver = webdriver.Chrome(options=options)
+
+        except:
+#            driver.quit()
+#            os.system('pkill -f ')
+#            os.system('kill %d' % os.getpid())
+            traceback.print_exc()
+            sys.exit()
+            return
+#        driver.set_window_size(1400,1000)
+        return
+    except:
+        traceback.print_exc()
+        driver=None
+        return None
+
+
+
+def run_once(jsobj):
+
+    table=db['rank_detection']
+    print(jsobj)
+    global driver
+
+
+#    i=random.randint(0,9)
+    i=100
+    if driver is None:
+        time.sleep(8)
+        re_get_webdriver()
+    if driver is None:
+        return
+    try:
+        kw=jsobj['kw']
+        if jsobj.get('domain') is None:
+            exclude=jsobj['exclude']
+            domain=None
+        else:
+            domain=jsobj['domain']
+            exclude=None
+
+#        driver.get('https://www.google.com?num=100')
+        driver.get('https://www.google.com?num=20')
+
+        time.sleep(3)
+        print(driver.current_url)
+        elmt = driver.find_element(By.XPATH, "//input[@name='q']")
+        time.sleep(1)
+        elmt.send_keys(kw)
+        elmt.send_keys(Keys.ENTER)
+        time.sleep(6)
+
+        elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']/a")
+
+        numresults=len(elmts)
+        time.sleep(9999)
+        print('搜尋結果數量',numresults)
+        if numresults==0:
+            send_msg('stop working...')
+            sys.exit()
+
+        idx=1
+        found=False
+        test_lst=[]
+        for elmt in elmts:
+            href=elmt.get_attribute('href')
+            txt=elmt.text
+            if len(txt)>10:
+                if domain is not None:
+                    if domain in href:
+                        print('found....')
+                        print('clicked....')
+                        print(href)
+                        print(txt)
+                        print("ranking", idx)
+                        found=True
+
+                        webdriver.ActionChains(driver).move_to_element(elmt).perform()
+                        elmt.click()
+#                        webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+                        table.insert({'ranking':idx,'kw':kw,'results':numresults,'url':href,'title':txt})
+
+                        time.sleep(6)
+                        break
+                else:
+                    if exclude not in href:
+                        test_lst.append(elmt)
+                    
+            idx+=1
+        if exclude is not None:
+            print('exclude')
+            elmt=random.choice(test_lst)
+            print(elmt)
+
+            webdriver.ActionChains(driver).move_to_element(elmt).perform()
+            webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+            time.sleep(5)
+
+        if not found:
+            table.insert({'ranking':-1,'kw':kw,'results':numresults,'url':'','title':'未收錄'})
+
+    except:
+        print('exception')
+        traceback.print_exc()
+
+    driver.quit()
+    sys.exit()
+
+#par1=sys.argv[1]
+#port=sys.argv[2]
+par1='tiny3'
+port='9962'
+os.system('docker container restart '+par1)
+#time.sleep(9)
+run_once({'domain':'m.hhh.com.tw','kw':'幸福 設計師'})