w

.favorites.json

@@ -16,5 +16,32 @@
         "workspaceRoot": null,
         "workspacePath": null,
         "id": "uQeJKy35vHfLxa6P"
+    },
+    {
+        "type": "File",
+        "name": "c:\\gitlab\\farms\\202106\\remote_yt_publish.py",
+        "parent_id": null,
+        "fsPath": "c:\\gitlab\\farms\\202106\\remote_yt_publish.py",
+        "workspaceRoot": null,
+        "workspacePath": null,
+        "id": "kgRhBXxnf3btymE1"
+    },
+    {
+        "type": "File",
+        "name": "c:\\gitlab\\farms\\0_projs\\projs.md",
+        "parent_id": null,
+        "fsPath": "c:\\gitlab\\farms\\0_projs\\projs.md",
+        "workspaceRoot": null,
+        "workspacePath": null,
+        "id": "BeR9IWGd4mnAxbfX"
+    },
+    {
+        "type": "File",
+        "name": "c:\\gitlab\\farms\\202106\\gen_yt_from_gsc.py",
+        "parent_id": null,
+        "fsPath": "c:\\gitlab\\farms\\202106\\gen_yt_from_gsc.py",
+        "workspaceRoot": null,
+        "workspacePath": null,
+        "id": "AErchNbdxYf1iQz3"
     }
 ]

.google-cookie

@@ -1,5 +1,3 @@
 #LWP-Cookies-2.0
-Set-Cookie3: 1P_JAR="2021-07-11-15"; path="/"; domain=".google.com"; path_spec; domain_dot; secure; expires="2021-08-10 15:42:07Z"; version=0
-Set-Cookie3: NID="218=O1CBm20lYnsV_ltrg-nL5DcUJNUBwoxWC6DN7AbiPyr8VZB_46f0DsJSxUCrCSpto6xERzmwwCV7FJ_m8V9KdAG6FSCEKeTG4ohk26LTz9-2nZZm8ktguyafTFc7fkVSqM8fubxeV3gAmvBML_TbOFXlF-Jn2nkLTBqo5j15EgQ"; path="/"; domain=".google.com"; path_spec; domain_dot; expires="2022-01-10 15:24:06Z"; HttpOnly=None; version=0
-Set-Cookie3: CGIC=""; path="/complete/search"; domain=".google.com"; path_spec; domain_dot; expires="2022-01-07 15:24:06Z"; HttpOnly=None; version=0
-Set-Cookie3: CGIC=""; path="/search"; domain=".google.com"; path_spec; domain_dot; expires="2022-01-07 15:24:06Z"; HttpOnly=None; version=0
+Set-Cookie3: 1P_JAR="2022-03-05-12"; path="/"; domain=".google.com"; path_spec; domain_dot; secure; expires="2022-04-04 12:37:25Z"; version=0
+Set-Cookie3: NID="511=jmFnFRJVAylFKm1tJEgc_m5zaMSlCb63QuX4aANXOz5cgS7C5dSXhqTN9abkS4qLLsAx1igUAb3FvCJFmO3ewVBVYup-OmV__TpyJ7a2Aav1aA0nKuybwsi6gQqyOkbWqmLJsc5aariaE_827PoCwVZHpAt9CB7g2sKO8UJ1o_U"; path="/"; domain=".google.com"; path_spec; domain_dot; expires="2022-09-04 12:37:25Z"; HttpOnly=None; version=0

.vscode/workbench.json

@@ -0,0 +1,22 @@
+{
+	"currentWorkbench": {
+		"files": [
+			{
+				"path": "kw_tools/choozmo/gsearch_general.py",
+				"alias": " 1 choozmo/gsearch_general.py"
+			},
+			{
+				"path": "kw_tools/choozmo/proxytest.py",
+				"alias": " 2 choozmo/proxytest.py"
+			},
+			{
+				"path": "kw_tools/hhh/yt/yt_list.py",
+				"alias": " 3 yt/yt_list.py"
+			},
+			{
+				"path": "kw_tools/hhh/yt_dl.py",
+				"alias": " 4 hhh/yt_dl.py"
+			}
+		]
+	}
+}

INNNews/commerce.py

@@ -0,0 +1,30 @@
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import redis
+import dataset
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+domain='beastparadise.net'
+cursor=db.query('SELECT term FROM seo.selected_kw where domain = "'+domain+'"  order by rand()')
+kw={}
+for c in cursor:
+    kw['kw']=c['term']
+print(kw['kw'])
+if os.name == 'nt':
+    py='python'
+else:
+    py='python3'
+
+os.system(py+' general_clickbot.py get --kw="'+kw['kw']+'" --domain='+domain)

INNNews/dbtest.py

@@ -0,0 +1,6 @@
+import dataset
+db = dataset.connect('mysql://johnny:pAsJohnny_2022@db.ptt.cx:3306/google_poi?charset=utf8mb4')
+cursor=db.query('SELECT count(*) as cnt FROM google_poi.shop_list3')
+for c in cursor:
+    print(c['cnt'])
+

INNNews/ga_daily_test.py

@@ -0,0 +1,146 @@
+#!/usr/bin/python3
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+from googleapiclient.discovery import build
+from oauth2client.service_account import ServiceAccountCredentials
+import dataset
+from datetime import datetime
+
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+db.query('delete from ga_pagepath')
+db.begin()
+
+
+SCOPES = ['https://www.googleapis.com/auth/analytics.readonly']
+KEY_FILE_LOCATION = '/root/src/kw_tools/monitor/choozmo-ga-beee24b7a4c1.json' #'c:\\keys\\choozmo-ga-beee24b7a4c1.json'
+VIEW_ID = '188916214'
+
+# line notify header
+headers = {
+        "Authorization": "Bearer " + "WekCRfnAirSiSxALiD6gcm0B56EejsoK89zFbIaiZQD",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+com_table = []
+
+def send_msg(kw):
+  # line notify send message
+    current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') #現在時間
+    hour = datetime.now().strftime('%H') 
+    # 判斷是否達標
+    complet="否"
+    for i in range(0,25):
+      if int(hour)+1==com_table[i][0]:
+        print(i)
+        if int(kw) > com_table[i][1] : 
+          complet="是"
+      elif int(hour) == 24:
+        if int(kw) > 70000 : 
+          complet="是"
+    params = {"message": "\n現在時間: " + current_time + "\n當前pageViews: "+kw + "\n是否達標: " + complet}  
+    print(params)
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+    print(r.text)
+
+
+def initialize_analyticsreporting():
+  """Initializes an Analytics Reporting API V4 service object.
+
+  Returns:
+    An authorized Analytics Reporting API V4 service object.
+  """
+  credentials = ServiceAccountCredentials.from_json_keyfile_name(
+      KEY_FILE_LOCATION, SCOPES)
+
+  # Build the service object.
+  analytics = build('analyticsreporting', 'v4', credentials=credentials)
+
+  return analytics
+
+
+def get_report(analytics,body):
+  """Queries the Analytics Reporting API V4.
+
+  Args:
+    analytics: An authorized Analytics Reporting API V4 service object.
+  Returns:
+    The Analytics Reporting API V4 response.
+  """
+  return analytics.reports().batchGet(
+      body={
+        'reportRequests':body
+      }
+  ).execute()
+
+
+def print_response(response):
+  """Parses and prints the Analytics Reporting API V4 response.
+
+  Args:
+    response: An Analytics Reporting API V4 response.
+  """
+  result=[]
+  for report in response.get('reports', []):
+    columnHeader = report.get('columnHeader', {})
+    dimensionHeaders = columnHeader.get('dimensions', [])
+    metricHeaders = columnHeader.get('metricHeader', {}).get('metricHeaderEntries', [])
+
+    for row in report.get('data', {}).get('rows', []):
+      dimensions = row.get('dimensions', [])
+      dateRangeValues = row.get('metrics', [])
+      ga_dict={}
+
+      for header, dimension in zip(dimensionHeaders, dimensions):
+#        print(header + ': ', dimension)
+        ga_dict[header]=dimension
+      
+      for i, values in enumerate(dateRangeValues):
+#        print('Date range:', str(i))
+        for metricHeader, value in zip(metricHeaders, values.get('values')):
+          ga_dict[metricHeader.get('name')]=value
+#          print(metricHeader.get('name') + ':', value)
+        result.append(ga_dict)
+    return result
+#        print(ga_dict)
+
+def main():
+  analytics = initialize_analyticsreporting()
+#(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)
+  current_time = datetime.now().strftime('%Y-%m-%d') #現在時間
+  body=[{ 'viewId': VIEW_ID,
+  'dateRanges': [{'startDate': current_time, 'endDate': current_time}],
+
+  'metrics': [{'expression': 'ga:users'},{'expression': 'ga:newusers'},{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'},{'expression': 'ga:bounceRate'},{'expression': 'ga:pageviewsPerSession'}],
+#  'dimensions': [{'name': 'ga:pagePath'}],
+#  'orderBys':[{"fieldName": "ga:pageviews", "sortOrder": "DESCENDING"}],
+  'pageSize': '100'
+  }]
+
+  response = get_report(analytics,body)
+  ga_dict=print_response(response)
+
+  result=[]
+  for elmt in ga_dict:
+    print(elmt)
+    hour = datetime.now().strftime('%H')
+    if int(hour)+1 > 8 :
+      send_msg(elmt['ga:pageviews'])
+  print('inserting.....')
+
+if __name__ == '__main__':
+#  creat_table()
+  main()

INNNews/local_1777.py

@@ -88,7 +88,7 @@ lst.append('https://innews.com.tw/24843/')
 #lst=['https://www.hhh.com.tw/columns/detail/3427/index.php']
     #for i in range(20):
 #while True:
-for i in range(50):
+for i in range(500):
     l=random.choice(lst)
     print(l)
     run_once(l)

INNNews/run_jared.py

@@ -13,21 +13,14 @@ import redis
 import time
 
 def run_once(pport,dockername):
-    time.sleep(5)
-#    r = redis.Redis(host='db.ptt.cx', port=6379, db=1,password='choozmo9')
-#    js=r.get('innews_sheet')
-#    jsobj=json.loads(js)
+#    mywords=['邱小莫 元宇宙 集仕多','元宇宙策展 集仕多','引新聞 集仕多','集仕多 元宇宙','集仕多 策展','集仕多 邱小莫','集仕多 主播','南橘','茶葉直播','連千毅nft','南投龍華宮','元欣','引新聞 炸物']
+#    mywords=['邱小莫 元宇宙 集仕多','元宇宙策展 集仕多','引新聞 集仕多','集仕多 元宇宙','集仕多 策展','集仕多 邱小莫','集仕多 主播','南橘','茶葉直播','連千毅nft','南投龍華宮','元欣','引新聞 炸物']
+#    mywords=['紅妃q木瓜','公園的好處','賣茶葉直播','引新聞','引新聞 集仕多','主播 引新聞','逗比 引新聞','南橘 引新聞','茶葉直播 引新聞','連千毅nft 引新聞','南投龍華宮 引新聞','元欣 引新聞' ,'引新聞 炸物','集仕多 site:innews.com.tw','晚安小雞造假']
+    mywords=['引新聞','引新聞 邱小莫']
 
-    mywords=['邱小莫 元宇宙','元宇宙策展 集仕多','引新聞 集仕多','集仕多 元宇宙','集仕多 策展','集仕多 邱小莫','集仕多 主播','南橘','茶葉直播','連千毅nft','南投龍華宮','元欣','引新聞 炸物']
     if True:
         kw={}
         kw['kw']=random.choice(mywords)
-    #    print(kw)
-        
-        
-        # os.chdir('/Users/zooeytsai/kw_tools/INNNews')
-#        os.chdir('c:/gitlab/kw_tools/kw_tools/INNNews')
-#        os.chdir('c:/gitlab/kw_tools/kw_tools/INNNews')
         intval=0
 
         print(    'python3 local_general_clickbot.py get --kw="'+kw['kw']+'" --domain=innews.com.tw --port='+str(pport))
@@ -41,14 +34,6 @@ def run_once(pport,dockername):
         if intval==-1:
             print('-1')
             sys.exit()
-#        time.sleep(5)
-
-    #    time.sleep(120)
-    # time.sleep(120)
-    # os.system('python3 general_clickbot.py get --kw="'+kw_innews+'" --domain=innews.com.tw')
-
-    # os.system('python notifytest.py send_msg')
-
 
 class JParams(object):
 
@@ -64,11 +49,22 @@ class JParams(object):
             run_once( 9923 ,'tiny2'  )
             run_once( 9924  ,'tiny3' )
 
-            time.sleep(20)
+            time.sleep(15)
 
         except:
-            os.system('docker container restart tiny1')
-            time.sleep(15)
+            print('exception.....')
+#            os.system('docker container restart tiny1')
+#            time.sleep(15)
 
 if __name__ == '__main__':
-  fire.Fire(JParams)
+    while True:
+        os.system('docker container restart tiny1')
+        os.system('docker container restart tiny2')
+        os.system('docker container restart tiny3')
+        time.sleep(1)
+        run_once( 9922 ,'tiny1'  )
+        run_once( 9923 ,'tiny2'  )
+        run_once( 9924  ,'tiny3' )
+
+
+#  fire.Fire(JParams)

SEO/click_and_save.py

@@ -0,0 +1,109 @@
+import random
+import sys
+import dataset
+from selenium import webdriver
+import traceback
+import datetime
+import codecs
+import time
+import urllib
+import argparse
+import schedule
+import logging
+import sys
+from logging.handlers import SysLogHandler
+import socket
+import pandas as pd
+
+_LOG_SERVER = ('hhh.ptt.cx', 514)
+logger = logging.getLogger('clickbot_100')
+handler1 = SysLogHandler(address=_LOG_SERVER,socktype=socket.SOCK_DGRAM)
+logger.addHandler(handler1)
+logger.debug('[click_and_save][DB]begin')
+
+
+def restart_browser():
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless') 
+    driver=webdriver.Chrome(options=options)
+    driver.set_window_size(950,6000)
+    return driver
+
+
+def process_one():
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    lst=[]
+    table=db['save_result_listclick']
+    cursor=db.query('select kw,page,domain from seo_clickjobs where category="202204" order by rand()')
+    for c in cursor:
+        lst.append(c)
+
+
+    entry=random.choice(lst)
+    term=entry['kw']
+    print(term)
+    domain=entry['domain']
+    logger.debug('[clickbot_100]['+term+']')
+    driver=restart_browser()
+    escaped_search_term=urllib.parse.quote(term)
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(escaped_search_term, 100,'zh-TW')
+
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(6)
+    fname=term.replace(' ','_')
+    df=pd.DataFrame()
+
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+
+    clickelmt=None
+    cnt=1
+    datadict={'搜尋詞':[],'結果標題':[],'結果網址':[],'結果名次':[]}
+    
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            if domain in href:
+                clickelmt=elmt
+                logger.debug('[clickbot_100]['+term+']['+str(cnt)+']')
+
+            print(href)
+            print(elmt.text)
+            datadict['搜尋詞'].append(term)
+            datadict['結果標題'].append(elmt.text)
+            datadict['結果網址'].append(href)
+            datadict['結果名次'].append(str(cnt))
+
+            table.insert({'title':elmt.text,'url':href,'keyword':term,'dt':datetime.datetime.now(),'num':cnt})
+            cnt+=1
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+    if clickelmt:
+        webdriver.ActionChains(driver).move_to_element(clickelmt).perform()
+        webdriver.ActionChains(driver).move_to_element(clickelmt).click().perform()
+    if len(datadict['結果標題'])<=0:
+        print('None')
+        driver.quit()
+        sys.exit()
+    df['搜尋詞']=datadict['搜尋詞']
+    df['結果標題']=datadict['結果標題']
+    df['結果網址']=datadict['結果網址']
+    df['結果名次']=datadict['結果名次']
+
+    driver.quit()
+
+process_one()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--loop')
+args = parser.parse_args()
+
+if args.loop:
+
+#    schedule.every(6).minutes.do(process_one)
+    schedule.every(0.4).minutes.do(process_one)
+
+    while True:
+        schedule.run_pending()
+        time.sleep(1)

SEO/crawl_web.py

@@ -0,0 +1,96 @@
+import traceback
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+# import rpyc
+import sys
+import docker
+# import googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+import html2text
+
+def process_one(driver):
+    lst=[]
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+#            print(href)
+            txt=elmt.text.split('\n')
+            print(txt[0])
+            lst.append({'title':txt[0],'url':href})
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+    return lst
+
+def process_query(driver,qs,number_results=10,language_code='zh-TW',enable_next=True):
+    escaped_search_term=urllib.parse.quote(qs)
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(escaped_search_term, number_results+1,language_code)
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(3)
+    totallst=[]
+    while True:
+        lst=process_one(driver)
+        totallst+=lst
+        try:
+            if enable_next:
+                time.sleep(3)
+                elmt=driver.find_element_by_xpath("//a[@id='pnnext']")
+                webdriver.ActionChains(driver).move_to_element(elmt).perform()
+                webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+            else:
+                break
+        except:
+            traceback.print_exc()
+            print('pnnext exception')
+            break
+        time.sleep(1.5)
+    return totallst
+
+
+result=[]
+driver=None
+
+def restart_browser():
+#    os.system('docker container restart p4444')
+#    time.sleep(10)
+
+    options = webdriver.ChromeOptions()
+#    options.add_argument("--proxy-server=http://80.48.119.28:8080")
+#    driver=webdriver.Chrome(executable_path='/Users/zooeytsai/Downloads/chromedriver',options=options)
+
+    driver=webdriver.Chrome(desired_capabilities=options.to_capabilities())
+    #driver = webdriver.Remote(
+    #    command_executor='http://127.0.0.1:4444/wd/hub',
+    #desired_capabilities=options.to_capabilities())
+#    desired_capabilities=DesiredCapabilities.CHROME)
+    driver.set_window_size(1400,1000)
+    return driver
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['kw_url_search_result']
+driver=restart_browser()
+lst=process_query(driver,'班尼斯 site:mobile01.com',number_results=50,language_code='zh-TW',enable_next=False)
+for l in lst:
+    table.insert(l)
+
+print(lst)
+
+#print(html2text.html2text("<p><strong>Zed's</strong> dead baby, <em>Zed's</em> dead.</p>"))

SEO/exp_100.py

@@ -0,0 +1,97 @@
+import random
+import sys
+import dataset
+from selenium import webdriver
+import traceback
+import datetime
+import codecs
+import time
+import urllib
+import argparse
+import schedule
+
+import logging
+import sys
+from logging.handlers import SysLogHandler
+import socket
+import pandas as pd
+import socket
+import os
+#logger.debug('[clickbot_100][清原]begin')
+
+hname=socket.gethostname()
+pid=str(os.getpid())
+
+
+def restart_browser():
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless') 
+    driver=webdriver.Chrome(options=options)
+    driver.set_window_size(950,6000)
+    return driver
+
+
+def process_one():
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    lst=[]
+#    table=db['google_rank']
+#    cursor = db.query('select term from seo.selected_kw')
+    # cursor=db.query('select term from selected_kw and term not in (SELECT distinct(keyword) FROM ig_tags.save_result where url like "%beastparadise.net%" and datediff(now(),dt)=0)')
+#    for c in cursor:
+#        lst.append(c['term'])
+
+#    term=random.choice(lst)
+    term='tha 娛樂城'
+    print(term)
+    driver=restart_browser()
+    escaped_search_term=urllib.parse.quote(term)
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(escaped_search_term, 100,'zh-TW')
+
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(6)
+    fname=term.replace(' ','_')
+    driver.save_screenshot('c:/tmp/seo/'+fname+'.png')
+    df=pd.DataFrame()
+    # driver.get_screenshot_as_file("/Users/zooeytsai/排名100.png")
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+
+    clickelmt=None
+    cnt=1
+    datadict={'搜尋詞':[],'結果標題':[],'結果網址':[],'結果名次':[]}
+    
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            if 'taroboba-yuan.com' in href:
+                clickelmt=elmt
+
+            print(href)
+            print(elmt.text)
+            datadict['搜尋詞'].append(term)
+            datadict['結果標題'].append(elmt.text)
+            datadict['結果網址'].append(href)
+            datadict['結果名次'].append(str(cnt))
+
+            cnt+=1
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+    if clickelmt:
+        webdriver.ActionChains(driver).move_to_element(clickelmt).perform()
+        webdriver.ActionChains(driver).move_to_element(clickelmt).click().perform()
+    if len(datadict['結果標題'])<=0:
+        print('None')
+        driver.quit()
+        sys.exit()
+    df['搜尋詞']=datadict['搜尋詞']
+    df['結果標題']=datadict['結果標題']
+    df['結果網址']=datadict['結果網址']
+    df['結果名次']=datadict['結果名次']
+
+#    df.to_excel('/Users/zooeytsai/'+fname+".xls")
+    df.to_excel('c:/tmp/seo/'+fname+".xls")
+
+    driver.quit()
+
+process_one()

SEO/fetch__url_content.py

@@ -0,0 +1,102 @@
+import traceback
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+# import rpyc
+import sys
+import docker
+# import googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+import html2text
+from userAgentRandomizer import userAgents
+from fp.fp import FreeProxy
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+db2 = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+
+
+cursor=db.query('select url from kw_url_search_result where url not in (select url from url_content) order by rand()')
+
+def restart_browser():
+    os.system('docker container restart proxy1')
+    ua = userAgents()
+    user_agent = ua.random()        
+    time.sleep(8)
+    options = webdriver.ChromeOptions()
+#    options.add_argument("--headless")
+#    proxy = FreeProxy().get()
+#    print(proxy)
+#    sys.exit()
+    options.add_argument('--proxy-server=socks5://127.0.0.1:9050')
+    options.add_argument("--user-agent=" +user_agent)
+    options.add_argument("--incognito")
+
+    driver=webdriver.Chrome(options=options)
+    driver.set_window_size(1400,1000)
+    driver.delete_all_cookies()
+
+    return driver
+
+
+def clean_txt(txt):
+    fulltxt=""
+    lines=txt.split("\n")
+    beginning=False
+    for l in lines:
+        ltxt=l.strip()
+        if '  * __'==ltxt:
+            continue
+        if '我要回覆'==ltxt:
+            continue
+        if beginning:
+            fulltxt+=l+"\n"
+        else:
+            if '  * __ 訂閱文章' in l:
+                beginning=True
+        if ('__ 連結 __ 回報 __ 只看樓主 __ 列印' in l):
+            break
+        if '__ 連結 __ 回報 __ 只看此人 __ 列印' in l:
+            break
+    print(fulltxt)
+    return fulltxt
+
+driver=restart_browser()
+table=db2['url_content']
+for c in cursor:
+    url=c['url']
+    print(c['url'])
+#    driver.get('https://whatismyipaddress.com/')
+#    time.sleep(9999)
+    driver.get(c['url'])
+    
+    time.sleep(5)
+    if 'Please Wait' in driver.title and 'Cloudflare' in driver.title:
+        driver=restart_browser()
+        continue
+    src=driver.page_source
+    h = html2text.HTML2Text()
+    h.ignore_links = True
+    txt=h.handle(src)
+    resulttxt=clean_txt(txt)
+    table.insert({'content':resulttxt,'url':url})
+    time.sleep(5)
+#    print()
+#    break
+
+
+#print(html2text.html2text("<p><strong>Zed's</strong> dead baby, <em>Zed's</em> dead.</p>"))

SEO/gnews_click.py

@@ -0,0 +1,76 @@
+import traceback
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+from selenium.webdriver.common.keys import Keys
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+# import rpyc
+import sys
+import docker
+# import googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+import html2text
+from userAgentRandomizer import userAgents
+
+
+def restart_browser():
+    os.system('docker container restart proxy1')
+    ua = userAgents()
+    user_agent = ua.random()        
+    time.sleep(8)
+    options = webdriver.ChromeOptions()
+#    options.add_argument("--headless")
+    options.add_argument('--proxy-server=socks5://127.0.0.1:9050')
+    options.add_argument("--user-agent=" +user_agent)
+    options.add_argument("--incognito")
+
+    driver=webdriver.Chrome(options=options)
+    driver.set_window_size(1400,1000)
+    driver.delete_all_cookies()
+
+    return driver
+
+
+driver=restart_browser()
+driver.get('https://news.google.com/topstories?hl=zh-TW&gl=TW&ceid=TW:zh-Hant')
+time.sleep(7)
+elmt=driver.find_element(By.XPATH,"//input[@aria-label='搜尋']")
+
+title_lst=['《咒術迴戰》降臨全家！不只推出獨家必收集點周邊 還能在1:1還原名場景與主角合照 引新聞',
+'梅雨季正是驗屋好時機！專家分享小撇步教你避免買到漏水屋 引新聞',
+'5月最夯球鞋款式大公開！女性消費者最愛「這個色調」 引新聞',
+'有影/本田仁美加入AKB48八年首登C位驚呼夢想成真！賣力學中文想挑戰翻唱《那些年》 引新聞',
+'萬綠叢中一點紅！白石麻衣化身「自衛隊」女教官 加入町田啓太「肉體派」新劇養眼陣容 引新聞',
+'超商變身辦公室！7-ELEVEN首創付費「多功能包廂專區」 遠距辦公上課更「便」民、開幕5折優惠 引新聞']
+
+title=random.choice(title_lst)
+#if elmt is None:
+#    elmt=driver.find_element(By.XPATH,"//input[@aria-label='搜尋']")
+if elmt is not None:
+    elmt.send_keys(title)
+    elmt.send_keys(Keys.ENTER)
+    elmt.send_keys(Keys.ENTER)
+
+time.sleep(7)
+
+elmts=driver.find_elements(By.XPATH,"//div[@jsname='esK7Lc']//div[@class='xrnccd']//a[@jsname='hXwDdf']")
+print(elmts[0].get_attribute('href'))
+print(elmts[0].text)
+
+
+time.sleep(9)
+

SEO/gsc-save-credential.py

@@ -0,0 +1,10 @@
+import searchconsole
+#account = searchconsole.authenticate(client_config='c:/keys/client_secret_162277274609-v1fsq5iscscl7e2ta4a8tc0og5tehl44.apps.googleusercontent.com.json',serialize='out.json')
+account = searchconsole.authenticate(client_config='c:/keys/client_secret_162277274609-v1fsq5iscscl7e2ta4a8tc0og5tehl44.apps.googleusercontent.com.json',credentials='c:/keys/out.json')
+
+#account.redirect_uri = 'https://localhost'
+webproperty = account['https://innews.com.tw/']
+
+report = webproperty.query.range('today', days=-7).dimension('query').get()
+print(report.rows)
+#http://localhost:8080

SEO/new_100.py

@@ -0,0 +1,117 @@
+import random
+import sys
+import dataset
+from selenium import webdriver
+import traceback
+import datetime
+import codecs
+import time
+import urllib
+import argparse
+import schedule
+
+import logging
+import sys
+from logging.handlers import SysLogHandler
+import socket
+import pandas as pd
+import socket
+import os
+_LOG_SERVER = ('hhh.ptt.cx', 514)
+logger = logging.getLogger('clickbot_100')
+handler1 = SysLogHandler(address=_LOG_SERVER,socktype=socket.SOCK_DGRAM)
+logger.addHandler(handler1)
+#logger.debug('[clickbot_100][清原]begin')
+
+hname=socket.gethostname()
+pid=str(os.getpid())
+logger.fatal('[clickbot_100]['+hname+']['+pid+']begin')
+
+
+def restart_browser():
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless') 
+    driver=webdriver.Chrome(options=options)
+    driver.set_window_size(950,6000)
+    return driver
+
+
+def process_one():
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    lst=[]
+    table=db['google_rank']
+    cursor = db.query('select term from seo.selected_kw')
+    # cursor=db.query('select term from selected_kw and term not in (SELECT distinct(keyword) FROM ig_tags.save_result where url like "%beastparadise.net%" and datediff(now(),dt)=0)')
+    for c in cursor:
+        lst.append(c['term'])
+
+    term=random.choice(lst)
+    print(term)
+    logger.debug('[clickbot_100]['+term+']')
+    driver=restart_browser()
+    escaped_search_term=urllib.parse.quote(term)
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(escaped_search_term, 100,'zh-TW')
+
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(6)
+    fname=term.replace(' ','_')
+    # driver.save_screenshot('c:/tmp/seo/'+fname+'.png')
+    df=pd.DataFrame()
+    # driver.get_screenshot_as_file("/Users/zooeytsai/排名100.png")
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+
+    clickelmt=None
+    cnt=1
+    datadict={'搜尋詞':[],'結果標題':[],'結果網址':[],'結果名次':[]}
+    
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            if 'taroboba-yuan.com' in href:
+                clickelmt=elmt
+                logger.debug('[clickbot_100]['+term+']['+str(cnt)+']')
+
+            print(href)
+            print(elmt.text)
+            datadict['搜尋詞'].append(term)
+            datadict['結果標題'].append(elmt.text)
+            datadict['結果網址'].append(href)
+            datadict['結果名次'].append(str(cnt))
+
+            table.insert({'title':elmt.text,'url':href,'keyword':term,'dt':datetime.datetime.now(),'num':cnt})
+            cnt+=1
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+    if clickelmt:
+        webdriver.ActionChains(driver).move_to_element(clickelmt).perform()
+        webdriver.ActionChains(driver).move_to_element(clickelmt).click().perform()
+    if len(datadict['結果標題'])<=0:
+        print('None')
+        driver.quit()
+        sys.exit()
+    df['搜尋詞']=datadict['搜尋詞']
+    df['結果標題']=datadict['結果標題']
+    df['結果網址']=datadict['結果網址']
+    df['結果名次']=datadict['結果名次']
+
+#    df.to_excel('/Users/zooeytsai/'+fname+".xls")
+    df.to_excel('c:/tmp/'+fname+".xls")
+
+    driver.quit()
+
+process_one()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--loop')
+args = parser.parse_args()
+
+if args.loop:
+
+#    schedule.every(6).minutes.do(process_one)
+    schedule.every(0.4).minutes.do(process_one)
+
+    while True:
+        schedule.run_pending()
+        time.sleep(1)

SEO/pet100.py

@@ -0,0 +1,114 @@
+import random
+import sys
+import dataset
+from selenium import webdriver
+import traceback
+import datetime
+import codecs
+import time
+import urllib
+import argparse
+import schedule
+import logging
+import sys
+from logging.handlers import SysLogHandler
+import socket
+import pandas as pd
+import socket
+import os
+_LOG_SERVER = ('hhh.ptt.cx', 514)
+logger = logging.getLogger('clickbot_100')
+handler1 = SysLogHandler(address=_LOG_SERVER,socktype=socket.SOCK_DGRAM)
+logger.addHandler(handler1)
+#logger.debug('[clickbot_100][清原]begin')
+
+hname=socket.gethostname()
+pid=str(os.getpid())
+logger.fatal('[clickbot_100]['+hname+']['+pid+']begin')
+
+
+def restart_browser():
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless') 
+    driver=webdriver.Chrome(options=options)
+    driver.set_window_size(950,6000)
+    return driver
+
+
+def process_one():
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    lst=[]
+    table=db['save_result']
+    cursor=db.query('select term from selected_kw where client="毛怪" and term not in (SELECT distinct(keyword) FROM seo.save_result where url like "%beastparadise.net%" and datediff(now(),dt)=0)')
+    for c in cursor:
+        lst.append(c['term'])
+
+    term=random.choice(lst)
+    print(term)
+    logger.debug('[clickbot_100]['+term+']')
+    driver=restart_browser()
+    escaped_search_term=urllib.parse.quote(term)
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(escaped_search_term, 100,'zh-TW')
+
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(6)
+    fname=term.replace(' ','_')
+    driver.save_screenshot('c:/tmp/seo/'+fname+'.png')
+    df=pd.DataFrame()
+
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+
+    clickelmt=None
+    cnt=1
+    datadict={'搜尋詞':[],'結果標題':[],'結果網址':[],'結果名次':[]}
+    
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            if 'taroboba-yuan.com' in href:
+                clickelmt=elmt
+                logger.debug('[clickbot_100]['+term+']['+str(cnt)+']')
+
+            print(href)
+            print(elmt.text)
+            datadict['搜尋詞'].append(term)
+            datadict['結果標題'].append(elmt.text)
+            datadict['結果網址'].append(href)
+            datadict['結果名次'].append(str(cnt))
+
+            table.insert({'title':elmt.text,'url':href,'keyword':term,'dt':datetime.datetime.now(),'num':cnt})
+            cnt+=1
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+    if clickelmt:
+        webdriver.ActionChains(driver).move_to_element(clickelmt).perform()
+        webdriver.ActionChains(driver).move_to_element(clickelmt).click().perform()
+    if len(datadict['結果標題'])<=0:
+        print('None')
+        driver.quit()
+        sys.exit()
+    df['搜尋詞']=datadict['搜尋詞']
+    df['結果標題']=datadict['結果標題']
+    df['結果網址']=datadict['結果網址']
+    df['結果名次']=datadict['結果名次']
+
+    df.to_excel('c:/tmp/seo/'+fname+".xls")
+
+    driver.quit()
+
+process_one()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--loop')
+args = parser.parse_args()
+
+if args.loop:
+
+#    schedule.every(6).minutes.do(process_one)
+    schedule.every(0.4).minutes.do(process_one)
+
+    while True:
+        schedule.run_pending()
+        time.sleep(1)

SEO/tiny_click100.py

@@ -0,0 +1,128 @@
+import random
+import sys
+import dataset
+from selenium import webdriver
+import traceback
+import datetime
+import codecs
+import time
+import urllib
+import argparse
+import schedule
+import logging
+import sys
+from logging.handlers import SysLogHandler
+import socket
+import pandas as pd
+import socket
+import os
+_LOG_SERVER = ('hhh.ptt.cx', 514)
+logger = logging.getLogger('clickbot_100')
+handler1 = SysLogHandler(address=_LOG_SERVER,socktype=socket.SOCK_DGRAM)
+logger.addHandler(handler1)
+#logger.debug('[clickbot_100][清原]begin')
+
+hname=socket.gethostname()
+pid=str(os.getpid())
+logger.fatal('[clickbot_100]['+hname+']['+pid+']begin')
+
+
+def restart_browser():
+    os.system('docker container restart headless-shell')
+    time.sleep(9)
+    chrome_options = webdriver.ChromeOptions()
+#    chrome_options.add_argument("--incognito")
+    chrome_options.add_argument("--headless")
+    chrome_options.add_argument("--no-sandbox")
+    chrome_options.add_argument("--disable-dev-shm-usage")
+    chrome_options.add_argument("start-maximized")
+    chrome_options.add_argument("user-data-dir=/tmp")
+    chrome_options.debugger_address="127.0.0.1:9222"
+    chrome98=r'C:\portable\webdriver\chrome98\chromedriver.exe'
+#    chrome98=r'/root/drivers/98/chromedriver'
+
+    driver = webdriver.Chrome(chrome_options=chrome_options,executable_path=chrome98)
+ 
+
+    return driver
+
+
+def process_one():
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+    lst=[]
+    table=db['save_result']
+    cursor=db.query('select term from selected_kw where client="清原" and term not in (SELECT distinct(keyword) FROM seo.save_result where url like "%taroboba-yuan.com%" and datediff(now(),dt)=0)')
+    for c in cursor:
+        lst.append(c['term'])
+
+    term=random.choice(lst)
+    print(term)
+    logger.debug('[clickbot_100]['+term+']')
+    driver=restart_browser()
+    escaped_search_term=urllib.parse.quote(term)
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(escaped_search_term, 100,'zh-TW')
+
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(6)
+    driver.save_screenshot('c:/tmp/test.png')
+
+    fname=term.replace(' ','_')
+    driver.save_screenshot('c:/tmp/seo/'+fname+'.png')
+    df=pd.DataFrame()
+
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+
+    clickelmt=None
+    cnt=1
+    datadict={'搜尋詞':[],'結果標題':[],'結果網址':[],'結果名次':[]}
+    
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            if 'taroboba-yuan.com' in href:
+                clickelmt=elmt
+                logger.debug('[clickbot_100]['+term+']['+str(cnt)+']')
+
+            print(href)
+            print(elmt.text)
+            datadict['搜尋詞'].append(term)
+            datadict['結果標題'].append(elmt.text)
+            datadict['結果網址'].append(href)
+            datadict['結果名次'].append(str(cnt))
+
+            table.insert({'title':elmt.text,'url':href,'keyword':term,'dt':datetime.datetime.now(),'num':cnt})
+            cnt+=1
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+    if clickelmt:
+        webdriver.ActionChains(driver).move_to_element(clickelmt).perform()
+        webdriver.ActionChains(driver).move_to_element(clickelmt).click().perform()
+    if len(datadict['結果標題'])<=0:
+        print('None')
+        driver.quit()
+        sys.exit()
+    df['搜尋詞']=datadict['搜尋詞']
+    df['結果標題']=datadict['結果標題']
+    df['結果網址']=datadict['結果網址']
+    df['結果名次']=datadict['結果名次']
+
+    df.to_excel('c:/tmp/seo/'+fname+".xls")
+
+    driver.quit()
+
+process_one()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--loop')
+args = parser.parse_args()
+
+if args.loop:
+
+    schedule.every(6).minutes.do(process_one)
+#    schedule.every(0.4).minutes.do(process_one)
+
+    while True:
+        schedule.run_pending()
+        time.sleep(1)

choozmo/cryptotest.py

@@ -0,0 +1,8 @@
+from passlib.context import CryptContext
+SECRET_KEY = "df2f77bd544240801a048bd4293afd8eeb7fff3cb7050e42c791db4b83ebadcd"
+ALGORITHM = "HS256"
+ACCESS_TOKEN_EXPIRE_DAYS = 5
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+print(pwd_context.hash('123456'))
+

choozmo/custom_profile_test.py

@@ -0,0 +1,43 @@
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import  googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+
+options = webdriver.ChromeOptions()
+options.add_argument("start-maximized")
+options.add_argument('user-data-dir=/opt/user')
+#options.add_argument('--profile-directory=Default')
+#options.add_argument('--profile-directory=Profile 1')
+#options.add_argument('--profile-directory=Profile 2')
+#options.add_argument('--profile-directory=Profile 3')
+#options.add_argument('--profile-directory=Profile 4')
+#options.add_argument('--profile-directory=Profile 5')
+#options.add_argument('--profile-directory=Profile 7')
+options.add_argument('--profile-directory=Profile 8')
+
+
+driver = webdriver.Remote(
+        command_executor='http://127.0.0.1:4444/wd/hub',
+    desired_capabilities=options.to_capabilities())
+driver.get('https://www.yahoo.com.tw')
+time.sleep(99999)
+#driver.set_window_size(1400,1000)

choozmo/db_clickjob.py

@@ -18,6 +18,7 @@ import time
 import rpyc
 import sys
 import docker
+import json
 
 driver=None
 headers = {

choozmo/fb_get_email.py

@@ -0,0 +1,101 @@
+import traceback
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import  googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+
+def process_one(driver):
+    lst=[]
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+#            print(href)
+            txt=elmt.text.split('\n')
+            print(txt[0])
+            lst.append({'title':txt[0],'url':href})
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+    return lst
+
+def process_query(driver,url):
+    driver.get(url)
+    time.sleep(4)
+    totallst=[]
+#    elmt=driver.find_element_by_xpath("//span[contains(text(),'關於')]/../span[contains(text(),'查看全部')]/a")
+    try:
+        elmt=driver.find_element_by_xpath("//span[contains(text(),'關於')]/..//a")
+        print(elmt)
+        print(elmt.text)
+        href=elmt.get_attribute('href')
+        driver.get(href)
+        time.sleep(5)
+        elmt=driver.find_element_by_xpath("//a[contains(@href,'mailto')]")
+        print(elmt.text)
+        print(elmt.get_attribute('href'))
+        return elmt.text
+#        time.sleep(3)
+    except:
+        print('not found')
+        return None    
+#    time.sleep(9999)
+#        try:
+#            elmt=driver.find_element_by_xpath("//a[@id='pnnext']")
+#        except:
+#            traceback.print_exc()
+#            print('pnnext exception')
+#            break
+#        time.sleep(1.5)
+#    return totallst
+
+
+result=[]
+driver=None
+
+def restart_browser():
+#    os.system('docker container restart p4444')
+#    time.sleep(10)
+
+    options = webdriver.ChromeOptions()
+#    options.add_argument("--proxy-server=socks5://130.61.93.198:1080")
+
+    driver=webdriver.Chrome(desired_capabilities=options.to_capabilities())
+    #driver = webdriver.Remote(
+    #    command_executor='http://127.0.0.1:4444/wd/hub',
+    #desired_capabilities=options.to_capabilities())
+#    desired_capabilities=DesiredCapabilities.CHROME)
+    driver.set_window_size(1400,1000)
+    return driver
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+cursor=db.query('select title,url from fb_gsearch order by rand()')
+lst=[]
+for c in cursor:
+    lst.append(c)
+    
+table=db['fb_progress']
+driver=restart_browser()
+for c in lst:
+    email=process_query(driver,c['url'])
+    table.insert({'url':c['url'],'email':email})
+    time.sleep(3)

choozmo/fb_gsearch.py

@@ -0,0 +1,21 @@
+import gsearch_general
+import sys
+import dataset
+import datetime
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['fb_gsearch']
+
+
+driver=gsearch_general.restart_browser()
+idx=gsearch_general.process_query(driver,'site:facebook.com pages 道路救援',number_results=100,language_code='zh-TW')
+print(idx)
+for x in idx:
+    x['dt']=datetime.datetime.now()
+    table.insert(x)
+
+if idx==None:
+    print(driver.page_source)
+    if '我們的系統偵測到您的電腦網路送出的流量有異常情況' in driver.page_source:
+        print('baned.....')
+        sys.exit()

choozmo/fetch_serp.py

@@ -0,0 +1,164 @@
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import  googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+
+#cursor=db.query('SELECT kw FROM hhh.hhh_contentgap_serp where ranking is not null;')
+#cursor=db.query('SELECT kw FROM hhh.hhh_contentgap_serp where kw not in (select distinct kw from hhh_contentgap_serp where id >= 155)')
+
+kwlst={}
+#for c in cursor:
+#    kwlst[c['kw']]=1
+
+
+
+
+
+table=db['hhh_contentgap_serp']
+curdir=os.path.realpath('.')
+
+#fr=codecs.open(curdir+os.sep+'contentgap.txt','r','utf-8')
+#fr=codecs.open(curdir+os.sep+'hhh\\seo\\contentgap.txt','r','utf-8')
+fr=codecs.open('C:\\gitlab\\kw_tools\\kw_tools\\hhh\\SEO\\contentgap.txt','r','utf-8')
+lines=fr.readlines()
+lst=[]
+for l in lines:
+    lst.append(l.replace('\n',''))
+
+
+
+
+
+headers = {
+        "Authorization": "Bearer " + "t35vhZtWNgvDNWHc3DJh0OKll3mcB9GvC8K2EAkBug2",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+def send_msg(kw):
+    params = {"message": "處理關鍵字: "+kw}  
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+
+def process_query(qs,number_results=10,language_code='en',pat='hhh.com.tw'):
+    global driver
+    escaped_search_term=urllib.parse.quote(qs)
+#    escaped_search_term = qs.replace(' ', '+')
+#    googleurl='https://www.google.com/search?q='+
+    googleurl='https://www.google.com/search?q=site:hsinfei.com'
+
+    driver.get(googleurl)
+    time.sleep(3)
+    print(driver.page_source)
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+    print(elmts)
+    idx=0
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            print(str(idx)+': '+href)
+            if pat in href:
+                return idx
+            idx+=1
+        except:
+            print('href exception')
+
+    try:
+        elmt=driver.find_element_by_xpath("//a[@id='pnnext']")
+        webdriver.ActionChains(driver).move_to_element(elmt).perform()
+        webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+    except:
+        print('pnnext exception')
+        return None
+
+    time.sleep(4)
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            print(str(idx)+': '+href)
+            if pat in href:
+                return idx
+            idx+=1
+
+        except:
+            print('href2 exception')
+
+    time.sleep(9999)
+
+result=[]
+driver=None
+
+def restart_browser():
+    os.system('docker container restart p4444')
+#    client = docker.from_env()
+#    ls=client.containers.list()
+#    print(ls)
+#    ls[0].restart()
+    time.sleep(10)
+
+    options = webdriver.ChromeOptions()
+#    options.add_argument("--proxy-server=https://47.241.72.41:80")
+
+    #driver=webdriver.Chrome(desired_capabilities=options.to_capabilities())
+    driver = webdriver.Remote(
+        command_executor='http://127.0.0.1:4444/wd/hub',
+#        command_executor='http://172.104.93.163:4444/wd/hub', 
+    #command_executor='http://dev2.choozmo.com:14444/wd/hub',
+    desired_capabilities=options.to_capabilities())
+#    desired_capabilities=DesiredCapabilities.CHROME)
+    driver.set_window_size(1400,1000)
+    print('driver ready')
+    return driver
+
+
+for l in lst:
+#for l in lst[21:]:
+
+#for l in lst[32:]:
+#for l in lst[42:]:
+
+    if True:
+#    if kwlst.get(l) is None:
+        driver=restart_browser()
+
+    #    l='房間 油漆'
+    #    idx=process_query(,number_results=100,language_code='zh-TW',pat='hhh.com.tw')
+        idx=process_query(l,number_results=100,language_code='zh-TW',pat='hhh.com.tw')
+#        if idx is None:
+#            sys.exit()
+        table.insert({'kw':l,'ranking':idx,'dt':datetime.datetime.now()})
+        print({'kw':l,'ranking':idx})
+        db.commit()
+    #    time.sleep(9999)
+
+#        time.sleep(4)

choozmo/gen_counter_db.py

@@ -0,0 +1,17 @@
+import time
+import dataset
+import os
+import codecs
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+table=db['kw_contentgap']
+fr=codecs.open('C:\\gitlab\\kw_tools\\kw_tools\\hhh\\SEO\\contentgap.txt','r','utf-8')
+lines=fr.readlines()
+lst=[]
+for l in lines:
+    table.insert({'kw':l.replace('\n','')})
+db.commit()
+
+
+#table=db['counter']
+#table.insert({'cnt':0})

choozmo/gmailtest.py

@@ -0,0 +1,72 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START gmail_quickstart]
+from __future__ import print_function
+
+import os.path
+
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+# If modifying these scopes, delete the file token.json.
+SCOPES = ['https://www.googleapis.com/auth/gmail.readonly']
+
+
+def main():
+    """Shows basic usage of the Gmail API.
+    Lists the user's Gmail labels.
+    """
+    creds = None
+    # The file token.json stores the user's access and refresh tokens, and is
+    # created automatically when the authorization flow completes for the first
+    # time.
+    if os.path.exists('token.json'):
+        creds = Credentials.from_authorized_user_file('token.json', SCOPES)
+    # If there are no (valid) credentials available, let the user log in.
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            flow = InstalledAppFlow.from_client_secrets_file(
+                'c:\\keys\\client_secret_392946835471.json', SCOPES)
+            creds = flow.run_local_server(port=0)
+        # Save the credentials for the next run
+        with open('token.json', 'w') as token:
+            token.write(creds.to_json())
+
+    try:
+        # Call the Gmail API
+        service = build('gmail', 'v1', credentials=creds)
+        results = service.users().labels().list(userId='me').execute()
+        labels = results.get('labels', [])
+
+        if not labels:
+            print('No labels found.')
+            return
+        print('Labels:')
+        for label in labels:
+            print(label['name'])
+
+    except HttpError as error:
+        # TODO(developer) - Handle errors from gmail API.
+        print(f'An error occurred: {error}')
+
+
+if __name__ == '__main__':
+    main()
+# [END gmail_quickstart]

choozmo/gsc_tree2.py

@@ -3,7 +3,7 @@ import dataset
 import os
 import networkx as nx
 from pyvis.network import Network
-
+import sys
 
 db = dataset.connect('sqlite:///:memory:')
 table=db['tmp']
@@ -25,10 +25,38 @@ def checkig():
     path2=os.path.sep.join(elmts[0:-1])
     keysdir=path2+os.path.sep+'../keys'+os.path.sep
 
-    account = searchconsole.authenticate(client_config='/Users/zooeytsai/Downloads/client_secret4.json')
-    webproperty = account['https://bennis.com.tw/']
+#    account = searchconsole.authenticate(client_config='/Users/zooeytsai/Downloads/client_secret4.json')
+
+#    account = searchconsole.authenticate(client_config='c:/keys/client_secret4.json',serialize='c:/keys/credentials20220524.json')
+    account = searchconsole.authenticate(client_config='c:/keys/client_secret4.json',credentials='c:/keys/credentials20220524.json')
+
+#    account = searchconsole.authenticate(client_config='c:/keys/client_secret.json',credentials='c:/keys/credentials.json')
+
+    import pprint
+    import codecs
+#    pprint.pprint(vars(account.service.sites))
+#    pprint.pprint(vars(account.service))
+    site_list = account.service.sites().list().execute()
+    print(site_list)
+#    print(account.service.sites().list())
+#    fw=codecs.open('c:/tmp/pprint.txt','w','utf-8')
+#    fw.write(pprint.pformat(vars(account)))
+#    fw.close()
+#    sys.exit()
+#  credentials='credentials.json'
+#    print(account.)
+
+#    webproperty = account['https://bennis.com.tw/']
+    webproperty = account['sc-domain:bennis.com.tw']
+
+
     print(webproperty)
-    report = webproperty.query.range('today', days=-2).dimension('query').get()
+#    report = webproperty.query.range('today', days=-7).dimension('query').get()
+    report=webproperty.query.range('today', days=-4).dimension('page','query').get()
+
+
+    print(report)
+
     # report = webproperty.query.range('2022-05-20', '2022-05-23').dimension('page', 'query').get()
 #    account = searchconsole.authenticate(client_config='C:\\gitlab\\kw_tools\\kw_tools\\hhh\\keys\\client_secret.json',credentials='C:\\gitlab\\kw_tools\\kw_tools\\hhh\\keys\\credentials.json')
     G = nx.Graph()
@@ -43,8 +71,6 @@ def checkig():
     total_idx=0
 
     for r in report.rows:
-        if 'hhh.com.tw/designers/cases/' not in r[0]:
-            continue
         if rdict.get(r[0]) is None:
             total_idx+=1
             rid=total_idx
@@ -55,7 +81,7 @@ def checkig():
         result.append(entry)
 
     print('list done')
-
+    print(result)
     for r in result:
         table.insert(r)
     db.commit()

choozmo/gsc_tree3.py

@@ -97,7 +97,9 @@ def checkig():
 
     total_idx=0
     for c in cursor:
-        G.add_edge(c['q'],c['rid'],weight=3,width=3,borderwidth=3)
+        G.add_edge(c['q'],c['rid'],borderwidth=3)
+#        G.add_edge(c['q'],c['rid'],weight=3,width=3,borderwidth=3)
+
     remove=[]
     G.remove_edges_from(nx.selfloop_edges(G))
     G2=G

choozmo/gsearch_libtest.py

@@ -0,0 +1,3 @@
+from googlesearch import search
+for url in search('"Breaking Code" WordPress blog', stop=20):
+    print(url)

choozmo/igool/igtree.py

@@ -50,8 +50,15 @@ import pickle
 #kw='馬卡龍'
 #kw='馬林糖'
 #kw='檸檬塔'
-kw='泡芙'
-
+#kw='泡芙'
+kw='mean snapchat'
+#kw='留學'
+#kw='勞力士'
+#kw='白蟻'
+#kw='影片製作'
+#kw='ai 合成'
+
+#kw='菲律賓'
 #kw='生巧克力'
 #kw='牛奶巧克力'
 
@@ -59,8 +66,8 @@ kw='泡芙'
 
 #s={'suggests':[]}
 s = suggests.suggests.get_suggests(kw, source='google')
-#G = nx.Graph()
-G = pickle.load( open( "gs2.p", "rb" ) )
+G = nx.Graph()
+#G = pickle.load( open( "gs2.p", "rb" ) )
 
 
 #G.remove_node('巧克力囊腫')

choozmo/imp_selected.py

@@ -0,0 +1,16 @@
+import dataset
+import codecs
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+
+
+table=db['selected_kw']
+
+fr=codecs.open('C:\\tmp\\selected.csv','r','utf-8')
+lines=fr.readlines()
+lst=[]
+for l in lines:
+    term=l.replace('\n','')
+    term=term.replace('\r','')
+    table.insert({'term':term,'client':'清原'})
+
+

choozmo/imp_ytlist.py

@@ -0,0 +1,34 @@
+import json
+import os
+import csv
+import dataset
+import datetime
+import codecs
+import sys
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['sbir_yt']
+
+fr=codecs.open('c:/tmp/sbir_yt.csv','r','utf-8')
+lines=fr.readlines()
+fr.close()
+for l in lines:
+    tmp=l.replace('https://youtu.be/','').strip()
+    table.insert({'url':tmp})
+
+sys.exit()
+
+
+table=db['customer_list']
+print (os.getcwd())
+print(os.path.realpath('.'))
+
+
+cursor=db.query('select * from customer_list')
+for c in cursor:
+    print(c['name'])
+    print(c['updated'])
+
+#jstr=json.dumps(qlist)
+#print(jstr)
+

choozmo/linkedin_detail.py

@@ -0,0 +1,91 @@
+import traceback
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.keys import Keys
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import  googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+
+
+def scrolling(driver,pgnum):
+    ub = driver.find_element_by_css_selector('body')
+    for i in range(pgnum):
+        ub.send_keys(Keys.PAGE_DOWN)
+        if pgnum>1:
+            time.sleep(0.3)
+
+
+def process_query(driver,c):
+    global db
+    url=c['href']
+    driver.get('https://www.linkedin.com/in/'+url+'/overlay/contact-info/')
+    time.sleep(3)
+    email=''
+    phone=''
+    try:
+        e_email=driver.find_element_by_xpath(".//a[contains(@href,'mailto:') ]")
+        print(e_email.text)
+        email=e_email.text
+        print(e_email.get_attribute('href'))
+    except:
+        print('no email')
+    try:
+        e_phone=driver.find_element_by_xpath("//section[@class='pv-contact-info__contact-type ci-phone' ]//li[contains(@class,'pv-contact-info__ci-container')]")
+        print(e_phone.text)
+        phone=e_phone.text
+    except:
+        print('no phone')
+
+    return {'email':email,'phone':phone}
+
+
+def restart_browser():
+#    os.system('docker container restart p4444')
+#    time.sleep(10)
+
+    options = webdriver.ChromeOptions()
+#    options.add_argument("--proxy-server=socks5://130.61.93.198:1080")
+    options.add_argument("start-maximized")
+    options.add_argument('user-data-dir=C:\\Users\\jared\\AppData\\Local\\Google\\Chrome\\User Data')
+    options.add_argument('--profile-directory=Default')
+
+    driver=webdriver.Chrome(desired_capabilities=options.to_capabilities())
+    #driver = webdriver.Remote(
+    #    command_executor='http://127.0.0.1:4444/wd/hub',
+    #desired_capabilities=options.to_capabilities())
+#    desired_capabilities=DesiredCapabilities.CHROME)
+    driver.set_window_size(1400,1000)
+    return driver
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+lst=[]
+cursor=db.query('select id,href from linkedin_list where email is null order by rand()')
+for c in cursor:
+    lst.append(c)
+
+driver=restart_browser()
+for c in lst:
+    data=process_query(driver,c)    
+    db.query('update linkedin_list set email="'+data['email']+'", phone="'+data['phone']+'" where id="'+str(c['id'])+'"')
+    print(data)
+    time.sleep(2)
+
+time.sleep(9999)

choozmo/mail_list_imp_csv.py

@@ -0,0 +1,19 @@
+import time
+from cv2 import line
+import dataset
+import os
+import codecs
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['mail_list']
+
+fr=codecs.open(r'C:\Users\jared\Downloads\meettaipei.csv','r','utf-8')
+lines=fr.readlines()
+fr.close()
+
+for l in lines:
+    l=l.replace('\n','').replace('\r','')
+    elmts=l.split(',')
+    body={'title':elmts[0],'email':elmts[1],'enabled':1}
+    print(body)
+    table.insert(body)

choozmo/phantomtest.py

@@ -0,0 +1,12 @@
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+
+driver = webdriver.Remote(
+    command_executor='http://127.0.0.1:8910',
+    desired_capabilities=DesiredCapabilities.PHANTOMJS)
+
+driver.get('http://www.yahoo.com')
+print(driver.page_source)
+#driver.find_element_by_css_selector('a[title="hello"]').click()
+
+driver.quit()

choozmo/profile_selenium.py

@@ -0,0 +1,87 @@
+import traceback
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import  googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+
+def process_one(driver):
+    lst=[]
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+#            print(href)
+            txt=elmt.text.split('\n')
+            print(txt[0])
+            lst.append({'title':txt[0],'url':href})
+        except:
+            print('href2 exception')
+            traceback.print_exc()
+    return lst
+
+def process_query(driver,qs,number_results=10,language_code='en'):
+    escaped_search_term=urllib.parse.quote(qs)
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(escaped_search_term, number_results+1,language_code)
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(3)
+    totallst=[]
+    while True:
+        lst=process_one(driver)
+        totallst+=lst
+        try:
+            elmt=driver.find_element_by_xpath("//a[@id='pnnext']")
+            webdriver.ActionChains(driver).move_to_element(elmt).perform()
+            webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+        except:
+            traceback.print_exc()
+            print('pnnext exception')
+            break
+        time.sleep(1.5)
+    return totallst
+
+
+result=[]
+driver=None
+
+def restart_browser():
+#    os.system('docker container restart p4444')
+#    time.sleep(10)
+
+    options = webdriver.ChromeOptions()
+    options.add_argument("start-maximized")
+    options.add_argument('user-data-dir=C:\\Users\\jared\\AppData\\Local\\Google\\Chrome\\User Data')
+    options.add_argument('--profile-directory=Profile 77')
+
+    driver=webdriver.Chrome(chrome_options=options)
+
+    #driver = webdriver.Remote(
+    #    command_executor='http://127.0.0.1:4444/wd/hub',
+    #desired_capabilities=options.to_capabilities())
+#    desired_capabilities=DesiredCapabilities.CHROME)
+    driver.set_window_size(1400,1000)
+    return driver
+
+
+driver=restart_browser()
+driver.get('http://facebook.com')
+time.sleep(9999)

choozmo/proxytest.py

@@ -0,0 +1,42 @@
+import urllib.request
+import socket
+import urllib.error
+import sys
+from Proxy_List_Scrapper import Scrapper, Proxy, ScrapperException
+
+def is_bad_proxy(pip):    
+    try:
+        proxy_handler = urllib.request.ProxyHandler({'http': pip})
+        opener = urllib.request.build_opener(proxy_handler)
+        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+        urllib.request.install_opener(opener)
+        req=urllib.request.Request('http://www.google.com')  # change the URL to test here
+        sock=urllib.request.urlopen(req)
+    except urllib.error.HTTPError as e:
+        print('Error code: ', e.code)
+        return e.code
+    except Exception as detail:
+        print("ERROR:", detail)
+        return True
+    return False
+
+def main():
+    socket.setdefaulttimeout(120)
+#    scrapper = Scrapper(category='GOOGLE', print_err_trace=False)
+    scrapper = Scrapper(category='ALL', print_err_trace=False)
+
+#    print(data)
+    # two sample proxy IPs
+    data = scrapper.getProxies()
+    for d in data.proxies:
+        currentProxy=d.ip+':'+str(d.port)
+        print(d)
+        if is_bad_proxy(currentProxy):
+            print("Bad Proxy %s" % (currentProxy))
+        else:
+            print("%s is working" % (currentProxy))
+
+
+
+if __name__ == '__main__':
+    main() 

choozmo/redis_test.py

@@ -0,0 +1,15 @@
+import redis
+import time
+import json
+import random
+r = redis.Redis(host='db.ptt.cx', port=6379, db=2,password='choozmo9')
+
+vlen=r.llen('vtuber')
+for i in range(0,vlen ):
+    print(r.lpop('vtuber'))
+
+#    print(r.lindex('vtuber', i))
+#print(data)
+#r.rpush('vtuber','123')
+#r.rpush('vtuber','456')
+

choozmo/save_search_result.py

@@ -0,0 +1,52 @@
+import gsearch_general
+import sys
+import dataset
+import datetime
+import codecs
+import time
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['save_result']
+
+driver=gsearch_general.restart_browser()
+fr=codecs.open('c:/tmp/food.csv','r','utf-8')
+lines=fr.readlines()
+fr.close()
+
+namedict={}
+cursor=db.query('select distinct(keyword) as kw from save_result')
+for c in cursor:
+    kw=c['kw'].replace('\n','')
+    kw=c['kw'].replace('\r','')
+    namedict[kw]=1
+
+print(namedict)
+print(lines)
+#time.sleep(9999)
+
+#keyword='檸檬原汁'
+#lines=['芋頭 外帶']
+#lines=['布丁 牛奶 冰']
+#lines=['芋圓 加盟']
+
+for l in lines:
+    l=l.replace('\r','')
+    l=l.replace('\n','')
+    if namedict.get(l) is not None:
+        continue
+    keyword=l
+    print(keyword)
+    idx=gsearch_general.process_query(driver,keyword,number_results=100,language_code='zh-TW',enable_next=False)
+    print(idx)
+    cnt=1
+    for x in idx:
+        x['keyword']=keyword
+        x['dt']=datetime.datetime.now()
+        x['num']=str(cnt)
+        cnt+=1
+        table.insert(x)
+#    if idx==None:
+#        print(driver.page_source)
+    if '我們的系統偵測到您的電腦網路送出的流量有異常情況' in driver.page_source:
+        print('baned.....')
+        sys.exit()
+

choozmo/seo_routine.py

@@ -0,0 +1,50 @@
+import gsearch_general
+import sys
+import dataset
+import datetime
+import codecs
+import time
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['save_result']
+
+driver=gsearch_general.restart_browser()
+#fr=codecs.open('c:/tmp/food.csv','r','utf-8')
+#lines=fr.readlines()
+#fr.close()
+
+#namedict={}
+lines=[]
+cursor=db.query('SELECT term FROM seo.selected_kw where term not in (SELECT keyword FROM seo.save_result where datediff(now(),dt)=0);')
+for c in cursor:
+    kw=c['term'].replace('\n','')
+    kw=c['term'].replace('\r','')
+    lines.append(kw)
+
+print(lines)
+#time.sleep(9999)
+
+#keyword='檸檬原汁'
+#lines=['芋頭 外帶']
+#lines=['布丁 牛奶 冰']
+#lines=['芋圓 加盟']
+
+for l in lines:
+    l=l.replace('\r','')
+    l=l.replace('\n','')
+    keyword=l
+    print(keyword)
+    idx=gsearch_general.process_query(driver,keyword,number_results=100,language_code='zh-TW',enable_next=False)
+    print(idx)
+    cnt=1
+    for x in idx:
+        x['keyword']=keyword
+        x['dt']=datetime.datetime.now()
+        x['num']=str(cnt)
+        cnt+=1
+        table.insert(x)
+#    if idx==None:
+#        print(driver.page_source)
+    if '我們的系統偵測到您的電腦網路送出的流量有異常情況' in driver.page_source:
+        print('baned.....')
+        sys.exit()
+

choozmo/serp_searchhome.py

@@ -0,0 +1,158 @@
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import  googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+
+#cursor=db.query('SELECT kw FROM hhh.hhh_contentgap_serp where ranking is not null;')
+#cursor=db.query('SELECT kw FROM hhh.hhh_contentgap_serp where kw not in (select distinct kw from hhh_contentgap_serp where id >= 155)')
+
+kwlst={}
+#for c in cursor:
+#    kwlst[c['kw']]=1
+
+
+
+
+
+table=db['searchome_contentgap_serp']
+curdir=os.path.realpath('.')
+
+#fr=codecs.open(curdir+os.sep+'contentgap.txt','r','utf-8')
+#fr=codecs.open(curdir+os.sep+'hhh\\seo\\contentgap.txt','r','utf-8')
+#fr=codecs.open('C:\\gitlab\\kw_tools\\kw_tools\\hhh\\SEO\\contentgap.txt','r','utf-8')
+#lines=fr.readlines()
+lst=[]
+#for l in lines:
+#    lst.append(l.replace('\n',''))
+cursor=db.query('select distinct kw from  searchome_contentgap_serp where ranking is null')
+for c in cursor:
+    lst.append(c['kw'])
+
+
+
+headers = {
+        "Authorization": "Bearer " + "t35vhZtWNgvDNWHc3DJh0OKll3mcB9GvC8K2EAkBug2",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+def send_msg(kw):
+    params = {"message": "處理關鍵字: "+kw}  
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+
+def process_query(qs,number_results=10,language_code='en',pat='searchome.net'):
+    global driver
+    escaped_search_term=urllib.parse.quote(qs)
+#    escaped_search_term = qs.replace(' ', '+')
+#    googleurl='https://www.google.com/search?q='+
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(escaped_search_term, number_results+1,language_code)
+
+    driver.get(googleurl)
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+    idx=0
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            print(str(idx)+': '+href)
+            if pat in href:
+                return idx
+            idx+=1
+        except:
+            print('href exception')
+
+    try:
+        elmt=driver.find_element_by_xpath("//a[@id='pnnext']")
+        webdriver.ActionChains(driver).move_to_element(elmt).perform()
+        webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+    except:
+        print('pnnext exception')
+        return None
+
+    time.sleep(4)
+    elmts=driver.find_elements_by_xpath("//div[@class='yuRUbf']/a")
+
+    for elmt in elmts:
+        try:
+            href=elmt.get_attribute('href')
+            print(str(idx)+': '+href)
+            if pat in href:
+                return idx
+            idx+=1
+
+        except:
+            print('href2 exception')
+
+
+
+result=[]
+driver=None
+
+def restart_browser():
+#    client = docker.from_env()
+#    ls=client.containers.list()
+#    print(ls)
+#    ls[0].restart()
+    time.sleep(10)
+
+    options = webdriver.ChromeOptions()
+#    options.add_argument("--proxy-server=socks5://191.96.42.80:1080")
+
+    driver=webdriver.Chrome(desired_capabilities=options.to_capabilities())
+#    driver = webdriver.Remote(
+#        command_executor='http://127.0.0.1:4444/wd/hub',
+#        command_executor='http://172.104.93.163:4444/wd/hub', 
+    #command_executor='http://dev2.choozmo.com:14444/wd/hub',
+#    desired_capabilities=options.to_capabilities())
+#    desired_capabilities=DesiredCapabilities.CHROME)
+    driver.set_window_size(1400,1000)
+    return driver
+
+for l in lst:
+#for l in lst[2:]:
+    if True:
+#    if kwlst.get(l) is None:
+        driver=restart_browser()
+
+    #    l='房間 油漆'
+    #    idx=process_query(,number_results=100,language_code='zh-TW',pat='hhh.com.tw')
+#        idx=process_query(l,number_results=100,language_code='zh-TW',pat='hhh.com.tw')
+        idx=process_query(l,number_results=100,language_code='zh-TW',pat='searchome.net')
+
+#        table.insert({'kw':l,'ranking':idx,'dt':datetime.datetime.now()})
+        table.upsert({'kw':l,'ranking':idx,'dt':datetime.datetime.now()},['kw'])
+
+        print({'kw':l,'ranking':idx})
+        db.commit()
+        driver.quit()
+    #    time.sleep(9999)
+
+#        time.sleep(4)

choozmo/test.py

@@ -0,0 +1,11 @@
+import dataset
+import codecs
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+
+
+#cursor=db.query('select keyword from save_result')
+cursor=db.query('select * from selected_kw')
+
+for c in cursor:
+    print(c)
+

console/hhh_start_process.py

@@ -0,0 +1,24 @@
+import rpyc
+import os
+# RDP HOME
+conn = rpyc.classic.connect("192.168.192.58",18812)
+conn.execute('import os')
+conn.execute('import subprocess')
+conn.execute('import psutil')
+
+rsubprocess=conn.modules.subprocess
+
+rpsutil=conn.modules.psutil
+#processlist=list()
+for process in rpsutil.process_iter():
+    try:
+        print(process.name())
+        print(process.exe())
+        print(process.username())
+    except:
+        print('except')
+#    print(process)
+
+#rsubprocess.Popen(['python', 'C:/gitlab/kw_tools/hhh/SEO/local_1777.py'],cwd=r'C:\gitlab\kw_tools\hhh\SEO')
+
+

docker/alpine-basic/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:alpine3.14
+# update apk repo
+RUN echo "http://dl-4.alpinelinux.org/alpine/v3.14/main" >> /etc/apk/repositories && \
+    echo "http://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories
+RUN apk update
+RUN apk add build-base
+RUN apk add libffi-dev
+RUN apk add mariadb-dev
+RUN apk add chromium chromium-chromedriver
+RUN apk add bash
+RUN apk add tmux
+RUN pip3 install --upgrade pip
+RUN pip3 install dataset
+RUN pip3 install mysqlclient
+RUN pip3 install selenium

docker/alpine-basic/script.txt

@@ -0,0 +1,3 @@
+docker tag alpine-basic choozmodocker/alpine-basic:latest
+docker push choozmodocker/alpine-basic:latest
+docker run -it --name poi -v c:/gitlab/GooglePoiCrawler:/app choozmodocker/alpine-chrome:latest /bin/bash

docker/alpine-chrome/script.txt

@@ -0,0 +1,3 @@
+docker tag alpine-chrome choozmodocker/alpine-chrome:latest
+docker push choozmodocker/alpine-chrome:latest
+docker run -it --name poi -v c:/gitlab/GooglePoiCrawler:/app choozmodocker/alpine-chrome:latest /bin/bash

docker/alpine-hhh/Dockerfile

@@ -0,0 +1,4 @@
+FROM choozmodocker/alpine-basic:latest
+RUN pip3 install userAgentRandomizer
+ENTRYPOINT ["python3", "/app/local_1777.py"]
+ 

docker/alpine-poi/Dockerfile

@@ -2,6 +2,7 @@ FROM choozmodocker/alpine-basic:latest
 RUN pip3 install bs4
 RUN pip3 install selenium-wire 
 RUN pip3 install pandas
+RUN pip3 install redis
 RUN pip3 install pymysql
 RUN pip3 install requests
 ENTRYPOINT ["python3", "/app/run4.py"]

docker/alpine-poi/script.txt

@@ -0,0 +1,6 @@
+docker tag alpine-poi choozmodocker/alpine-poi:latest
+docker push choozmodocker/alpine-poi:latest
+docker run -it --name poi1 -v c:/gitlab/GooglePoiCrawler:/app choozmodocker/alpine-poi:latest
+
+
+docker run -it --name poi -v c:/gitlab/GooglePoiCrawler:/app alpine-poi

docker/alpine-seo/Dockerfile

@@ -0,0 +1,3 @@
+FROM choozmodocker/alpine-basic:latest
+ENTRYPOINT ["python3", "/app/click_commerce.py"]
+

docker/alpine-test/Dockerfile

@@ -2,8 +2,11 @@ FROM joyzoursky/python-chromedriver:3.9-alpine-selenium
 RUN apk add bash
 RUN apk add --virtual build-deps gcc python3-dev musl-dev
 RUN apk add --no-cache mariadb-dev
+RUN apk add libffi-dev
 RUN apk add tmux
 RUN pip3 install --upgrade pip
+RUN pip3 install bs4
+RUN pip3 install selenium-wire
 RUN pip3 install dataset
 RUN pip3 install mysqlclient
 

docker/chrome99-test/Dockerfile

@@ -0,0 +1,27 @@
+FROM selenium/standalone-chrome:99.0
+RUN sudo apt-get update
+RUN sudo apt-get -y install gcc python3-dev libmariadb-dev
+RUN sudo apt-get -y install python3
+RUN sudo apt-get -y install python3-pip
+RUN sudo apt-get -y install tmux
+RUN sudo pip3 install --upgrade pip
+RUN sudo pip3 install bs4
+RUN sudo pip3 install selenium-wire 
+RUN sudo pip3 install dataset
+RUN sudo pip3 install mysqlclient
+RUN sudo pip3 install pandas
+RUN sudo pip3 install pymysql
+
+#RUN apk add bash
+#RUN apk add --no-cache mariadb-dev
+#RUN apk add libffi-dev
+#RUN apk add tmux
+#RUN pip3 install --upgrade pip
+#RUN pip3 install bs4
+#RUN pip3 install selenium-wire
+#RUN pip3 install dataset
+#RUN pip3 install mysqlclient
+
+#RUN pip3 install selenium-wire
+CMD ["python3"]
+

docker/gat/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "cheatsheets",
+  "description": "Devhints.io",
+  "version": "1.0.0",
+  "author": "Rico Sta. Cruz <rstacruz@users.noreply.github.com>",
+  "dependencies": {
+    "autoprefixer": "^9.8.2",
+    "dom101": "^2.2.1",
+    "hint.css": "^2.6.0",
+    "isotope-layout": "^3.0.6",
+    "lodash.noop": "^3.0.1",
+    "modularscale-sass": "^3.0.10",
+    "onmount": "^1.3.0",
+    "postcss-modules": "^2.0.0",
+    "prismjs": "^1.20.0",
+    "sanitize.css": "^11.0.1",
+    "sass": "^1.26.8"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.10.2",
+    "@babel/preset-env": "^7.10.2",
+    "@rstacruz/prettier-plugin-markdown-code-fences": "^1.0.0",
+    "jest": "26.0.1",
+    "jest-html": "1.5.0",
+    "netlify-plugin-minify-html": "^0.2.3",
+    "npm-run-all": "^4.1.5",
+    "parcel-bundler": "^1.12.4",
+    "prettier": "^2.0.5",
+    "wait-on": "^5.0.1"
+  },
+  "homepage": "https://devhints.io/",
+  "jest": {
+    "snapshotSerializers": [
+      "<rootDir>/node_modules/jest-html"
+    ]
+  },
+  "license": "MIT",
+  "main": "index.js",
+  "private": true,
+  "repository": "https://github.com/rstacruz/cheatsheets.git",
+  "scripts": {
+    "build": "run-s -s 'parcel:*:build' jekyll:build",
+    "dev": "run-p -sl jekyll:watch 'parcel:*:watch'",
+    "jekyll:build": "bundle exec jekyll build",
+    "jekyll:watch": "wait-on assets/packed/app.js && wait-on _includes/2017/critical/critical-sheet.css && bundle exec jekyll serve --safe --trace --drafts --watch --incremental --host ${HOST:-0.0.0.0} --port ${PORT:-3000}",
+    "jest-html": "jest-html",
+    "parcel:app:build": "parcel build '_parcel/app.js' -d assets/packed --no-source-maps --no-autoinstall",
+    "parcel:app:watch": "parcel watch '_parcel/app.js' -d assets/packed --no-source-maps --no-autoinstall",
+    "parcel:build": "run-s -s 'parcel:*:build'",
+    "parcel:critical:build": "parcel build '_parcel/critical*.js' -d _includes/2017/critical --no-source-maps --no-autoinstall",
+    "parcel:critical:watch": "parcel watch '_parcel/critical*.js' -d _includes/2017/critical --no-source-maps --no-autoinstall",
+    "predev": "rm -rf assets/packed _includes/2017/critical",
+    "prejekyll:build": "bundle",
+    "prejekyll:watch": "bundle",
+    "prettier:format": "prettier --write '_parcel/**/*.{js,scss}'",
+    "test": "jest",
+    "test:smoke": "bash _support/smoke_test.sh"
+  },
+  "volta": {
+    "node": "16.14.2",
+    "yarn": "1.22.18"
+  }
+}

docker/headless-clickbot/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.9.13-alpine3.16
+RUN echo "http://dl-4.alpinelinux.org/alpine/v3.16/main" >> /etc/apk/repositories && \
+    echo "http://dl-4.alpinelinux.org/alpine/v3.16/community" >> /etc/apk/repositories
+RUN apk update
+RUN apk add build-base
+RUN apk add libffi-dev
+RUN apk add mariadb-dev
+RUN apk add chromium chromium-chromedriver
+RUN apk add bash
+RUN apk add tmux
+RUN pip3 install --upgrade pip
+RUN pip3 install dataset
+RUN pip3 install mysqlclient
+RUN pip3 install selenium
+
+RUN pip3 install selenium
+RUN pip3 install redis
+RUN pip3 install requests
+ENTRYPOINT ["python3", "/app/general_clickbot_hhh.py"]

docker/hhh-backstage-docker/Dockerfile

@@ -0,0 +1,4 @@
+FROM shengeih/nginx-php7:v2
+RUN sed -i '107s/^.*$/pm.max_children = 25/' /opt/docker/etc/php/fpm/pool.d/application.conf
+
+

docker/hhh-chrome/Dockerfile

@@ -0,0 +1,20 @@
+FROM zenika/alpine-chrome
+USER root
+RUN apk update
+RUN apk add python3
+RUN apk add py3-pip
+RUN apk add python3-dev
+RUN apk add build-base
+RUN apk add libffi-dev
+RUN apk add mariadb-dev
+RUN apk add chromium chromium-chromedriver
+RUN apk add tmux
+RUN apk add bash
+RUN pip3 install --upgrade pip
+RUN pip3 install selenium
+RUN pip3 install dataset
+RUN pip3 install userAgentRandomizer
+RUN pip3 install mysqlclient
+USER chrome
+#ENTRYPOINT ["python3", "/app/fast_9222.py"]
+ENTRYPOINT ["python3", "/app/local_1777.py"]

docker/ubuntu-video/Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:20.04
+RUN apt-get update
+RUN apt-get -y install gcc python3-dev libmariadb-dev
+RUN apt-get -y install python3
+RUN apt-get -y install python3-pip
+RUN apt-get -y install tmux
+RUN pip3 install --upgrade pip
+#RUN sudo pip3 install bs4
+#RUN sudo pip3 install selenium-wire 
+RUN pip3 install dataset
+RUN pip3 install mysqlclient
+#RUN sudo pip3 install pandas
+#RUN sudo pip3 install pymysql
+
+#RUN apk add bash
+#RUN apk add --no-cache mariadb-dev
+#RUN apk add libffi-dev
+#RUN apk add tmux
+#RUN pip3 install --upgrade pip
+#RUN pip3 install bs4
+#RUN pip3 install selenium-wire
+#RUN pip3 install dataset
+#RUN pip3 install mysqlclient
+
+#RUN pip3 install selenium-wire
+CMD ["python3"]
+

geckodriver.log

@@ -0,0 +1,409 @@
+geckodriver: error: Found argument '--websocket-port' which wasn't expected, or isn't valid in this context
+
+USAGE:
+    geckodriver [FLAGS] [OPTIONS]
+
+For more information try --help
+
+1653954618713	geckodriver	INFO	Listening on 127.0.0.1:51155
+1653954621817	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "51156" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilezNiOzl"
+1653954622186	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilezNiOzl\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:51156/devtools/browser/8ca380fe-aee1-499f-a3f3-07bd1ff34387
+1653954623730	Marionette	INFO	Listening on port 51175
+Read port: 51175
+1653954623822	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1653954643083	Marionette	WARN	TimedPromise timed out after 500 ms: stacktrace:
+TimedPromise/<@chrome://remote/content/marionette/sync.js:235:19
+TimedPromise@chrome://remote/content/marionette/sync.js:220:10
+interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10
+webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31
+1653954654634	Marionette	INFO	Stopped listening on port 51175
+1653954721868	geckodriver	INFO	Listening on 127.0.0.1:51252
+1653954724954	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "51253" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileOugi5i"
+1653954725346	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileOugi5i\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:51253/devtools/browser/21883633-5135-4ad8-bb3e-cb6f95e129a9
+1653954726739	Marionette	INFO	Listening on port 51273
+Read port: 51273
+1653954726841	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653954825331	geckodriver	INFO	Listening on 127.0.0.1:51369
+1653954828412	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "51370" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileELBfKY"
+1653954828790	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileELBfKY\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:51370/devtools/browser/14e7320f-1e16-4efb-82e0-054073ca84e6
+1653954830219	Marionette	INFO	Listening on port 51390
+Read port: 51390
+1653954830300	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653955002867	geckodriver	INFO	Listening on 127.0.0.1:51629
+1653955005943	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "51630" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileMqKK4P"
+1653955006317	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileMqKK4P\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:51630/devtools/browser/fcdf7d7f-806f-4e1f-97c4-9aa3fcc4b215
+1653955007840	Marionette	INFO	Listening on port 51654
+Read port: 51654
+1653955007943	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653955045107	Marionette	INFO	Stopped listening on port 51654
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+1653955138657	geckodriver	INFO	Listening on 127.0.0.1:51789
+1653955141760	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "51790" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile5yrkKG"
+1653955142304	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile5yrkKG\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:51790/devtools/browser/5796523b-cddb-43c5-b402-9be4fd5e4a5f
+1653955144102	Marionette	INFO	Listening on port 51811
+Read port: 51811
+1653955144188	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653955317239	geckodriver	INFO	Listening on 127.0.0.1:52040
+1653955320311	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "52041" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileYCUSpS"
+1653955320708	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileYCUSpS\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:52041/devtools/browser/06ea1198-882c-4368-9f81-8a09031b0467
+1653955321860	Marionette	INFO	Listening on port 52060
+Read port: 52060
+1653955321883	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1653955340008	Marionette	WARN	TimedPromise timed out after 500 ms: stacktrace:
+TimedPromise/<@chrome://remote/content/marionette/sync.js:235:19
+TimedPromise@chrome://remote/content/marionette/sync.js:220:10
+interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10
+webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D2%25E5%2588%2586%25E6%259D%25BF%26source%3Dhp%26ei%3D_FqVYq-YON7P2roPnKKLwAY%26iflsig%3DAJiK0e8AAAAAYpVpDHPrcnovBNhRJc8uHAWzFt2WmFMM%26ved%3D0ahUKEwivhvfct4j4AhXep1YBHRzRAmgQ4dUDCAY%26uact%3D5%26oq%3D2%25E5%2588%2586%25E6%259D%25BF%26gs_lcp%3DCgdnd3Mtd2l6EANQAFgAYABoAHAAeACAAQCIAQCSAQCYAQA%26sclient%3Dgws-wiz&q=EgRo9EgHGJK21ZQGIhBuPCKkLVxIFh9q01kW_m34MgFy, line 1: SyntaxError: expected expression, got ','
+1653955358597	Marionette	INFO	Stopped listening on port 52060
+
+###!!! [Parent][PGPUParent] Error: RunMessage(msgname=PGPU::Msg_ShutdownVR) Channel closing: too late to send/recv, messages will be lost
+
+1653955379163	geckodriver	INFO	Listening on 127.0.0.1:52164
+1653955382231	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "52165" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilecSfDVB"
+1653955382604	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilecSfDVB\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:52165/devtools/browser/29ececfd-17b0-4e5e-8b7b-caf94d350db5
+1653955383911	Marionette	INFO	Listening on port 52184
+Read port: 52184
+1653955384011	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653955639405	geckodriver	INFO	Listening on 127.0.0.1:52397
+1653955642472	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "52398" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileluC48a"
+1653955642851	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileluC48a\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:52398/devtools/browser/6c20843c-0bf5-4041-b2d6-5de78e009733
+1653955644186	Marionette	INFO	Listening on port 52418
+Read port: 52418
+1653955644254	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1653955668017	Marionette	INFO	Stopped listening on port 52418
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+console.warn: TopSitesFeed: Failed to fetch data from Contile server: NetworkError when attempting to fetch resource.
+JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
+1653955694584	geckodriver	INFO	Listening on 127.0.0.1:52506
+1653955697665	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "52507" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile2Y92VI"
+1653955698032	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile2Y92VI\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:52507/devtools/browser/71c4bfa9-ab1d-43c9-8c4d-cbe884abd6bc
+1653955699367	Marionette	INFO	Listening on port 52528
+Read port: 52528
+1653955699441	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1653955705276	Marionette	INFO	Stopped listening on port 52528
+JavaScript error: resource://autofill/FormAutofillContent.jsm, line 748: TypeError: can't access dead object
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+console.error: services.settings: 
+  main/query-stripping Signature failed  TypeError: NetworkError: Network request failed
+console.warn: TopSitesFeed: Failed to fetch data from Contile server: NetworkError when attempting to fetch resource.
+JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
+
+###!!! [Parent][PGPUParent] Error: RunMessage(msgname=PGPU::Msg_ShutdownVR) Channel closing: too late to send/recv, messages will be lost
+
+1653955728048	geckodriver	INFO	Listening on 127.0.0.1:52575
+1653955731105	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "52576" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilexjx8NM"
+1653955731468	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilexjx8NM\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:52576/devtools/browser/7082b543-ac1e-4178-9407-95f05c7ed9ac
+1653955732912	Marionette	INFO	Listening on port 52597
+Read port: 52597
+1653955732998	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E7%2599%25BE%25E8%2591%2589%25E7%25AA%2597%26source%3Dhp%26ei%3DmFyVYpPTHMyFr7wP7N2JiA0%26iflsig%3DAJiK0e8AAAAAYpVqqENtMeBfsdxcRCcmYijaNpMVey8X%26ved%3D0ahUKEwiT_5WhuYj4AhXMwosBHexuAtEQ4dUDCAY%26uact%3D5%26oq%3D%25E7%2599%25BE%25E8%2591%2589%25E7%25AA%2597%26gs_lcp%3DCgdnd3Mtd2l6EANQAFgAYABoAHAAeACAAQCIAQCSAQCYAQA%26sclient%3Dgws-wiz&q=EgS589ogGK-51ZQGIhAZrE3b9xdmO8zGi9_6lvzEMgFy, line 1: SyntaxError: expected expression, got ','
+1653955802384	geckodriver	INFO	Listening on 127.0.0.1:52693
+1653955805468	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "52694" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileDl0SRI"
+1653955805829	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileDl0SRI\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:52694/devtools/browser/429129ca-5001-48cb-a00d-36186fefe055
+1653955807184	Marionette	INFO	Listening on port 52717
+Read port: 52717
+1653955807246	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E6%25B2%25B9%25E6%2580%25A7%25E6%25BC%2586%26source%3Dhp%26ei%3D4lyVYqf_HYmKmAW3o53gDg%26iflsig%3DAJiK0e8AAAAAYpVq8sMCx1brI9aPz6XDfgF2A_Xv_Y0U%26ved%3D0ahUKEwin-LvEuYj4AhUJBaYKHbdRB-wQ4dUDCAk%26uact%3D5%26oq%3D%25E6%25B2%25B9%25E6%2580%25A7%25E6%25BC%2586%26gs_lcp%3DCgdnd3Mtd2l6EAM6BQgAEIAEOggILhCABBDUAjoFCC4QgARQAFgNYKYvaABwAHgAgAGQCIgB8BaSAQU2LTEuMpgBAKABAQ%26sclient%3Dgws-wiz&q=EgS53GWPGPy51ZQGIhDgH7OAPyqOucYLlnEb7i4sMgFy, line 1: SyntaxError: expected expression, got ','
+1653955954951	geckodriver	INFO	Listening on 127.0.0.1:52871
+1653955958014	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "52872" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileOzLngL"
+1653955958308	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileOzLngL\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:52872/devtools/browser/4eb62965-24f8-40c8-90e1-afc47a6c2a6f
+1653955959514	Marionette	INFO	Listening on port 52894
+Read port: 52894
+1653955959559	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1653955980968	Marionette	WARN	TimedPromise timed out after 500 ms: stacktrace:
+TimedPromise/<@chrome://remote/content/marionette/sync.js:235:19
+TimedPromise@chrome://remote/content/marionette/sync.js:220:10
+interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10
+webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D10%25E5%259D%25AA%25E5%25A5%2597%25E6%2588%25BF%25E8%25A3%259D%25E6%25BD%25A2%26source%3Dhp%26ei%3De12VYtuwEYqEmAW_9LnACw%26iflsig%3DAJiK0e8AAAAAYpVri8zw1qxQ4kHHNsOif6Zw53gxQTXL%26ved%3D0ahUKEwib2qmNuoj4AhUKAqYKHT96DrgQ4dUDCAY%26uact%3D5%26oq%3D10%25E5%259D%25AA%25E5%25A5%2597%25E6%2588%25BF%25E8%25A3%259D%25E6%25BD%25A2%26gs_lcp%3DCgdnd3Mtd2l6EANQAFgAYABoAHAAeACAAQCIAQCSAQCYAQA%26sclient%3Dgws-wiz&q=EgTBIH-bGJS71ZQGIhD3sqeFrAqCjEDH1Gj0WP7tMgFy, line 1: SyntaxError: expected expression, got ','
+1653955999553	Marionette	INFO	Stopped listening on port 52894
+1653956049353	geckodriver	INFO	Listening on 127.0.0.1:53000
+1653956052449	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "53001" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile1coSoa"
+1653956052832	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile1coSoa\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:53001/devtools/browser/b2be4ab7-6886-4019-a9f1-051c6eed1224
+1653956054328	Marionette	INFO	Listening on port 53022
+Read port: 53022
+1653956054448	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E9%2581%258E%25E7%25A2%25B3%25E9%2585%25B8%25E9%2588%2589%25E9%2599%25A4%25E9%259C%2589%26source%3Dhp%26ei%3D2V2VYs_uH4GkmAXElaXoAg%26iflsig%3DAJiK0e8AAAAAYpVr6Qv6KJNrJzF-1_qnuaw5WAeHhj1V%26ved%3D0ahUKEwiPv6G6uoj4AhUBEqYKHcRKCS0Q4dUDCAY%26uact%3D5%26oq%3D%25E9%2581%258E%25E7%25A2%25B3%25E9%2585%25B8%25E9%2588%2589%25E9%2599%25A4%25E9%259C%2589%26gs_lcp%3DCgdnd3Mtd2l6EANQAFgAYABoAHAAeACAAQCIAQCSAQCYAQA%26sclient%3Dgws-wiz&q=EgQzD-NtGO671ZQGIhCuzmWxJUvdNhp8jIb0kIZZMgFy, line 1: SyntaxError: expected expression, got ','
+1653956090510	Marionette	INFO	Stopped listening on port 53022
+[Parent 4952, IPC I/O Parent] WARNING: file /builds/worker/checkouts/gecko/ipc/chromium/src/base/process_util_win.cc:167
+1653956109939	geckodriver	INFO	Listening on 127.0.0.1:53120
+1653956112998	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "53121" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileExgpD4"
+1653956113405	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileExgpD4\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:53121/devtools/browser/687eb149-ff5b-45fc-ab48-25ee279a15e3
+1653956114606	Marionette	INFO	Listening on port 53140
+Read port: 53140
+1653956114676	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E9%2599%25BD%25E5%258F%25B0%25E6%2594%25B9%25E9%2580%25A0diy%26source%3Dhp%26ei%3DFl6VYpe7EZHt0ASqhojQDQ%26iflsig%3DAJiK0e8AAAAAYpVsJu6X8vKg_XRjSXdosWk-ILOEIl1a%26ved%3D0ahUKEwiXnp7Xuoj4AhWRNpQKHSoDAtoQ4dUDCAY%26uact%3D5%26oq%3D%25E9%2599%25BD%25E5%258F%25B0%25E6%2594%25B9%25E9%2580%25A0diy%26gs_lcp%3DCgdnd3Mtd2l6EAMyBQgAEIAEOgQIABAeOgYIABAeEAVQAFgWYKovaABwAHgAgAGiB4gB8RySAQM2LTSYAQCgAQE%26sclient%3Dgws-wiz&q=EgRn-6cKGKy81ZQGIhAceb9n4s7Pado1EgELQfasMgFy, line 1: SyntaxError: expected expression, got ','
+1653956148523	Marionette	INFO	Stopped listening on port 53140
+console.warn: TopSitesFeed: Failed to fetch data from Contile server: NetworkError when attempting to fetch resource.
+JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
+1653964486010	geckodriver	INFO	Listening on 127.0.0.1:63276
+1653964489089	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "63277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilekmNT2M"
+1653964489548	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilekmNT2M\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:63277/devtools/browser/82e2956b-abcf-4b93-8739-b12cca49b368
+1653964490938	Marionette	INFO	Listening on port 63297
+Read port: 63297
+1653964490970	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E6%25B5%25B4%25E5%25AE%25A4%25E9%25A2%25A8%25E6%25B0%25B4%26source%3Dhp%26ei%3DzX6VYoDZOMTg-AbLvZPYAg%26iflsig%3DAJiK0e8AAAAAYpWM3W-oSX64clmyxxJDc2_3EhQA2VE-%26ved%3D0ahUKEwjA84bx2Yj4AhVEMN4KHcveBCsQ4dUDCAg%26uact%3D5%26oq%3D%25E6%25B5%25B4%25E5%25AE%25A4%25E9%25A2%25A8%25E6%25B0%25B4%26gs_lcp%3DCgdnd3Mtd2l6EANQAFgAYABoAHAAeACAAQCIAQCSAQCYAQA%26sclient%3Dgws-wiz&q=EgS53Gb9GOT91ZQGIhDfagGPSE2bT3Sp0NlxYOaZMgFy, line 1: SyntaxError: expected expression, got ','
+1653964528387	Marionette	INFO	Stopped listening on port 63297
+
+###!!! [Parent][PGPUParent] Error: RunMessage(msgname=PGPU::Msg_ShutdownVR) Channel closing: too late to send/recv, messages will be lost
+
+1653964546492	geckodriver	INFO	Listening on 127.0.0.1:63392
+1653964549570	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "63393" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileAU4LRe"
+1653964549981	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileAU4LRe\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:63393/devtools/browser/148dfd65-a3b3-486c-8199-15f73806d85e
+1653964551116	Marionette	INFO	Listening on port 63411
+Read port: 63411
+1653964551251	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E5%25AE%25A2%25E8%25AE%258A%25E6%25B3%25A8%25E6%2584%258F%26source%3Dhp%26ei%3DCn-VYpauB6m3mAWWkoVY%26iflsig%3DAJiK0e8AAAAAYpWNGmWd37Kgf6OUpEFfYB8cboXpdgXQ%26ved%3D0ahUKEwiW2-CN2oj4AhWpG6YKHRZJAQsQ4dUDCAk%26uact%3D5%26oq%3D%25E5%25AE%25A2%25E8%25AE%258A%25E6%25B3%25A8%25E6%2584%258F%26gs_lcp%3DCgdnd3Mtd2l6EAMyBggAEB4QBToFCAAQgAQ6CwguEIAEEMcBEK8BOgUILhCABDoOCC4QgAQQxwEQrwEQ1AI6BAgAEB5QAFgSYKUvaABwAHgAgAGvCIgB1BySAQc1LTEuMi4xmAEAoAEB%26sclient%3Dgws-wiz&q=EgS53GWnGKP-1ZQGIhB7MiXF2SSvNSNxD8PZjGpCMgFy, line 1: SyntaxError: expected expression, got ','
+1653964591588	Marionette	INFO	Stopped listening on port 63411
+
+###!!! [Parent][PGPUParent] Error: RunMessage(msgname=PGPU::Msg_ShutdownVR) Channel closing: too late to send/recv, messages will be lost
+
+1653964624115	geckodriver	INFO	Listening on 127.0.0.1:63535
+1653964627183	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "63536" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilehiz9Se"
+1653964627593	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilehiz9Se\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:63536/devtools/browser/9909e2bb-b82d-4c6d-a173-d3047738a3d7
+1653964629060	Marionette	INFO	Listening on port 63606
+Read port: 63606
+1653964629184	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1653964649939	Marionette	WARN	TimedPromise timed out after 500 ms: stacktrace:
+TimedPromise/<@chrome://remote/content/marionette/sync.js:235:19
+TimedPromise@chrome://remote/content/marionette/sync.js:220:10
+interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10
+webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E8%25A3%259D%25E6%25BD%25A2%25E5%259C%25B0%25E6%259D%25BF%25E4%25BF%259D%25E8%25AD%25B7%25E5%25B7%25A5%25E7%25A8%258B%25E8%25B2%25BB%25E7%2594%25A8%26source%3Dhp%26ei%3DWH-VYvToEtXFhwOHhZ34BQ%26iflsig%3DAJiK0e8AAAAAYpWNaKyBPA2Z_-kg1tr6FvMn1AAoqiSc%26ved%3D0ahUKEwj09ISz2oj4AhXV4mEKHYdCB18Q4dUDCAg%26uact%3D5%26oq%3D%25E8%25A3%259D%25E6%25BD%25A2%25E5%259C%25B0%25E6%259D%25BF%25E4%25BF%259D%25E8%25AD%25B7%25E5%25B7%25A5%25E7%25A8%258B%25E8%25B2%25BB%25E7%2594%25A8%26gs_lcp%3DCgdnd3Mtd2l6EANQAFgAYABoAHAAeACAAQCIAQCSAQCYAQA%26sclient%3Dgws-wiz&q=EhAqC_TBAAIAAAAAAAAAAAJQGPD-1ZQGIhBTmE-I49Va9Ex0Ro1y8yL7MgFy, line 1: SyntaxError: expected expression, got ','
+1653964668960	Marionette	INFO	Stopped listening on port 63606
+
+###!!! [Parent][PGPUParent] Error: RunMessage(msgname=PGPU::Msg_ShutdownVR) Channel closing: too late to send/recv, messages will be lost
+
+1653965175375	geckodriver	INFO	Listening on 127.0.0.1:64108
+1653965178441	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "64109" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileZYrRSH"
+1653965178772	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileZYrRSH\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:64109/devtools/browser/1e8e4f02-d513-41ae-9877-19f86bfe411b
+1653965180048	Marionette	INFO	Listening on port 64129
+Read port: 64129
+1653965180112	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E5%25A9%259A%25E5%2589%258D%25E5%2585%25B1%25E5%2590%258C%25E6%258C%2581%25E6%259C%2589%26source%3Dhp%26ei%3Df4GVYu_HFtrKmAW-iqKoBQ%26iflsig%3DAJiK0e8AAAAAYpWPj3w1NzrrTTLJgPmzCZcezuoX_m4O%26ved%3D0ahUKEwivg-e53Ij4AhVaJaYKHT6FCFUQ4dUDCAk%26uact%3D5%26oq%3D%25E5%25A9%259A%25E5%2589%258D%25E5%2585%25B1%25E5%2590%258C%25E6%258C%2581%25E6%259C%2589%26gs_lcp%3DCgdnd3Mtd2l6EAMyBQgAEIAEMgUIABCABFAAWBpgsy9oAHAAeAKAAfwIiAHHI5IBAzctNJgBAKABAQ%26sclient%3Dgws-wiz&q=EgS53GUuGJiD1pQGIhBBqwdp3-uYfumtk8alAIMwMgFy, line 1: SyntaxError: expected expression, got ','
+1653965488695	geckodriver	INFO	Listening on 127.0.0.1:64372
+1653965491767	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "64373" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilepvqfVE"
+1653965492143	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilepvqfVE\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:64373/devtools/browser/c0f949d5-b64c-4eb8-a341-913a0be6235b
+1653965493333	Marionette	INFO	Listening on port 64391
+Read port: 64391
+1653965493460	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.error: Region.jsm: "Error fetching region" (new Error("TIMEOUT", "resource://gre/modules/Region.jsm", 772))
+console.error: Region.jsm: "Failed to fetch region" (new Error("TIMEOUT", "resource://gre/modules/Region.jsm", 419))
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653965808392	geckodriver	INFO	Listening on 127.0.0.1:60140
+1653965811464	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "60141" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileKCgClN"
+1653965811879	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileKCgClN\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:60141/devtools/browser/5fff5b95-5857-4c0f-a86d-eea7ee039517
+1653965813309	Marionette	INFO	Listening on port 60159
+Read port: 60159
+1653965813355	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1653965832308	Marionette	WARN	TimedPromise timed out after 500 ms: stacktrace:
+TimedPromise/<@chrome://remote/content/marionette/sync.js:235:19
+TimedPromise@chrome://remote/content/marionette/sync.js:220:10
+interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10
+webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E6%25B7%258B%25E6%25B5%25B4%25E9%2596%2593%25E5%25B0%25BA%25E5%25AF%25B8%26source%3Dhp%26ei%3D-IOVYvjGI4WlhwPm86GQBg%26iflsig%3DAJiK0e8AAAAAYpWSCBX6M84YzfHhVxBI8SOLLBx8m9vn%26ved%3D0ahUKEwj4ot_n3oj4AhWF0mEKHeZ5CGIQ4dUDCAY%26uact%3D5%26oq%3D%25E6%25B7%258B%25E6%25B5%25B4%25E9%2596%2593%25E5%25B0%25BA%25E5%25AF%25B8%26gs_lcp%3DCgdnd3Mtd2l6EANQAFgAYABoAHAAeACAAQCIAQCSAQCYAQA%26sclient%3Dgws-wiz&q=EhAmBWQAADD7D-aL54J97iwxGI-I1pQGIhDqnwJC6HLRXPEWg-44ZjGTMgFy, line 1: SyntaxError: expected expression, got ','
+1653965851428	Marionette	INFO	Stopped listening on port 60159
+
+###!!! [Parent][PGPUParent] Error: RunMessage(msgname=PGPU::Msg_ShutdownVR) Channel closing: too late to send/recv, messages will be lost
+
+1653965870509	geckodriver	INFO	Listening on 127.0.0.1:60265
+1653965873594	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "60266" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilebE1Zgx"
+1653965873966	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilebE1Zgx\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:60266/devtools/browser/badc07ac-dc9b-4664-88fb-b8b703486363
+1653965875218	Marionette	INFO	Listening on port 60288
+Read port: 60288
+1653965875248	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653965913716	Marionette	INFO	Stopped listening on port 60288
+1653966037157	geckodriver	INFO	Listening on 127.0.0.1:60511
+1653966040201	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "60512" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile4XobvE"
+1653966040522	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile4XobvE\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:60512/devtools/browser/1ebc10aa-604f-4ff8-93e2-237df3b66383
+1653966042133	Marionette	INFO	Listening on port 60538
+Read port: 60538
+1653966042175	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653966082812	Marionette	INFO	Stopped listening on port 60538
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+1653966133286	geckodriver	INFO	Listening on 127.0.0.1:60685
+1653966136348	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "60686" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilec3Vm3z"
+1653966136833	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilec3Vm3z\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:60686/devtools/browser/f31d1a3b-f66a-4e65-8d85-3a4ee8946174
+1653966137967	Marionette	INFO	Listening on port 60703
+Read port: 60703
+1653966138028	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1653966156787	Marionette	WARN	TimedPromise timed out after 500 ms: stacktrace:
+TimedPromise/<@chrome://remote/content/marionette/sync.js:235:19
+TimedPromise@chrome://remote/content/marionette/sync.js:220:10
+interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10
+webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E8%25BC%2595%25E9%259A%2594%25E9%2596%2593%26source%3Dhp%26ei%3DPYWVYuF505L4BrCaivAI%26iflsig%3DAJiK0e8AAAAAYpWTTXP1PT4YpP4Z4drAqtxMVsU2-w51%26ved%3D0ahUKEwihjLmC4Ij4AhVTCd4KHTCNAo4Q4dUDCAg%26uact%3D5%26oq%3D%25E8%25BC%2595%25E9%259A%2594%25E9%2596%2593%26gs_lcp%3DCgdnd3Mtd2l6EANQAFgAYABoAHAAeACAAQCIAQCSAQCYAQA%26sclient%3Dgws-wiz&q=EhAqC_TBAAIAAAAAAAAAAAJSGNOK1pQGIhANRllSqqp4zrLxJfDL6JK4MgFy, line 1: SyntaxError: expected expression, got ','
+1653966192950	geckodriver	INFO	Listening on 127.0.0.1:60807
+1653966196052	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "60808" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilehNptjg"
+1653966196369	Marionette	INFO	Marionette enabled
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofilehNptjg\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:60808/devtools/browser/ad9ac0a0-1a13-4e7d-8e66-07d3665c7c16
+1653966197612	Marionette	INFO	Listening on port 60827
+Read port: 60827
+1653966197707	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1653966218256	Marionette	WARN	TimedPromise timed out after 500 ms: stacktrace:
+TimedPromise/<@chrome://remote/content/marionette/sync.js:235:19
+TimedPromise@chrome://remote/content/marionette/sync.js:220:10
+interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10
+webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653966236773	Marionette	INFO	Stopped listening on port 60827
+
+###!!! [Child][PContentChild] Error: RunMessage(msgname=PHttpChannel::Msg_DeleteSelf) Channel closing: too late to send/recv, messages will be lost
+
+
+###!!! [Parent][PGPUParent] Error: RunMessage(msgname=PGPU::Msg_ShutdownVR) Channel closing: too late to send/recv, messages will be lost
+
+1653966486952	geckodriver	INFO	Listening on 127.0.0.1:54282
+1653966490020	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--headless" "--remote-debugging-port" "54283" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileWC0kbD"
+*** You are running in headless mode.
+1653966490480	Marionette	INFO	Marionette enabled
+[GFX1-]: RenderCompositorSWGL failed mapping default framebuffer, no dt
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileWC0kbD\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:54283/devtools/browser/5ee3113b-e5d6-4dfc-9f67-b73fb6f2d96d
+1653966491374	Marionette	INFO	Listening on port 54309
+Read port: 54309
+1653966491466	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D%25E6%2588%25BF%25E5%25AD%2590%25E6%25BC%258F%25E6%25B0%25B4%26source%3Dhp%26ei%3DnoaVYoTkKLeHr7wPk7a7qAE%26iflsig%3DAJiK0e8AAAAAYpWUrgxqnz4K9dtduqQNDaJ9FTCXj8Cl%26ved%3D0ahUKEwiEq4qr4Yj4AhW3w4sBHRPbDhUQ4dUDCAk%26uact%3D5%26oq%3D%25E6%2588%25BF%25E5%25AD%2590%25E6%25BC%258F%25E6%25B0%25B4%26gs_lcp%3DCgdnd3Mtd2l6EAMyBQgAEIAEMgUIABCABDIFCAAQgAQyBQgAEIAEMgUIABCABDIFCAAQgAQyBQgAEIAEMgUIABCABDIFCAAQgAQyBQgAEIAEOgUILhCABDoLCC4QgAQQxwEQ0QNQAFgRYKsvaABwAHgBgAGHCIgB5h-SAQM3LTSYAQCgAQE%26sclient%3Dgws-wiz&q=EhAmIAAHYAEAAAAA___HWeZKGLeN1pQGIhAxG__o4e-R53nKk5smHQoXMgFy, line 1: SyntaxError: expected expression, got ','
+1653966532103	Marionette	INFO	Stopped listening on port 54309
+
+###!!! [Parent][PImageBridgeParent] Error: RunMessage(msgname=PImageBridge::Msg_WillClose) Channel closing: too late to send/recv, messages will be lost
+
+1653966617353	geckodriver	INFO	Listening on 127.0.0.1:54536
+1653966620405	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--headless" "--remote-debugging-port" "54537" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile3bOuHg"
+*** You are running in headless mode.
+1653966620796	Marionette	INFO	Marionette enabled
+[GFX1-]: RenderCompositorSWGL failed mapping default framebuffer, no dt
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofile3bOuHg\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:54537/devtools/browser/8406f2c8-bcd0-4569-a57f-3b9fabea0743
+1653966621683	Marionette	INFO	Listening on port 54556
+Read port: 54556
+1653966621789	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.error: Region.jsm: "Error fetching region" (new Error("TIMEOUT", "resource://gre/modules/Region.jsm", 772))
+console.error: Region.jsm: "Failed to fetch region" (new Error("TIMEOUT", "resource://gre/modules/Region.jsm", 419))
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+JavaScript error: https://www.google.com/sorry/index?continue=https://www.google.com/search%3Fq%3D15%25E5%259D%25AA%25E8%25A3%259D%25E6%25BD%25A2%26source%3Dhp%26ei%3DIYeVYpryKsTR2roPub-X4AE%26iflsig%3DAJiK0e8AAAAAYpWVMa2FXw_Mrw9US6znM4unQsHw_1sO%26ved%3D0ahUKEwjahsjp4Yj4AhXEqFYBHbnfBRwQ4dUDCAY%26uact%3D5%26oq%3D15%25E5%259D%25AA%25E8%25A3%259D%25E6%25BD%25A2%26gs_lcp%3DCgdnd3Mtd2l6EANQAFgAYABoAHAAeACAAQCIAQCSAQCYAQA%26sclient%3Dgws-wiz&q=EgRb2-zFGLeO1pQGIhA_bBoUacRMbmJJTCEh1PZOMgFy, line 1: SyntaxError: expected expression, got ','
+1653966659874	Marionette	INFO	Stopped listening on port 54556
+[Parent 13868, IPC I/O Parent] WARNING: file /builds/worker/checkouts/gecko/ipc/chromium/src/base/process_util_win.cc:167
+[Parent 13868, IPC I/O Parent] WARNING: file /builds/worker/checkouts/gecko/ipc/chromium/src/base/process_util_win.cc:167
+1653966993555	geckodriver	INFO	Listening on 127.0.0.1:64814
+1653966996663	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--headless" "--remote-debugging-port" "64815" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileBds4MF"
+*** You are running in headless mode.
+1653966997097	Marionette	INFO	Marionette enabled
+[GFX1-]: RenderCompositorSWGL failed mapping default framebuffer, no dt
+console.warn: SearchSettings: "get: No settings file exists, new profile?" (new NotFoundError("Could not open the file at C:\\Users\\jared\\AppData\\Local\\Temp\\rust_mozprofileBds4MF\\search.json.mozlz4", (void 0)))
+DevTools listening on ws://localhost:64815/devtools/browser/420bd998-b6b2-4159-bac1-83f8162cbd36
+1653966997954	Marionette	INFO	Listening on port 64833
+Read port: 64833
+1653966998033	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+console.warn: LoginRecipes: "getRecipes: falling back to a synchronous message for:" "https://www.google.com"
+1653967050870	Marionette	INFO	Stopped listening on port 64833
+
+###!!! [Parent][PImageBridgeParent] Error: RunMessage(msgname=PImageBridge::Msg_WillClose) Channel closing: too late to send/recv, messages will be lost
+

google-ads/ads-selenium.py

@@ -0,0 +1,68 @@
+from seleniumwire import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import gzip
+import brotli
+
+def restart_browser():
+#    os.system('docker container restart headless-shell')
+#   time.sleep(9)
+    chrome_options = webdriver.ChromeOptions()
+#    chrome_options.add_argument("--incognito")
+#    chrome_options.add_argument("--headless")
+    chrome_options.add_argument("--no-sandbox")
+    chrome_options.add_argument("--disable-dev-shm-usage")
+    chrome_options.add_argument("start-maximized")
+    chrome_options.add_argument("user-data-dir=C:\\Users\\jared\\AppData\\Local\\Google\\Chrome\\User Data")
+    chrome_options.add_argument("--user-profile=Profile 7")
+
+#    chrome_options.debugger_address="127.0.0.1:9222"
+#    chrome98=r'C:\portable\webdriver\chrome98\chromedriver.exe'
+#    chrome98=r'/root/drivers/98/chromedriver'
+
+#    driver = webdriver.Chrome(chrome_options=chrome_options,executable_path=chrome98)
+    driver = webdriver.Chrome(chrome_options=chrome_options)
+ 
+
+    return driver
+
+
+driver=restart_browser()
+#driver.get('https://ads.google.com/aw/campaigns?ocid=891665672&workspaceId=0&euid=459838964&__u=6055300436&uscid=891665672&__c=9718115528&authuser=1')
+driver.get('https://ads.google.com/aw/campaigns?ocid=400572474&euid=459838964&__u=6055300436&uscid=400572474&__c=7078600426&authuser=1')
+time.sleep(21)
+for request in driver.requests:
+    if request.response:
+            if 'Get?' in request.url :
+                if 'Overview' in  request.url :
+                    resp=request.response.body
+                    if 'gzip' in request.response.headers.get('Content-Encoding'):
+                        resp = gzip.decompress(request.response.body)
+
+                    if 'br' in request.response.headers.get('Content-Encoding'):
+                        resp = brotli.decompress(request.response.body)
+
+                    jstext = resp.decode('utf-8')
+                    print(jstext)
+                    fw=codecs.open('c:/tmp/json.txt','w','utf-8')
+                    fw.write(jstext)
+                    fw.close()
+#                    output = parsing_js(jstext)
+#                    time.sleep(1)
+
+
+time.sleep(9999)

google-ads/gen_user_cred.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""This example will create an OAuth2 refresh token for the Google Ads API.
+
+This example works with both web and desktop app OAuth client ID types.
+
+https://console.cloud.google.com
+
+IMPORTANT: For web app clients types, you must add "http://127.0.0.1" to the
+"Authorized redirect URIs" list in your Google Cloud Console project before
+running this example. Desktop app client types do not require the local
+redirect to be explicitly configured in the console.
+
+Once complete, download the credentials and save the file path so it can be
+passed into this example.
+
+This example is a very simple implementation, for a more detailed example see:
+https://developers.google.com/identity/protocols/oauth2/web-server#python
+"""
+
+import argparse
+import hashlib
+import os
+import re
+import socket
+import sys
+from urllib.parse import unquote
+
+# If using Web flow, the redirect URL must match exactly what’s configured in GCP for
+# the OAuth client.  If using Desktop flow, the redirect must be a localhost URL and
+# is not explicitly set in GCP.
+from google_auth_oauthlib.flow import Flow
+
+_SCOPE = "https://www.googleapis.com/auth/adwords"
+_SERVER = "127.0.0.1"
+_PORT = 8080
+_REDIRECT_URI = f"http://{_SERVER}:{_PORT}"
+
+
+def main(client_secrets_path, scopes):
+  """The main method, starts a basic server and initializes an auth request.
+
+    Args:
+        client_secrets_path: a path to where the client secrets JSON file is
+          located on the machine running this example.
+        scopes: a list of API scopes to include in the auth request, see:
+            https://developers.google.com/identity/protocols/oauth2/scopes
+  """
+  flow = Flow.from_client_secrets_file(client_secrets_path, scopes=scopes)
+  flow.redirect_uri = _REDIRECT_URI
+
+  # Create an anti-forgery state token as described here:
+  # https://developers.google.com/identity/protocols/OpenIDConnect#createxsrftoken
+  passthrough_val = hashlib.sha256(os.urandom(1024)).hexdigest()
+
+  authorization_url, state = flow.authorization_url(
+      access_type="offline",
+      state=passthrough_val,
+      prompt="consent",
+      include_granted_scopes="true",
+  )
+
+  # Prints the authorization URL so you can paste into your browser. In a
+  # typical web application you would redirect the user to this URL, and they
+  # would be redirected back to "redirect_url" provided earlier after
+  # granting permission.
+  print("Paste this URL into your browser: ")
+  print(authorization_url)
+  print(f"\nWaiting for authorization and callback to: {_REDIRECT_URI}")
+
+  # Retrieves an authorization code by opening a socket to receive the
+  # redirect request and parsing the query parameters set in the URL.
+  code = unquote(_get_authorization_code(passthrough_val))
+
+  # Pass the code back into the OAuth module to get a refresh token.
+  flow.fetch_token(code=code)
+  refresh_token = flow.credentials.refresh_token
+
+  print(f"\nYour refresh token is: {refresh_token}\n")
+  print(
+      "Add your refresh token to your client library configuration as "
+      "described here: "
+      "https://developers.google.com/google-ads/api/docs/client-libs/python/configuration"
+  )
+
+
+def _get_authorization_code(passthrough_val):
+  """Opens a socket to handle a single HTTP request containing auth tokens.
+
+    Args:
+        passthrough_val: an anti-forgery token used to verify the request
+          received by the socket.
+
+    Returns:
+        a str access token from the Google Auth service.
+    """
+  # Open a socket at _SERVER:_PORT and listen for a request
+  sock = socket.socket()
+  sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+  sock.bind((_SERVER, _PORT))
+  sock.listen(1)
+  connection, address = sock.accept()
+  data = connection.recv(1024)
+  # Parse the raw request to retrieve the URL query parameters.
+  params = _parse_raw_query_params(data)
+
+  try:
+    if not params.get("code"):
+      # If no code is present in the query params then there will be an
+      # error message with more details.
+      error = params.get("error")
+      message = f"Failed to retrieve authorization code. Error: {error}"
+      raise ValueError(message)
+    elif params.get("state") != passthrough_val:
+      message = "State token does not match the expected state."
+      raise ValueError(message)
+    else:
+      message = "Authorization code was successfully retrieved."
+  except ValueError as error:
+    print(error)
+    sys.exit(1)
+  finally:
+    response = ("HTTP/1.1 200 OK\n"
+                "Content-Type: text/html\n\n"
+                f"<b>{message}</b>"
+                "<p>Please check the console output.</p>\n")
+
+    connection.sendall(response.encode())
+    connection.close()
+
+  return params.get("code")
+
+
+def _parse_raw_query_params(data):
+  """Parses a raw HTTP request to extract its query params as a dict.
+
+    Note that this logic is likely irrelevant if you're building OAuth logic
+    into a complete web application, where response parsing is handled by a
+    framework.
+
+    Args:
+        data: raw request data as bytes.
+
+    Returns:
+        a dict of query parameter key value pairs.
+    """
+  # Decode the request into a utf-8 encoded string
+  decoded = data.decode("utf-8")
+  # Use a regular expression to extract the URL query parameters string
+  match = re.search("GET\s\/\?(.*) ", decoded)
+  params = match.group(1)
+  # Split the parameters to isolate the key/value pairs
+  pairs = [pair.split("=") for pair in params.split("&")]
+  # Convert pairs to a dict to make it easy to access the values
+  return {key: val for key, val in pairs}
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(
+      description=(
+          "Generates OAuth2 refresh token using the Web application flow. "
+          "To retrieve the necessary client_secrets JSON file, first "
+          "generate OAuth 2.0 credentials of type Web application in the "
+          "Google Cloud Console (https://console.cloud.google.com). "
+          "Make sure 'http://_SERVER:_PORT' is included the list of "
+          "'Authorized redirect URIs' for this client ID."),)
+  # The following argument(s) should be provided to run the example.
+  parser.add_argument(
+      "-c",
+      "--client_secrets_path",
+      required=True,
+      type=str,
+      help=("Path to the client secrets JSON file from the Google Developers "
+            "Console that contains your client ID, client secret, and "
+            "redirect URIs."),
+  )
+  parser.add_argument(
+      "--additional_scopes",
+      default=None,
+      type=str,
+      nargs="+",
+      help="Additional scopes to apply when generating the refresh token.",
+  )
+  args = parser.parse_args()
+
+  configured_scopes = [_SCOPE]
+
+  if args.additional_scopes:
+    configured_scopes.extend(args.additional_scopes)
+
+  main(args.client_secrets_path, configured_scopes)

google-ads/get_campaign.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""This example illustrates how to get all campaigns.
+
+To add campaigns, run add_campaigns.py.
+"""
+
+
+import argparse
+import sys
+
+from google.ads.googleads.client import GoogleAdsClient
+from google.ads.googleads.errors import GoogleAdsException
+
+
+def main(client, customer_id):
+    ga_service = client.get_service("GoogleAdsService")
+
+    query = """
+        SELECT
+          campaign.id,
+          campaign.name
+        FROM campaign
+        ORDER BY campaign.id"""
+
+    # Issues a search request using streaming.
+    stream = ga_service.search_stream(customer_id=customer_id, query=query)
+
+    for batch in stream:
+        for row in batch.results:
+            print(
+                f"Campaign with ID {row.campaign.id} and name "
+                f'"{row.campaign.name}" was found.'
+            )
+
+
+if __name__ == "__main__":
+    # GoogleAdsClient will read the google-ads.yaml configuration file in the
+    # home directory if none is specified.
+#    googleads_client = GoogleAdsClient.load_from_storage(version="v10")
+    googleads_client = GoogleAdsClient.load_from_storage(r"c:\gitlab\kw_tools\kw_tools\google-ads\google-ads.yaml",version="v10")
+
+    parser = argparse.ArgumentParser(
+        description="Lists all campaigns for specified customer."
+    )
+    # The following argument(s) should be provided to run the example.
+    parser.add_argument(
+        "-c",
+        "--customer_id",
+        type=str,
+        required=True,
+        help="The Google Ads customer ID.",
+    )
+    args = parser.parse_args()
+
+    try:
+        main(googleads_client, args.customer_id)
+    except GoogleAdsException as ex:
+        print(
+            f'Request with ID "{ex.request_id}" failed with status '
+            f'"{ex.error.code().name}" and includes the following errors:'
+        )
+        for error in ex.failure.errors:
+            print(f'\tError with message "{error.message}".')
+            if error.location:
+                for field_path_element in error.location.field_path_elements:
+                    print(f"\t\tOn field: {field_path_element.field_name}")
+        sys.exit(1)
+

google-ads/google-ads.yaml

@@ -0,0 +1,83 @@
+# Developer token
+##########################################################################################
+# A developer token is required when making requests to the Google Ads API regardless of #
+# whether you're using the OAuth2 or Service Account configurations. To obtain a         #
+# developer token see:                                                                   #
+# https://developers.google.com/google-ads/api/docs/first-call/dev-token                 #
+##########################################################################################
+developer_token: eoizFOU3k7uAtK2S4IJJ6A
+
+# Use proto plus
+##########################################################################################
+# This parameter specifies whether the client library should return proto-plus messages  #
+# or protobuf messages. This value should be explicitly set to either "True" or "False", #
+# For more information on the differences between these two types, see our Protobuf      #
+# Messages guide:                                                                        #
+# https://developers.google.com/google-ads/api/docs/client-libs/python/protobuf-messages #
+##########################################################################################
+use_proto_plus: True
+
+# OAuth2 configuration
+##########################################################################################
+# The below configuration parameters are used to authenticate using the recommended      #
+# OAuth2 flow. For more information on authenticating with OAuth2 see:                   #
+# https://developers.google.com/google-ads/api/docs/oauth/overview                       #
+##########################################################################################
+client_id: 996866162565-akepc8cqlcd1o7scrh18n9hoa1pao21g.apps.googleusercontent.com
+client_secret: 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
+refresh_token: 1//0eOZ5yyVkxH-VCgYIARAAGA4SNwF-L9IrvHJLv2Dyu3cpB1KQh8o7vQ_Yp4Y0i0zjjgeTfrCW6osOLRpPON3EmIaTb7eGsHJJGEU
+
+
+
+# Service Account configuration
+##########################################################################################
+# To authenticate with a service account add the appropriate values to the below         #
+# configuration parameters and remove the three OAuth2 credentials above. The            #
+# "json_key_file_path" value should be a path to your local private key json file, and   #
+# "impersonated_email" should be the email address that is being used to impersonate the #
+# credentials making requests. for more information on service accounts, see:            #
+# https://developers.google.com/google-ads/api/docs/oauth/service-accounts               #
+##########################################################################################
+# json_key_file_path: INSERT_PATH_TO_JSON_KEY_FILE_HERE
+# impersonated_email: INSERT_DOMAIN_WIDE_DELEGATION_ACCOUNT
+
+# Login customer ID configuration
+##########################################################################################
+# Required for manager accounts only: Specify the login customer ID used to authenticate #
+# API calls. This will be the customer ID of the authenticated manager account. It       #
+# should be set without dashes, for example: 1234567890 instead of 123-456-7890. You can #
+# also specify this later in code if your application uses multiple manager account +    #
+# OAuth pairs.                                                                           #
+##########################################################################################
+login_customer_id: 6569683490
+
+# Logging configuration
+##########################################################################################
+# Below you may specify the logging configuration. This will be provided as an input to  #
+# logging.config.dictConfig. Use the "level" block under the root logger configuration   #
+# to adjust the logging level. Note in the "format" field that log messages are          #
+# truncated to 5000 characters by default. You can change this to any length by removing #
+# the ".5000" portion or changing it to a different number.                              #
+# ########################################################################################
+# logging:
+  # version: 1
+  # disable_existing_loggers: False
+  # formatters:
+    # default_fmt:
+      # format: '[%(asctime)s - %(levelname)s] %(message).5000s'
+      # datefmt: '%Y-%m-%d %H:%M:%S'
+  # handlers:
+    # default_handler:
+      # class: logging.StreamHandler
+      # formatter: default_fmt
+  # loggers:
+    # "":
+      # handlers: [default_handler]
+      # level: INFO
+
+# Proxy configuration
+##########################################################################################
+# Below you can specify an optional proxy configuration to be used by requests. If you   #
+# don't have username and password, just specify host and port.                          #
+# ########################################################################################
+# http_proxy: http://user:password@localhost:8000

hhh/Browser_ads_kw.py

@@ -29,6 +29,7 @@ def get_designer_statistics():
 #    googleurl='https://ads.google.com/aw/overview?ocid=600024232&euid=459838964&__u=6055300436&uscid=600024232&__c=2195332968&authuser=0&subid=ALL-zh-TW-et-g-aw-c-home-awhp_xin1_signin!o2'
 #    googleurl='https://ads.google.com/aw/keywordplanner/home?ocid=600024232&euid=459838964&__u=6055300436&uscid=600024232&__c=2195332968&authuser=0&subid=ALL-zh-TW-et-g-aw-c-home-awhp_xin1_signin%21o2'
     googleurl='https://ads.google.com/aw/keywordplanner/home?ocid=600024232&euid=459838964&__u=6055300436&uscid=600024232&__c=2195332968&authuser=0&subid=ALL-zh-TW-et-g-aw-c-home-awhp_xin1_signin%21o2'
+#    googleurl='https://ads.google.com/aw/overview?ocid=732105824&euid=459838964&__u=6055300436&uscid=732105824&__c=5922164576&authuser=0'
     jb.get(googleurl)
     driver=jb.get_driver()
     driver.refresh()

hhh/GAP_DB_TO_SHEET.py

@@ -32,7 +32,7 @@ def save_sheet(df,filename,tabname,startpos='A1'):
 
 def do_jobs():
     db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
-    cursor=db.query('SELECT id,title,href,kw FROM hhh.gap20;')
+    cursor=db.query('SELECT id,title,href,kw FROM hhh.gap20v7;')
     df = pd.DataFrame(columns=('id','title','href','kw'))
 
     idx=0
@@ -41,7 +41,7 @@ def do_jobs():
         df.loc[idx]=[c['id'],c['title'],c['href'],c['kw']]
     #    df.loc[idx]=['okok',333]
         idx+=1
-    save_sheet(df,'Content_Gap_URLs','20210723')
+    save_sheet(df,'Content_Gap_URLs','2021_DEC_P1')
 
 t = threading.Thread(target = do_jobs)
 t.start()

hhh/SEO/dbtest.py

@@ -0,0 +1,14 @@
+import dataset
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+#    cursor=db.query("SELECT q,cnt FROM hhh.gap_searchome order by cnt desc")
+#    cursor=db.query("SELECT q,cnt FROM hhh.gap_hhh order by cnt desc")
+#    cursor=db.query("SELECT q,cnt FROM hhh.gap_searchome where q not in (select q from hhh.gap_hhh) order by cnt desc;")
+#    cursor=db.query("SELECT q,cnt FROM hhh.gap_d100 where q not in (select q from hhh.gap_hhh) order by cnt desc;")
+#cursor=db.query('SELECT * FROM seo.term_progress limit 10')
+#for c in cursor:
+#    print(c['title'])
+
+table=db['test']
+table.insert({'id':1234,'name':'test name','title':'中文測市'})
+

hhh/SEO/exp_id.py

@@ -0,0 +1,16 @@
+import dataset
+
+import codecs
+import dataset
+import json
+
+db = dataset.connect('mysql://hhh7796hhh:lYmWsu^ujcA1@127.0.0.1:13317/xoops?charset=utf8mb4')
+db2 = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db2['columnids']
+
+cursor=db.query('SELECT * FROM xoops._hcolumn where datediff(now(),creat_time) < 300')
+
+
+for c in cursor:
+    table.insert({'cid':c['hcolumn_id']})
+

hhh/SEO/phantom_18888.py

@@ -0,0 +1,142 @@
+#import redis
+import time
+import traceback
+#import json
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+
+import codecs
+import random
+import requests
+import time
+import sys
+import docker
+import codecs
+import random
+import os
+import time
+
+
+driver=None
+headers = {
+        "Authorization": "Bearer " + "t35vhZtWNgvDNWHc3DJh0OKll3mcB9GvC8K2EAkBug2",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+
+
+def send_msg(kw):
+    params = {"message": "處理關鍵字: "+kw}  
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+
+def process_query(qs):
+    q=qs[0]
+    domain=qs[2]
+    cnt=qs[1]
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+    if cnt > 0:
+        for i in range(cnt):
+            elmt=driver.find_element_by_xpath("//a[@id='pnnext']")
+            webdriver.ActionChains(driver).move_to_element(elmt).perform()
+            webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+    time.sleep(3)
+
+
+
+    elmts=driver.find_elements_by_xpath("//div[@class='g']//div[@class='yuRUbf']//a")
+    idx=1
+    ranking=-1
+    for elmt in elmts:
+        href=elmt.get_attribute('href')
+        txt=elmt.text
+        if len(txt)>10:
+#            if 'hhh.com.tw' in href:
+#            if 'hhh.com.tw' in href:
+#            if 'ai.choozmo.com' in href:
+            if domain in href:
+#            if 'searchome.net' in href:
+                webdriver.ActionChains(driver).move_to_element(elmt).perform()
+                webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+                break
+
+
+def re_get_webdriver():
+    global driver
+    result=[]
+#    client = docker.from_env()
+#    ls=client.containers.list()
+#    print(ls)
+#    for l in ls:
+#        if 'p17777' in l.name:
+#            ls[0].restart()
+
+#    time.sleep(4)
+    if driver is not None:
+        print('closing....')
+        driver.quit()
+#    options = webdriver.EdgeOptions()
+    try:
+        os.system('docker run -it corbinu/alpine-w3m '+url)
+        return
+    except:
+        import traceback
+        traceback.print_exc()
+        driver=None
+        return None
+    driver=None
+
+def run_once(url):
+    try:
+        os.system('docker run -it corbinu/alpine-w3m '+url)
+    except:
+        print('exception')
+
+
+
+fpath=__file__
+elmts=fpath.split('\\')
+fpath=fpath.replace(elmts[-1],'urls.csv')
+
+os.system('docker restart p17777')
+
+
+
+
+print(fpath)
+
+lst=[]
+
+fr=codecs.open(fpath,'r','utf-8')
+lines=fr.readlines()
+for l in lines:
+    elmts=l.split(',')
+    lst.append('https://www.hhh.com.tw'+elmts[0])
+fr.close()
+
+#lst=['https://www.hhh.com.tw/columns/detail/3427/index.php']
+    #for i in range(20):
+while True:
+    l=random.choice(lst)
+    print(l)
+#    driver.get(l)
+    run_once(l)
+#    time.sleep(2)
+
+

hhh/hhh_fetch_gap20.py

@@ -19,7 +19,7 @@ import dataset
 
 db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
 
-table=db['gap20v6']
+table=db['gap20v7']
 def find_master_by_designer(masters,designer):
     for m in masters:
         if m['designer']==designer:
@@ -93,6 +93,21 @@ def get_designer_statistics(kw):
 
 
 qlist=[]
+
+
+#get_designer_statistics("化妝 台 設計 site:hhh.com.tw")
+#get_designer_statistics("油漆 顏色 site:hhh.com.tw")
+#get_designer_statistics("牆壁 裝飾 site:hhh.com.tw")
+#get_designer_statistics("主 臥室 設計 site:hhh.com.tw")
+#get_designer_statistics("套房 裝潢 site:hhh.com.tw")
+#get_designer_statistics("商業 空間 設計 site:hhh.com.tw")
+#get_designer_statistics("台南 室內 設計 site:hhh.com.tw")
+#get_designer_statistics("電視 牆 設計 作品 site:hhh.com.tw")
+#get_designer_statistics("臥室 裝潢 site:hhh.com.tw")
+get_designer_statistics("衛浴 設計 site:hhh.com.tw")
+
+
+
 #get_designer_statistics("北歐 風 site:hhh.com.tw")
 #get_designer_statistics("設計 site:hhh.com.tw")
 #get_designer_statistics("工業 風 site:hhh.com.tw")
@@ -217,7 +232,7 @@ qlist=[]
 #get_designer_statistics("工業 風 辦公室 site:hhh.com.tw")
 
 #get_designer_statistics("小坪 數 浴室 乾 濕 分離 site:hhh.com.tw")
-get_designer_statistics("電視 牆 收納 櫃 site:hhh.com.tw")
+#get_designer_statistics("電視 牆 收納 櫃 site:hhh.com.tw")
 
 
 

hhh/hhh_showgraph.py

@@ -0,0 +1,178 @@
+from instaloader import Instaloader, Profile
+import traceback
+import copy
+import operator
+import dataset
+import pandas as pd
+import networkx as nx
+#import pysftp
+import codecs
+import pyvis
+import sys
+import pickle
+import os
+import searchconsole
+from pyvis.network import Network
+import jieba
+pname='hhh.rb'
+
+db = dataset.connect('sqlite:///'+pname+".db")
+table=db['tmp']
+#pname='cont'
+#pname='damanwoo'
+#pname='drama'
+#pname='news'
+
+#pname='www'
+
+#pname='ipromise'
+#pname='sports'
+#pname='rumor'
+#pname='korea'
+pname='hhh.rb'
+
+rid=0
+
+def get_css():
+    fr=codecs.open('jared/data/css.txt','r','utf-8')
+    lines=fr.readlines()
+    content=' '.join(lines)
+    fr.close()
+    return content
+
+def modify_file(fname):
+    fr=codecs.open(fname,'r','utf-8')
+    lines=fr.readlines()
+    fr.close()
+#    css=get_css()
+    css=''
+    content_output=''
+    for l in lines:
+        if '<body>' in l[0:10]:
+            content_output+=l
+            content_output+='\n<div id="google">\n'
+            continue
+        if '<style type="text' in l[0:22]:
+            content_output+=l
+            content_output+="\n"+css+"\n"
+            continue
+        if '<div id = "mynetwork"' in l[0:30]:
+            content_output+=l
+            content_output+='\n</div>\n'
+            continue
+
+        content_output+=l
+
+    fw=codecs.open("mod_"+fname,'w','utf-8')
+    fw.write(content_output)
+    fw.close()
+
+
+def gen_pic():
+    G=None
+#    if os.path.exists(pname):
+#        G = pickle.load( open( pname, "rb" ) )
+#    else:
+#        G = nx.Graph()
+    G = nx.Graph()
+
+    finallist=[]
+
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having ) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q   ) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q   having count(url) >20) as tbl1 ) order by q')
+#    cursor=db.query('select distinct q,url from tmp where url in (select distinct url from  (select url,count(q) from tmp where length(q)> 2 group by url   having count(q) >5 and count(q)<10 ) as tbl1) order by url')
+    cursor=db.query('select distinct q,url from tmp where url in (select distinct url from  (select url,count(q) from tmp where length(q)> 2 group by url   having count(q) >4 and count(q)<9 ) as tbl1) order by url')
+
+    urldict={}
+    for c in cursor:
+        url=c['url'].replace('https://www.hhh.com.tw','')
+        url=url.replace('https://hhh.com.tw','')
+        url=url.replace('https://m.hhh.com.tw','')
+
+        q=c['q']
+        if urldict.get(url) is None:
+            urldict[url]=[q]
+        else:
+            urldict[url].append(q)
+    print(len(urldict.keys()))
+    cnt=0
+    for k,v in urldict.items():
+        if len(v)>=2:
+            for itm in v:
+                G.add_edge(k,itm,weight=3,width=3,borderwidth=3)
+                cnt+=1
+                if cnt%1000 == 0:
+                    print(cnt)
+
+#            G.add_edge(c['q'],c['rid'],weight=3,width=3,borderwidth=3)
+
+#    pickle.dump( G, open( pname, "wb" ) )
+
+#    G2 = [G.subgraph(c).copy() for c in nx.connected_components(G)]
+#    remove = [node for node,degree in dict(G.degree()).items() if degree <2]
+#    G.remove_nodes_from(remove)
+
+    remove=[]
+#    for n in G.nodes:
+#        if '承諾' in n:
+#            remove.append(n)
+#        if 'promise' in n:
+#            remove.append(n)
+#    G.remove_nodes_from(remove)
+
+    to_remove=[]
+    for n in G:
+        dg=G.degree(n)
+        if dg > 15:
+            to_remove.append(n)
+    G.remove_nodes_from(to_remove)
+
+    G.remove_edges_from(nx.selfloop_edges(G))
+    G.remove_nodes_from(list(nx.isolates(G)))
+#    lst= [G.subgraph(c).copy() for c in nx.connected_components(G)]
+#    lst=[]
+#    for c in nx.connected_components(G):
+#        cc=G.subgraph(c).copy()
+#        if cc.number_of_nodes()>7:
+#            lst.append(cc)
+
+#        if nx.diameter(cc, e=None, usebounds=False)>1:
+#            lst.append(cc)
+
+#    G2=nx.compose_all(lst)
+    G2=G
+#    pyG = Network(height="750px", width="100%",bgcolor="#333333",font_color="white")
+    pyG = Network(height="600px", width="100%",bgcolor="#444444",font_color="white")
+
+
+
+
+    pyG.from_nx(G2)
+    pyG.show(pname+'.html')
+    modify_file(pname+'.html')
+
+#    cnopts = pysftp.CnOpts()
+#    cnopts.hostkeys = None
+#    s = pysftp.Connection(host='www.choozmo.com', username='jared', password='sstc5202',cnopts=cnopts)
+#    local_path = "mod_"+pname+".html"
+#    remote_path = "/home/nginx/farmoutput/tags/"+"mod_"+pname+".html"
+#    s.put(local_path, remote_path)
+
+
+    return finallist
+
+gen_pic()
+#r=checkig('信用卡')
+#print(r)
+
+#        network.on( 'click', function(properties) {
+#    var ids = properties.nodes;
+#    var clickedNodes = nodes.get(ids);
+# var copyText = clickedNodes[0].label;
+# var promise = navigator.clipboard.writeText(copyText);
+#//    console.log('clicked nodes:', clickedNodes);
+#});
+

hhh/hhh_tree.py

@@ -25,8 +25,10 @@ table=db['tmp']
 
 #pname='ipromise'
 #pname='sports'
-pname='rumor'
+#pname='rumor'
 #pname='korea'
+pname='hhh.rb'
+
 rid=0
 
 def get_css():
@@ -85,12 +87,15 @@ def checkig(pgnum):
 #    webproperty = account['https://ipromise.com.tw/']
 #    webproperty = account['https://'+pname+'.face8ook.org/']
 #    webproperty = account['https://www.damanwoo.com/']
-    webproperty = account['https://hhh.com.tw/']
+#    webproperty = account['https://hhh.com.tw/']
+    webproperty = account['https://innews.com.tw/']
 
 #    report=webproperty.query.range('2021-03-01', '2021-06-17').dimension('page','query').get()
 #    report=webproperty.query.range('2021-06-01', '2021-06-17').dimension('page','query').get()
 #    report=webproperty.query.range('2020-06-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/(491|31|293|278|31|24|594|356|307|491|33|385)', 'equals').get()
-    report=webproperty.query.range('2020-03-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/'+pgnum, 'contains').get()
+#    report=webproperty.query.range('2020-03-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/'+pgnum, 'contains').get()
+#    report=webproperty.query.range('2021-12-01', '2022-01-18').dimension('page','query').get()
+    report=webproperty.query.range('2022-01-01', '2022-04-16').dimension('page','query').get()
 
 
     urlq={}
@@ -100,7 +105,7 @@ def checkig(pgnum):
         else:
             urlq[r[0]].append(r[1])
 
-
+    print(urlq)
     allrows=[]
     for k,v in urlq.items():
 #        if len(v)<40:
@@ -135,7 +140,8 @@ def gen_pic():
 #    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 ) order by q')
 #    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 ) order by q')
 #    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having ) as tbl1 ) order by q')
-    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 1 group by q  ) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q   ) as tbl1 ) order by q')
+    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q   having count(url) <= 3) as tbl1 ) order by q')
 
     riddict={}
     prev=''
@@ -225,10 +231,11 @@ def gen_pic():
 #r=checkig('募資')
 #r=checkig('遠赤外線')
 #lst=['491','31','293','278','24','594','356','307','491','33','385']
-lst=['491','31','293','278','24','594','356','307','491','33','385']
+#lst=['491','31','293','278','24','594','356','307','491','33','385']
 
-for l in lst:
-    r=checkig(l)
+#for l in lst:
+#    r=checkig(l)
+checkig('12')
 gen_pic()
 #r=checkig('信用卡')
 #print(r)

hhh/hhh_tree2.py

@@ -0,0 +1,190 @@
+#from instaloader import Instaloader, Profile
+import traceback
+import copy
+import operator
+import dataset
+import pandas as pd
+import networkx as nx
+#import pysftp
+import codecs
+import pyvis
+import sys
+import pickle
+import os
+import searchconsole
+from pyvis.network import Network
+import jieba
+pname='hhh.rb'
+
+db = dataset.connect('sqlite:///'+pname+".db")
+
+def destroy_db():
+    global db
+    try:
+        db.query('drop table tmp')
+    except:
+        traceback.print_exc()
+
+table=db['tmp']
+#pname='cont'
+#pname='damanwoo'
+#pname='drama'
+#pname='news'
+
+#pname='www'
+
+#pname='ipromise'
+#pname='sports'
+#pname='rumor'
+#pname='korea'
+
+rid=0
+
+def get_css():
+    fr=codecs.open('jared/data/css.txt','r','utf-8')
+    lines=fr.readlines()
+    content=' '.join(lines)
+    fr.close()
+    return content
+
+def modify_file(fname):
+    fr=codecs.open(fname,'r','utf-8')
+    lines=fr.readlines()
+    fr.close()
+#    css=get_css()
+    css=''
+    content_output=''
+    for l in lines:
+        if '<body>' in l[0:10]:
+            content_output+=l
+            content_output+='\n<div id="google">\n'
+            continue
+        if '<style type="text' in l[0:22]:
+            content_output+=l
+            content_output+="\n"+css+"\n"
+            continue
+        if '<div id = "mynetwork"' in l[0:30]:
+            content_output+=l
+            content_output+='\n</div>\n'
+            continue
+
+        content_output+=l
+
+    fw=codecs.open("mod_"+fname,'w','utf-8')
+    fw.write(content_output)
+    fw.close()
+
+def checkig(pgnum):
+    global instl
+    global table
+    global pname
+    global rid
+    lst=[]
+    cntdict={}
+    codelist={}
+    idx=0
+    flag_break=False
+
+    fname=os.path.abspath(__file__)
+    elmts=fname.split(os.path.sep)
+    path2=os.path.sep.join(elmts[0:-1])
+    keysdir=path2+os.path.sep+'keys'+os.path.sep
+
+#    account = searchconsole.authenticate(client_config='c:/keys/client_secret.json',credentials='c:/keys/credentials.json')
+#    account = searchconsole.authenticate(client_config='c:/keys/client_secret.json',credentials='c:/keys/credentials.json')
+#account = searchconsole.authenticate(client_config='c:/keys/client_secret_162277274609-v1fsq5iscscl7e2ta4a8tc0og5tehl44.apps.googleusercontent.com.json',serialize='out.json')
+    account = searchconsole.authenticate(client_config='c:/keys/client_secret_162277274609-v1fsq5iscscl7e2ta4a8tc0og5tehl44.apps.googleusercontent.com.json',credentials='c:/keys/out.json')
+
+#account.redirect_uri = 'https://localhost'
+#http://localhost:8080
+
+#    account = searchconsole.authenticate(client_config=keysdir+'client_secret copy.json',credentials=keysdir+'credentials copy.json')
+    print(account.webproperties)
+#    sys.exit()
+#    webproperty = account['https://ipromise.com.tw/']
+#    webproperty = account['https://'+pname+'.face8ook.org/']
+#    webproperty = account['https://www.damanwoo.com/']
+#    webproperty = account['https://hhh.com.tw/']
+    webproperty = account['https://innews.com.tw/']
+
+#    report=webproperty.query.range('2021-03-01', '2021-06-17').dimension('page','query').get()
+#    report=webproperty.query.range('2021-06-01', '2021-06-17').dimension('page','query').get()
+#    report=webproperty.query.range('2020-06-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/(491|31|293|278|31|24|594|356|307|491|33|385)', 'equals').get()
+#    report=webproperty.query.range('2020-03-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/'+pgnum, 'contains').get()
+    report=webproperty.query.range('2022-04-01', '2022-04-16').dimension('page','query').get()
+
+
+    urlq={}
+    for r in report.rows:
+        if urlq.get(r[0]) is None:
+            urlq[r[0]]=[r[1]]
+        else:
+            urlq[r[0]].append(r[1])
+
+#    print(urlq)
+    allrows=[]
+    for k,v in urlq.items():
+        for q in v:
+            elmts=q.split(' ')
+            for elmt in elmts:
+                table.insert({'q':elmt,'rid':rid,'url':k})
+        rid+=1
+        allrows.append([r[0],r[1] ])
+
+    db.commit()
+
+
+def gen_pic():
+    global db
+    G=None
+#    if os.path.exists(pname):
+#        G = pickle.load( open( pname, "rb" ) )
+#    else:
+#        G = nx.Graph()
+    G = nx.Graph()
+
+    finallist=[]
+
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having ) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q   ) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q   having count(url) >=3) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q   having count(url) >=3) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where rid in (select distinct rid from  (select rid,count(q) from tmp where length(q)> 2 group by rid   having count(q) >=15) as tbl1 ) order by q')
+    cursor=db.query('select distinct q,rid from tmp where rid in (select distinct rid from  (select rid,count(q) from tmp where length(q)> 2 group by rid   having count(q) >=50) as tbl1 ) order by q')
+
+    riddict={}
+    prev=''
+    curnode=''
+    cururl=''
+
+    total_idx=0
+    cnt=0
+    for c in cursor:
+        print(str(c['rid'])+":"+c['q'])
+        G.add_edge(c['q'],str(c['rid']),weight=3,width=3,borderwidth=3)
+        cnt+=1
+    print(cnt)
+
+#    pickle.dump( G, open( pname, "wb" ) )
+
+    remove=[]
+    G.remove_edges_from(nx.selfloop_edges(G))
+    G.remove_nodes_from(list(nx.isolates(G)))
+    G2=G
+    pyG = Network(height="600px", width="100%",bgcolor="#444444",font_color="white")
+
+    pyG.from_nx(G2)
+    pyG.show(pname+'.html')
+#    modify_file(pname+'.html')
+
+
+    return finallist
+
+
+#    cursor=db.query('(select q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 )')
+
+#destroy_db()
+#checkig('12')
+gen_pic()

hhh/imp_content_top.py

@@ -0,0 +1,43 @@
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import  googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+
+
+table=db['content_top_terms']
+curdir=os.path.realpath('.')
+
+#fr=codecs.open(curdir+os.sep+'contentgap.txt','r','utf-8')
+#fr=codecs.open(curdir+os.sep+'hhh\\seo\\contentgap.txt','r','utf-8')
+fr=codecs.open('C:\\gitlab\\kw_tools\\top.csv','r','utf-8')
+lines=fr.readlines()
+lst=[]
+for l in lines:
+    term=l.replace('\n','')
+    table.insert({'term':term})
+
+
+
+
+

hhh/notify_tests.py

@@ -7,11 +7,11 @@ headers = {
 #        "Authorization": "Bearer " + "WekCRfnAirSiSxALiD6gcm0B56EejsoK89zFbIaiZQD",
 #        "Authorization": "Bearer " + "h7QtBUeJLTrYOY89sMqgNEKcM8qlwbkI3olYgGDghb4",
 #        "Authorization": "Bearer " + "t35vhZtWNgvDNWHc3DJh0OKll3mcB9GvC8K2EAkBug2",
-        "Authorization": "Bearer " + "1dbtJHbWVbrooXmQqc4r8OyRWDryjD4TMJ6DiDsdgsX",
+#        "Authorization": "Bearer " + "1dbtJHbWVbrooXmQqc4r8OyRWDryjD4TMJ6DiDsdgsX",
 #        "Authorization": "Bearer " + "7vilzohcyQMPLfAMRloUawiTV4vtusZhxv8Czo7AJX8",
 #        "Authorization": "Bearer " + "HOB1kVNgIb81tTB4Ort1BfhVp9GFo6NlToMQg88vEhh",
-
- 
+#        "Authorization": "Bearer " + "6SDULL1Ebklduc6TFxa97AFto5Sj21kyJ30CxiLiSoi",
+        "Authorization": "Bearer " + "OZDcq7sVKwr3F6YNLtBF3LuIgpa4Ql9eAnBWeD7sHTJ", 
 
         "Content-Type": "application/x-www-form-urlencoded"
 }

hhh/tools/api_db.bat

@@ -0,0 +1,2 @@
+ssh -L 13317:hhh-v57-cluster.cluster-cmab1ctkglka.ap-northeast-2.rds.amazonaws.com:3306 root@dev2.choozmo.com
+

hhh/yt/oauthtest.py

@@ -0,0 +1,8 @@
+from google_auth_oauthlib.flow import Flow
+
+flow = Flow.from_client_secrets_file(
+    'c:\\keys\\client_secret_244642730545-m6s4um0g6p3289ig7k92oseusisibnfp.apps.googleusercontent.com.json',
+    scopes=['openid', 'https://www.googleapis.com/auth/userinfo.email', 'https://www.googleapis.com/auth/userinfo.profile'],
+    redirect_uri='urn:ietf:wg:oauth:2.0:oob')
+
+#

hhh/yt/replace_test.py

@@ -0,0 +1,8 @@
+from youtube_upload.client import YoutubeUploader
+import dataset
+import sys
+import codecs
+
+db = dataset.connect('mysql://hhh7796hhh:lYmWsu^ujcA1@127.0.0.1:13317/xoops?charset=utf8mb4')
+
+cursor=db.query('select * from youtube_list where youtube_video_id="'+yt+'"')

hhh/yt/ytupload.py

@@ -0,0 +1,35 @@
+from simple_youtube_api.Channel import Channel
+from simple_youtube_api.LocalVideo import LocalVideo
+
+# loggin into the channel
+channel = Channel()
+#channel.login("c:\\keys\\client_secret_244642730545-m6s4um0g6p3289ig7k92oseusisibnfp.apps.googleusercontent.com.json","c:\\keys\\cre.storage")
+channel.login("c:\\keys\\client_secret_244642730545-m6s4um0g6p3289ig7k92oseusisibnfp.apps.googleusercontent.com.json",storage_path="c:\\tmp\\storage.json")
+
+#, "credentials.storage"
+# setting up the video that is going to be uploaded
+video = LocalVideo(file_path="C:\\data\\demo\\test.mp4")
+
+# setting snippet
+video.set_title("My Title")
+video.set_description("This is a description")
+video.set_tags(["this", "tag"])
+video.set_category("gaming")
+video.set_default_language("en-US")
+
+# setting status
+video.set_embeddable(True)
+video.set_license("creativeCommon")
+video.set_privacy_status("private")
+video.set_public_stats_viewable(True)
+
+# setting thumbnail
+#video.set_thumbnail_path('test_thumb.png')
+
+# uploading video and printing the results
+video = channel.upload_video(video)
+print(video.id)
+print(video)
+
+# liking video
+video.like()

imp_contentgap.py

@@ -0,0 +1,43 @@
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import datetime
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import  googlesearch
+import codecs
+import sys
+import time
+import dataset
+import os
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+
+
+table=db['contentgap_terms']
+curdir=os.path.realpath('.')
+
+#fr=codecs.open(curdir+os.sep+'contentgap.txt','r','utf-8')
+#fr=codecs.open(curdir+os.sep+'hhh\\seo\\contentgap.txt','r','utf-8')
+fr=codecs.open('C:\\gitlab\\kw_tools\\kw_tools\\hhh\\SEO\\contentgap.txt','r','utf-8')
+lines=fr.readlines()
+lst=[]
+for l in lines:
+    term=l.replace('\n','')
+    table.insert({'term':term})
+
+
+
+
+

log.txt

@@ -0,0 +1,2 @@
+2022/05/31 10:45:27	|	process pid: 21596 	|	    1 attempt	|	    0 successful click	|	    0 timeout	|	     1 recaptcha 	|	    0 error
+2022/05/31 10:45:42	|	process pid: 21596 	|	    2 attempt	|	    0 successful click	|	    0 timeout	|	     2 recaptcha 	|	    0 error

monitor/GA_organic.py

@@ -0,0 +1,160 @@
+#!/usr/bin/python3
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+from googleapiclient.discovery import build
+from oauth2client.service_account import ServiceAccountCredentials
+import dataset
+from datetime import datetime
+
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+db.query('delete from ga_pagepath')
+db.begin()
+
+table=db['ga_pagepath']
+
+SCOPES = ['https://www.googleapis.com/auth/analytics.readonly']
+#KEY_FILE_LOCATION = '/root/src/kw_tools/monitor/choozmo-ga-beee24b7a4c1.json' #'c:\\keys\\choozmo-ga-beee24b7a4c1.json'
+KEY_FILE_LOCATION = 'c:\\keys\\choozmo-ga-beee24b7a4c1.json' #'c:\\keys\\choozmo-ga-beee24b7a4c1.json'
+
+VIEW_ID = '188916214'
+
+# line notify header
+headers = {
+        "Authorization": "Bearer " + "WekCRfnAirSiSxALiD6gcm0B56EejsoK89zFbIaiZQD",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+com_table = []
+def creat_table():
+  for i in range(0,24):
+    com_table.append([i,2920*i])
+  com_table.append([24,70000])
+  print(com_table)
+
+def send_msg(kw):
+  # line notify send message
+    current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') #現在時間
+    hour = datetime.now().strftime('%H') 
+    # 判斷是否達標
+    complet="否"
+    for i in range(0,25):
+      if int(hour)+1==com_table[i][0]:
+        print(i)
+        if int(kw) > com_table[i][1] : 
+          complet="是"
+      elif int(hour) == 24:
+        if int(kw) > 70000 : 
+          complet="是"
+    params = {"message": "\n現在時間: " + current_time + "\n當前pageViews: "+kw + "\n是否達標: " + complet}  
+    print(params)
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+    print(r.text)
+
+
+def initialize_analyticsreporting():
+  """Initializes an Analytics Reporting API V4 service object.
+
+  Returns:
+    An authorized Analytics Reporting API V4 service object.
+  """
+  credentials = ServiceAccountCredentials.from_json_keyfile_name(
+      KEY_FILE_LOCATION, SCOPES)
+
+  # Build the service object.
+  analytics = build('analyticsreporting', 'v4', credentials=credentials)
+
+  return analytics
+
+
+def get_report(analytics,body):
+  """Queries the Analytics Reporting API V4.
+
+  Args:
+    analytics: An authorized Analytics Reporting API V4 service object.
+  Returns:
+    The Analytics Reporting API V4 response.
+  """
+  return analytics.reports().batchGet(
+      body={
+        'reportRequests':body
+      }
+  ).execute()
+
+
+def print_response(response):
+  """Parses and prints the Analytics Reporting API V4 response.
+
+  Args:
+    response: An Analytics Reporting API V4 response.
+  """
+  result=[]
+  for report in response.get('reports', []):
+    columnHeader = report.get('columnHeader', {})
+    dimensionHeaders = columnHeader.get('dimensions', [])
+    metricHeaders = columnHeader.get('metricHeader', {}).get('metricHeaderEntries', [])
+
+    for row in report.get('data', {}).get('rows', []):
+      dimensions = row.get('dimensions', [])
+      dateRangeValues = row.get('metrics', [])
+      ga_dict={}
+
+      for header, dimension in zip(dimensionHeaders, dimensions):
+#        print(header + ': ', dimension)
+        ga_dict[header]=dimension
+      
+      for i, values in enumerate(dateRangeValues):
+#        print('Date range:', str(i))
+        for metricHeader, value in zip(metricHeaders, values.get('values')):
+          ga_dict[metricHeader.get('name')]=value
+#          print(metricHeader.get('name') + ':', value)
+        result.append(ga_dict)
+    return result
+#        print(ga_dict)
+
+def main():
+  analytics = initialize_analyticsreporting()
+#(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)
+  current_time = datetime.now().strftime('%Y-%m-%d') #現在時間
+  body=[{ 'viewId': VIEW_ID,
+  'dateRanges': [{'startDate': current_time, 'endDate': current_time}],
+
+  'metrics': [{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'}],
+  'dimensions': [{'name': 'ga:sourceMedium'}],
+#  'orderBys':[{"fieldName": "ga:pageviews", "sortOrder": "DESCENDING"}],
+  'pageSize': '100'
+  }]
+
+  response = get_report(analytics,body)
+  ga_dict=print_response(response)
+
+  result=[]
+
+
+
+  for elmt in ga_dict:
+    hour = datetime.now().strftime('%H')
+    if 'google' in elmt['ga:sourceMedium'] and 'organic' in elmt['ga:sourceMedium']:
+      print(elmt)
+
+
+#    if int(hour)+1 > 8 :
+#      send_msg(elmt['ga:pageviews'])
+#  print('inserting.....')
+
+if __name__ == '__main__':
+#  creat_table()
+  main()

monitor/apptest.py

@@ -0,0 +1,14 @@
+from appium import webdriver
+import time
+import os
+
+desired_caps = {}  # 空字典 需要給予測試手機相關資訊
+desired_caps['platformName'] = 'Android'  # 測試的平台
+desired_caps['platformVersion'] = '12.0'  # 手機os版本
+desired_caps['deviceName'] = 'R5CR11LF0LP'  # 看你手機的名稱是什麼
+desired_caps['automationName'] = "UiAutomator2"  # 一定要的！
+desired_caps['autoGrantPermissions'] = True  # 自動同意授權
+desired_caps['appPackage'] = 'com.android.chrome'
+desired_caps['appActivity'] = 'com.google.android.apps.chrome.Main' 
+desired_caps['noReset'] = False  # 改成 True，會保留原先資料，但自動化腳本會失敗
+driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps)

monitor/docker_clickbot.py

@@ -0,0 +1,139 @@
+import time
+import json
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+import requests
+import datetime
+import dataset
+import time
+import traceback
+import sys
+import fire
+import random
+import pymysql
+pymysql.install_as_MySQLdb()
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['general_log']
+
+
+driver = None
+
+
+def rua():
+    pool = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 OPR/68.0.3618.125",
+    ]
+    return random.choice(pool)
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+
+def process_query(qs):
+    q=qs[0]
+    domain=qs[1]
+    global driver
+    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(urllib.parse.quote(q), 100,'zh-TW')
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(6)
+
+    while True:
+        try:
+            elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']/a")
+            print('尋找')
+            break
+        except:
+            pass
+
+    idx=1
+    ranking=-1
+    print('搜尋結果數量',len(elmts))
+#    driver.save_screenshot('c:/tmp/test.png')
+
+    for elmt in elmts:
+
+        href=elmt.get_attribute('href')
+        txt=elmt.text
+        if len(txt)>10:
+            if domain in href:
+                print('clicked....')
+                print(href)
+                print(txt)
+                print("ranking", idx)
+                table.insert({'kw':q,'domain':domain,'ranking':idx,'title':txt,'url':href,'dt':datetime.datetime.now(),'num':1})
+                webdriver.ActionChains(driver).move_to_element(elmt).perform()
+                webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+                break
+        idx+=1
+
+def run_once(q):
+    global driver
+    result=[]
+    s = Service('/root/driver/chromedriver')
+    # s = Service('/Users/zooeytsai/Downloads/chromedriver 2')
+    user_agent = rua()
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless')
+    options.add_argument('--remote-debugging-port='+str(q[2]))
+#    options.add_experimental_option("debuggerAddress", "127.0.0.1:{q[2]}")
+    options.add_argument("--user-agent=" +user_agent)
+    options.add_argument("--incognito")
+
+    driver = webdriver.Chrome(options=options,service=s)
+    
+    driver.delete_all_cookies()
+    driver.set_window_size(1400,1000)
+
+    print('到此')
+    process_query(q)
+    time.sleep(3)
+    driver.quit()
+
+
+#for c in lst:
+#while True:
+#    try:
+#        c=random.choice(lst)
+#    except:
+#        traceback.print_exc()
+#    sleepint=random.randint(320,520)
+#    time.sleep(sleepint)
+
+class JParams(object):
+
+  def get(self, kw,domain,port):
+    print(kw)
+    print(domain)
+    run_once( (kw,domain,port)   )
+
+
+if __name__ == '__main__':
+  fire.Fire(JParams)
+
+
+  def get(self, kw, domain, port):
+      print('kw')
+      print(domain)
+      run_once((kw, domain, port))
+

monitor/general_click_tor.py

@@ -0,0 +1,178 @@
+import time
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.keys import Keys
+
+import codecs
+import random
+import datetime
+import dataset
+import time
+import traceback
+import sys
+import random
+import socket
+import requests
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['general_log']
+
+
+driver = None
+headers = {
+        "Authorization": "Bearer " + "6SDULL1Ebklduc6TFxa97AFto5Sj21kyJ30CxiLiSoi",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+
+def rua():
+    pool = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 OPR/68.0.3618.125",
+    ]
+    return random.choice(pool)
+
+
+
+
+def send_msg(kw):
+    hname=socket.gethostname()
+    params = {"message": hname+": "+kw}  
+    r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
+
+
+
+def empty_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+
+
+def process_query(qs):
+    q=qs[0]
+    domain=qs[1]
+    global driver
+#    googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(urllib.parse.quote(q), 100,'zh-TW')
+    googleurl = 'https://www.google.com/'
+
+    print(googleurl)
+    driver.get(googleurl)
+    time.sleep(6)
+#    if 'sorry' in driver.page_url:
+#        return None
+    try:
+        elmt = driver.find_element(By.XPATH, "//button[@id='L2AGLb']")
+        if elmt:
+            elmt.click()           
+    except:
+        print('exception')
+
+
+    try:
+        elmt = driver.find_element(By.XPATH, "//input[@aria-label='搜尋']")
+        if elmt:
+            elmt.send_keys(q)
+            elmt.send_keys(Keys.ENTER)
+    except:
+        print(elmt)
+
+
+
+    while True:
+        try:
+            elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']/a")
+            print('尋找')
+            time.sleep(2)
+            break
+        except:
+            pass
+
+    idx=1
+    ranking=-1
+    print('搜尋結果數量',len(elmts))
+    if len(elmts) <=0:
+        send_msg('network failed...')
+#    driver.save_screenshot('c:/tmp/test.png')
+
+    for elmt in elmts:
+
+        href=elmt.get_attribute('href')
+        txt=elmt.text
+        if len(txt)>10:
+            if domain in href:
+                print('clicked....')
+                print(href)
+                print(txt)
+                print("ranking", idx)
+                table.insert({'kw':q,'domain':domain,'ranking':idx,'title':txt,'url':href,'dt':datetime.datetime.now(),'num':1})
+                webdriver.ActionChains(driver).move_to_element(elmt).perform()
+                webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
+                break
+        idx+=1
+    return "ok"
+
+def run_once(q):
+    global driver
+    result=[]
+    os.system('docker container restart proxy1')
+    time.sleep(12)
+#    s = Service('/root/driver/chromedriver')
+    # s = Service('/Users/zooeytsai/Downloads/chromedriver 2')
+    user_agent = rua()
+    options = webdriver.ChromeOptions()
+#    options.add_argument('--headless')
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+
+#    options.add_argument("start-maximized")
+#    options.add_argument('--remote-debugging-port='+str(q[2]))
+#    options.add_argument('--remote-debugging-port=9222')
+#    options.add_argument("--user-agent=" +user_agent)
+    options.add_argument("--incognito")
+
+#    driver = webdriver.Chrome(options=options,service=s)
+    print('before init')
+#    driver = webdriver.Chrome(options=options)
+
+#    from tbselenium.tbdriver import TorBrowserDriver
+#    driver=TorBrowserDriver("C:\\Users\\jared\\OneDrive\\桌面\\Tor Browser\\")
+#    driver=TorBrowserDriver("C:/portable/tor/Tor Browser/")
+#    driver=TorBrowserDriver("/root/tor-browser_en-US/")
+
+
+
+#    driver.get('https://check.torproject.org')
+
+    print('after init')
+ 
+    driver.delete_all_cookies()
+#    driver.set_window_size(1400,1000)
+
+    print('到此')
+    data=process_query(q)
+    if data is not None:
+        time.sleep(3)
+    driver.quit()
+    sys.exit()
+
+cursor = db.query('SELECT query FROM seo.hhh_gsc_imp where position >=2.5 and position <=8.5 order by rand() limit 1')
+query=None
+for c in cursor:
+    query=c['query']
+    print(query)
+    break
+
+run_once((query,'hhh.com.tw'))
+

monitor/general_clickbot_hhh.py

@@ -101,10 +101,8 @@ def run_once(q):
     user_agent = rua()
     options = webdriver.ChromeOptions()
     options.add_argument('--headless')
-<<<<<<< HEAD
     options.add_argument('--remote-debugging-port=9222')
     options.add_experimental_option("debuggerAddress", f"127.0.0.1:{q[2]}")
-=======
     options.add_argument("--no-sandbox")
     options.add_argument("--disable-dev-shm-usage")
 ###    options.add_argument("--proxy-server=socks5://192.168.192.156:1080")
@@ -116,7 +114,6 @@ def run_once(q):
 #    options.add_argument("--user-agent=" +user_agent)
 #    options.add_argument('--remote-debugging-port=9222')
 #    options.add_experimental_option("debuggerAddress", "127.0.0.1:{q[2]}")
->>>>>>> d38b39b4e07b1ed66eb4980b06240a2e31f3ac76
     options.add_argument("--user-agent=" +user_agent)
     options.add_argument("--incognito")
 

monitor/imp_csv.py

@@ -0,0 +1,20 @@
+import codecs
+import dataset
+
+fr=codecs.open('c:/tmp/Queries.csv','r','utf-8')
+lines=fr.readlines()
+fr.close()
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
+table=db['hhh_gsc_imp']
+
+for l in lines:
+#    print(l)
+    elmts=l.strip().split(',')
+    print(elmts[0])
+    print(elmts[4])
+    try:
+        table.insert({'query':elmts[0],'position':elmts[4]})
+    except:
+        print('except')
+

monitor/local_loop.py

@@ -0,0 +1,5 @@
+import os
+import time
+while True:
+    os.system('docker container restart hhh1')
+    time.sleep(30)

monitor/test_github.py

@@ -0,0 +1,204 @@
+import os
+import sys
+from datetime import datetime
+from time import sleep
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from threading import Thread
+from queue import Queue
+import argparse
+
+#chrome_driver_executable_path = ''
+#if getattr(sys, 'frozen', False):
+#    # publish one file
+#   chrome_driver_executable_path = sys._MEIPASS + r'\resource\chromedriver.exe'
+#else:
+    # source
+#    chrome_driver_executable_path = os.getcwd() + r'\resource\chromedriver.exe'
+
+capa = DesiredCapabilities.CHROME
+capa["pageLoadStrategy"] = "none"
+capa['loggingPrefs'] = {'driver': 'OFF', 'server': 'OFF', 'browser': 'OFF'}
+
+attempt = 0
+successful_attempt = 0
+final_sleep = 0
+timeout_ex = 0
+other_ex = 0
+recaptcha_ex = 0
+
+
+def main_script(keyword, site_url, max_successful_clicks, browser_visibility_flag):
+    global attempt
+    global successful_attempt
+    global final_sleep
+    global timeout_ex
+    global other_ex
+    global recaptcha_ex
+
+    while successful_attempt < max_successful_clicks:
+
+        options = Options()
+        if not browser_visibility_flag:
+            options.add_argument('--headless')
+        options.add_argument("--js-flags=--expose-gc")
+        options.add_argument("--enable-precise-memory-info")
+        options.add_argument("--disable-popup-blocking")
+        options.add_argument("--disable-default-apps")
+        options.add_argument("disable-infobars")
+        options.add_argument('--disable-contextual-search')
+        options.add_argument("--disable-notifications")
+        options.add_argument('--incognito')
+        options.add_argument('--disable-application-cache')
+        options.add_argument('--no-sandbox')
+        options.add_argument('--disk-cache-size=0')
+        options.add_argument('–-disable-restore-session-state')
+        options.add_argument('--disable-extensions')
+        options.add_argument('test-type')
+        options.add_argument('--silent')
+        options.add_argument('--log-level=3')
+        options.add_argument("--proxy-server=socks5://127.0.0.1:9050")
+
+        driver = None
+
+        try:
+            driver = webdriver.Chrome(
+#                executable_path=chrome_driver_executable_path,
+                chrome_options=options,
+                desired_capabilities=capa)
+            wait = WebDriverWait(driver, 40)
+            driver.implicitly_wait(10)
+            driver.set_page_load_timeout(80)
+
+            # driver.get('http://google.com/search?q=' + urllib.parse.urlencode(keyword)) # does not work
+            driver.get('http://google.com/')
+            wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'input[name="q"]')))
+            driver.execute_script("window.stop();")
+            driver.execute_script("window.stop();")
+
+            attempt += 1
+            elm = driver.find_element_by_css_selector('input[name="q"]')
+            elm.send_keys(keyword)
+            elm.send_keys(Keys.RETURN)
+            sleep(3)
+            recaptcha = None
+            try:
+                recaptcha = driver.find_elements_by_css_selector('#recaptcha')
+            except:
+                pass
+            if not recaptcha:
+                wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#res')))
+                driver.execute_script("window.stop();")
+                site_links = driver.find_elements_by_css_selector('#res a')
+                site_links_filtered = filter(lambda x: site_url in x.get_attribute('href'), site_links)
+                site_link = next(site_links_filtered)
+                site_link.click()
+                wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'body')))
+                sleep(10)
+                elm = driver.find_element_by_css_selector('body')
+                elm.send_keys(Keys.RETURN)
+                successful_attempt += 1
+                final_sleep = 240 # staying in the website
+            else:
+                driver.execute_script("window.stop();")
+                recaptcha_ex += 1
+                final_sleep = 0
+
+        except TimeoutException as ex:
+            final_sleep = 0
+            timeout_ex += 1
+        except Exception as ex:
+            final_sleep = 0
+            other_ex += 1
+        else:
+            info_message = '{}\t|\tprocess pid: {: <6}\t|\t{: >5} attempt\t|\t{: >5} successful click\t|\t{: >5} timeout\t|\t {: >5} recaptcha \t|\t{: >5} error'.format(
+                datetime.now().strftime('%Y/%m/%d %H:%M:%S'), os.getpid(), attempt,
+                successful_attempt,
+                timeout_ex,
+                recaptcha_ex,
+                other_ex
+            )
+            print(info_message)
+            f = open('log.txt', 'a')
+            f.write(info_message + '\n')
+            f.close()
+        finally:
+            sleep(final_sleep)
+            if driver:
+                driver.close()
+            del driver
+
+    print()
+    print('process pid: {} | finished {} successful clicks! \nprocess stopped...'.format(os.getpid(),
+                                                                                         max_successful_clicks))
+    f = open('log.txt', 'a')
+    f.write('process pid: {} | finished {} successful clicks! \nprocess stopped...'.format(os.getpid(),
+                                                                                           max_successful_clicks) + '\n')
+    f.close()
+    print()
+
+
+class WorkerThread(Thread):
+    def __init__(self, q, keyword, site_url, max_successful_clicks, browser_visibility_flag):
+        super().__init__()
+        self.setDaemon(True)
+        self.q = q
+        self.keyword = keyword
+        self.site_url = site_url
+        self.max_successful_clicks = max_successful_clicks
+        self.browser_visibility_flag = browser_visibility_flag
+
+    def run(self):
+        while True:
+            print()
+            print('automated searching started...')
+            print()
+            main_script(self.keyword, self.site_url, self.max_successful_clicks, self.browser_visibility_flag)
+            self.q.task_done()
+
+
+#parser = argparse.ArgumentParser(prog='python automate_search_google.py',
+#                                 description='A script to automatically search keywords on google and click on your desired website link as much as you want.')
+
+#parser.add_argument('k', help='list of keywords to search in format of "[\'keyword 1\',\'keyword 2\']"')
+#parser.add_argument('u', help='Url of your desired website to be clicked without http and www. eg: example.com')
+#parser.add_argument('m', type=int, help='maximum successful click count you want. eg: 500')
+#parser.add_argument('-v', '--visible', action='store_true',
+#                    help='browser visibility flag. if present the browser become visible')
+
+#args = parser.parse_args()
+
+#keywords = eval(args.k)
+keywords=['幸福空間']
+message = '  please start tor browser first  '.upper()
+print()
+message_str = '#' * 10 + message + '#' * 10
+print('#' * len(message_str))
+print('#' * len(message_str))
+print('#' * 10 + ' ' * len(message) + '#' * 10)
+print(message_str)
+print('#' * 10 + ' ' * len(message) + '#' * 10)
+print('#' * len(message_str))
+print('#' * len(message_str))
+print()
+
+my_queue = Queue()
+
+for key in keywords:
+    worker = WorkerThread(my_queue, key, 'hhh.com.tw', 1, 1)
+    worker.start()
+
+for i in keywords:
+    my_queue.put(i)
+    sleep(1)
+
+my_queue.join()
+
+c = input('Press Enter to exit program... ')
+exit(0)

newproj/alston_exp.py

@@ -0,0 +1,18 @@
+import dataset
+
+
+from pymysql import*
+import xlwt
+import pandas.io.sql as sql
+# connect the mysql with the python
+con=connect(user="choozmo",password="pAssw0rd",host="db.ptt.cx",database="google_poi")
+
+# read the data
+#df=sql.read_sql('select * from shop_list2',con)
+df=sql.read_sql('select * from swire_store_list',con)
+
+
+# print the data
+print(df)
+# export the data into the excel sheet
+df.to_excel('storelist.xlsx')

newproj/sitemap_reader.py

@@ -0,0 +1,27 @@
+import codecs
+from bs4 import BeautifulSoup 
+
+# Reading the data inside the xml file to a variable under the name  data
+with codecs.open('c:/tmp/sitemap.xml', 'r','utf-8') as f:
+    data = f.read() 
+
+# Passing the stored data inside the beautifulsoup parser 
+bs_data = BeautifulSoup(data, 'xml') 
+
+# Finding all instances of tag   
+b_unique = bs_data.find_all('url') 
+cnt=0
+for el in b_unique:
+    loc=el.find('loc')
+    print(loc.text)
+#b_name = bs_data.find('child', {'name':'Acer'}) 
+
+#print(b_unique) 
+
+# Using find() to extract attributes of the first instance of the tag 
+#b_name = bs_data.find('child', {'name':'Acer'}) 
+#print(b_name) 
+
+# Extracting the data stored in a specific attribute of the `child` tag 
+#value = b_name.get('qty') 
+#print(value)