|
@@ -54,7 +54,7 @@ def process_query(qs):
|
|
googleurl = 'https://www.google.com/?num=100'
|
|
googleurl = 'https://www.google.com/?num=100'
|
|
driver.get(googleurl)
|
|
driver.get(googleurl)
|
|
time.sleep(6)
|
|
time.sleep(6)
|
|
- send_kw_elmt = driver.find_element(By.XPATH, '/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input')
|
|
|
|
|
|
+ send_kw_elmt = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[1]/div[3]/form/div[1]/div[1]/div[1]/div/div[2]/input')))
|
|
send_kw_elmt.send_keys(q)
|
|
send_kw_elmt.send_keys(q)
|
|
time.sleep(3)
|
|
time.sleep(3)
|
|
send_kw_elmt.send_keys(Keys.ENTER)
|
|
send_kw_elmt.send_keys(Keys.ENTER)
|
|
@@ -76,24 +76,30 @@ def process_query(qs):
|
|
table.insert(
|
|
table.insert(
|
|
{'kw': q, 'domain': domain, 'ranking': idx, 'title': txt, 'url': href, 'dt': datetime.datetime.now()})
|
|
{'kw': q, 'domain': domain, 'ranking': idx, 'title': txt, 'url': href, 'dt': datetime.datetime.now()})
|
|
webdriver.ActionChains(driver).move_to_element(elmts[0]).perform()
|
|
webdriver.ActionChains(driver).move_to_element(elmts[0]).perform()
|
|
|
|
+ time.sleep(3)
|
|
webdriver.ActionChains(driver).move_to_element(elmts[0]).click().perform()
|
|
webdriver.ActionChains(driver).move_to_element(elmts[0]).click().perform()
|
|
time.sleep(5)
|
|
time.sleep(5)
|
|
- for elmt in elmts:
|
|
|
|
- href=elmt.get_attribute('href')
|
|
|
|
- txt=elmt.text
|
|
|
|
- if len(txt)>10:
|
|
|
|
- if domain in href:
|
|
|
|
- print('clicked....')
|
|
|
|
- print('點擊網址',href)
|
|
|
|
- print('標題',txt)
|
|
|
|
- print("ranking", idx)
|
|
|
|
- table.insert({'kw':q,'domain':domain,'ranking':idx,'title':txt,'url':href,'dt':datetime.datetime.now()})
|
|
|
|
- webdriver.ActionChains(driver).move_to_element(elmt).perform()
|
|
|
|
- webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
|
|
|
|
- time.sleep(5)
|
|
|
|
- break
|
|
|
|
- idx+=1
|
|
|
|
|
|
+ else:
|
|
|
|
+ for elmt in elmts:
|
|
|
|
+ href=elmt.get_attribute('href')
|
|
|
|
+ txt=elmt.text
|
|
|
|
+ if len(txt)>10:
|
|
|
|
+ if domain in href:
|
|
|
|
+ print('clicked....')
|
|
|
|
+ print('點擊網址',href)
|
|
|
|
+ print('標題',txt)
|
|
|
|
+ print("ranking", idx)
|
|
|
|
+ webdriver.ActionChains(driver).move_to_element(elmt).perform()
|
|
|
|
+ time.sleep(3)
|
|
|
|
+ webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
|
|
|
|
+ table.insert({'kw':q,'domain':domain,'ranking':idx,'title':txt,'url':href,'dt':datetime.datetime.now()})
|
|
|
|
+ time.sleep(5)
|
|
|
|
+ break
|
|
|
|
+ idx+=1
|
|
db.close()
|
|
db.close()
|
|
|
|
+ print('資料庫關閉')
|
|
|
|
+ driver.quit()
|
|
|
|
+
|
|
def run_once(q):
|
|
def run_once(q):
|
|
global driver
|
|
global driver
|
|
result=[]
|
|
result=[]
|
|
@@ -102,42 +108,27 @@ def run_once(q):
|
|
options = webdriver.ChromeOptions()
|
|
options = webdriver.ChromeOptions()
|
|
options.add_argument('--headless')
|
|
options.add_argument('--headless')
|
|
options.add_argument('--remote-debugging-port=9222')
|
|
options.add_argument('--remote-debugging-port=9222')
|
|
- options.add_experimental_option("debuggerAddress", "127.0.0.1:9922")
|
|
|
|
|
|
+ options.add_experimental_option("debuggerAddress",f"127.0.0.1:{q[2]}")
|
|
# options.add_argument("--user-agent=" +user_agent)
|
|
# options.add_argument("--user-agent=" +user_agent)
|
|
options.add_argument("--incognito")
|
|
options.add_argument("--incognito")
|
|
-<<<<<<< HEAD
|
|
|
|
r = redis.Redis(host='db.ptt.cx', port=6379, db=2,password='choozmo9')
|
|
r = redis.Redis(host='db.ptt.cx', port=6379, db=2,password='choozmo9')
|
|
data=r.get('google_proxy')
|
|
data=r.get('google_proxy')
|
|
jstext=data.decode('utf-8')
|
|
jstext=data.decode('utf-8')
|
|
jsobj=json.loads(jstext)
|
|
jsobj=json.loads(jstext)
|
|
|
|
+ #print('Free proxy',jsobj)
|
|
proxy=random.choice(jsobj)
|
|
proxy=random.choice(jsobj)
|
|
- print('Freeproxy',proxy)
|
|
|
|
- change_ip = ["'--proxy-server='+proxy","--proxy-server=socks5://127.0.0.1:9050","--proxy-server=socks5://192.53.174.202:8180"]
|
|
|
|
- options.add_argument('--proxy-server=socks5://192.53.174.202:8180')
|
|
|
|
- driver = webdriver.Chrome(
|
|
|
|
- options=options,service=s)
|
|
|
|
-=======
|
|
|
|
- if 'sorry' in driver.current_url:
|
|
|
|
- r = redis.Redis(host='db.ptt.cx', port=6379, db=2,password='choozmo9')
|
|
|
|
- data=r.get('google_proxy')
|
|
|
|
- jstext=data.decode('utf-8')
|
|
|
|
- jsobj=json.loads(jstext)
|
|
|
|
- print('Free proxy',jsobj)
|
|
|
|
- proxy=random.choice(jsobj)
|
|
|
|
- change_ip_list = ['--proxy-server=%s' % proxy,"--proxy-server=socks5://127.0.0.1:9050","--proxy-server=socks5://192.53.174.202:8180"]
|
|
|
|
- change_ip = random.choice(change_ip_list)
|
|
|
|
- options.add_argument(change_ip)
|
|
|
|
- print('使用代理ip',change_ip)
|
|
|
|
|
|
+ i5 = "--proxy-server=socks5://172.104.93.163:41800"
|
|
|
|
+ change_ip_list = ['--proxy-server=%s' % proxy,"--proxy-server=socks5://127.0.0.1:9050","--proxy-server=socks5://192.53.174.202:8180"]
|
|
|
|
+ change_ip = random.choice(change_ip_list)
|
|
|
|
+ options.add_argument(i5)
|
|
|
|
+ #print('使用代理ip',change_ip)
|
|
|
|
|
|
driver = webdriver.Chrome(options=options,service=s)
|
|
driver = webdriver.Chrome(options=options,service=s)
|
|
->>>>>>> 28b37f8f0a95c485aaf3da04362d35694cb233c9
|
|
|
|
|
|
|
|
driver.delete_all_cookies()
|
|
driver.delete_all_cookies()
|
|
driver.set_window_size(1400,1000)
|
|
driver.set_window_size(1400,1000)
|
|
|
|
|
|
process_query(q)
|
|
process_query(q)
|
|
- time.sleep(3)
|
|
|
|
- driver.quit()
|
|
|
|
|
|
|
|
|
|
|
|
#for c in lst:
|
|
#for c in lst:
|
|
@@ -152,13 +143,7 @@ def run_once(q):
|
|
class JParams(object):
|
|
class JParams(object):
|
|
|
|
|
|
def get(self, kw,domain,port):
|
|
def get(self, kw,domain,port):
|
|
-<<<<<<< HEAD
|
|
|
|
- os.system('docker container restart tiny1')
|
|
|
|
- time.sleep(1)
|
|
|
|
- run_once( ('台北 禮儀 社','sctt.com.tw','tiny1') )
|
|
|
|
-=======
|
|
|
|
run_once( (kw,domain,port) )
|
|
run_once( (kw,domain,port) )
|
|
->>>>>>> 28b37f8f0a95c485aaf3da04362d35694cb233c9
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if __name__ == '__main__':
|