general_click_proxy.py 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205
  1. import time
  2. from selenium import webdriver
  3. from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
  4. import time
  5. import os
  6. import urllib.parse
  7. from selenium.webdriver.support.ui import WebDriverWait
  8. from selenium.webdriver.common.by import By
  9. from selenium.webdriver.chrome.service import Service
  10. from selenium.webdriver.support import expected_conditions as EC
  11. from selenium.webdriver.common.keys import Keys
  12. import codecs
  13. import random
  14. import datetime
  15. import dataset
  16. import time
  17. import traceback
  18. import sys
  19. import random
  20. import socket
  21. import requests
  22. db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
  23. table=db['general_log']
  24. driver = None
  25. headers = {
  26. "Authorization": "Bearer " + "6SDULL1Ebklduc6TFxa97AFto5Sj21kyJ30CxiLiSoi",
  27. "Content-Type": "application/x-www-form-urlencoded"
  28. }
  29. def scrolling(driver,pgnum):
  30. ub = driver.find_element_by_css_selector('body')
  31. for i in range(pgnum):
  32. ub.send_keys(Keys.PAGE_DOWN)
  33. if pgnum>1:
  34. time.sleep(0.3)
  35. def rua():
  36. pool = [
  37. "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0",
  38. "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
  39. "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
  40. "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36",
  41. "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
  42. "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
  43. "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 OPR/68.0.3618.125",
  44. ]
  45. return random.choice(pool)
  46. def send_msg(kw):
  47. hname=socket.gethostname()
  48. params = {"message": hname+": "+kw}
  49. r = requests.post("https://notify-api.line.me/api/notify",headers=headers, params=params)
  50. def empty_query(q):
  51. global driver
  52. googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
  53. driver.get(googleurl)
  54. time.sleep(3)
  55. def process_query(qs):
  56. q=qs[0]
  57. domain=qs[1]
  58. global driver
  59. # googleurl = 'https://www.google.com/search?q={}&num={}&hl={}'.format(urllib.parse.quote(q), 100,'zh-TW')
  60. googleurl = 'https://www.google.com/'
  61. print(googleurl)
  62. driver.get(googleurl)
  63. time.sleep(6)
  64. if 'sorry' in driver.current_url:
  65. print('sorry...............')
  66. return None
  67. try:
  68. elmt = driver.find_element(By.XPATH, "//button[@id='L2AGLb']")
  69. if elmt:
  70. elmt.click()
  71. except:
  72. print('exception')
  73. try:
  74. # elmt = driver.find_element(By.XPATH, "//input[@aria-label='搜尋']")
  75. elmt = driver.find_element(By.XPATH, "//input[@class='gLFyf gsfi']")
  76. if elmt:
  77. time.sleep(1)
  78. elmt.send_keys(q)
  79. time.sleep(6)
  80. elmt.send_keys(Keys.ENTER)
  81. except:
  82. print(elmt)
  83. time.sleep(6)
  84. while True:
  85. try:
  86. elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']/a")
  87. print('尋找')
  88. time.sleep(2)
  89. break
  90. except:
  91. pass
  92. idx=1
  93. ranking=-1
  94. print('搜尋結果數量',len(elmts))
  95. if len(elmts) <=0:
  96. send_msg('network failed...')
  97. for elmt in elmts:
  98. href=elmt.get_attribute('href')
  99. txt=elmt.text
  100. if len(txt)>10:
  101. if domain in href:
  102. print('clicked....')
  103. print(href)
  104. print(txt)
  105. print("ranking", idx)
  106. driver.execute_script("return document.body.scrollHeight")
  107. driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
  108. elmt.click()
  109. table.insert({'kw':q,'domain':domain,'ranking':idx,'title':txt,'url':href,'dt':datetime.datetime.now(),'num':1})
  110. # webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
  111. print('clicked')
  112. time.sleep(12)
  113. break
  114. idx+=1
  115. return "ok"
  116. def run_once(q):
  117. global driver
  118. result=[]
  119. os.system('docker container restart proxy1')
  120. time.sleep(12)
  121. # s = Service('/root/driver/chromedriver')
  122. # s = Service('/Users/zooeytsai/Downloads/chromedriver 2')
  123. user_agent = rua()
  124. options = webdriver.ChromeOptions()
  125. options.add_argument('--headless')
  126. options.add_argument("--no-sandbox")
  127. options.add_argument("--disable-dev-shm-usage")
  128. ### options.add_argument("--proxy-server=socks5://192.168.192.156:1080")
  129. options.add_argument("--proxy-server=socks5://127.0.0.1:9050")
  130. # options.add_argument("start-maximized")
  131. # options.add_argument('--remote-debugging-port='+str(q[2]))
  132. # options.add_argument('--remote-debugging-port=9222')
  133. # options.add_argument("--user-agent=" +user_agent)
  134. options.add_argument("--incognito")
  135. # driver = webdriver.Chrome(options=options,service=s)
  136. print('before init')
  137. # driver = webdriver.Chrome(options=options)
  138. profile = webdriver.FirefoxProfile()
  139. profile.set_preference("network.proxy.type", 1)
  140. profile.set_preference("network.proxy.socks", "127.0.0.1")
  141. profile.set_preference("network.proxy.socks_port", 9050)
  142. profile.set_preference("network.proxy.socks_version", 5)
  143. profile.update_preferences()
  144. options = webdriver.FirefoxOptions()
  145. options.add_argument('--headless')
  146. driver = webdriver.Firefox(firefox_profile=profile,options=options)
  147. print('after init')
  148. driver.delete_all_cookies()
  149. driver.set_window_size(1400,1000)
  150. # driver.set_window_size(900, 3000)
  151. print('到此')
  152. data=process_query(q)
  153. if data is not None:
  154. time.sleep(3)
  155. driver.quit()
  156. sys.exit()
  157. cursor = db.query('SELECT query FROM seo.hhh_gsc_imp where position >=2.5 and position <=8.5 order by rand() limit 1')
  158. query=None
  159. for c in cursor:
  160. query=c['query']
  161. print(query)
  162. break
  163. run_once((query,'hhh.com.tw'))