ranking_world.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251
  1. # import redis
  2. import time
  3. import traceback
  4. # import json
  5. from selenium import webdriver
  6. from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
  7. import time
  8. import os
  9. from selenium.webdriver.support.ui import WebDriverWait
  10. from selenium.webdriver.common.by import By
  11. from selenium.webdriver.support import expected_conditions as EC
  12. import dataset
  13. from selenium.webdriver.common.keys import Keys
  14. import json
  15. import random
  16. import time
  17. import sys
  18. import codecs
  19. import pandas as pd
  20. import random
  21. import os
  22. import time
  23. import datetime
  24. from selenium.webdriver.chrome.service import Service
  25. import dataset
  26. import pymysql
  27. pymysql.install_as_MySQLdb()
  28. import requests
  29. driver = None
  30. db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
  31. headers = {
  32. "Authorization": "Bearer " + "6SDULL1Ebklduc6TFxa97AFto5Sj21kyJ30CxiLiSoi",
  33. "Content-Type": "application/x-www-form-urlencoded"
  34. }
  35. def send_msg(kw):
  36. params = {"message": kw}
  37. r = requests.post("https://notify-api.line.me/api/notify", headers=headers, params=params)
  38. def re_get_webdriver():
  39. global port
  40. global driver
  41. result = []
  42. if driver is not None:
  43. print('closing....')
  44. driver.quit()
  45. os.system('killall chrome')
  46. print('quit....')
  47. driver = None
  48. try:
  49. # ua = userAgents()
  50. #
  51. # user_agent = ua.random()
  52. s = Service('/Users/mac/Downloads/121/chromedriver')
  53. options = webdriver.ChromeOptions()
  54. options.add_argument("--no-sandbox")
  55. options.add_argument("--disable-dev-shm-usage")
  56. # options.add_argument("--headless")
  57. options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
  58. options.add_argument("--incognito")
  59. driver = None
  60. try:
  61. if os.name == 'nt':
  62. driver = webdriver.Chrome(options=options)
  63. else:
  64. driver = webdriver.Chrome(service=s, options=options)
  65. except:
  66. traceback.print_exc()
  67. return
  68. driver.delete_all_cookies()
  69. driver.set_window_size(950, 20000)
  70. return
  71. except:
  72. traceback.print_exc()
  73. driver = None
  74. return None
  75. def scrolling(driver, pgnum):
  76. ub = driver.find_element("css selector",'body')
  77. for i in range(pgnum):
  78. ub.send_keys(Keys.PAGE_DOWN)
  79. if pgnum > 1:
  80. time.sleep(0.3)
  81. def run_once(jsobj):
  82. table = db['google_rank']
  83. date = jsobj['date']
  84. print(jsobj)
  85. global driver
  86. i = 100
  87. if driver is None:
  88. time.sleep(8)
  89. re_get_webdriver()
  90. if driver is None:
  91. return
  92. try:
  93. kw = jsobj['kw']
  94. fname = jsobj['fname']
  95. url = jsobj['url']
  96. # if jsobj.get('domain') is None:
  97. # exclude = jsobj['exclude']
  98. # domain = None
  99. # else:
  100. # domain = jsobj['domain']
  101. # exclude = None
  102. city_map = {'chicago': ['42.04866173771369', '-87.68260072643513'],
  103. 'miami': ['25.764458843530548', '-80.19787522585152'],
  104. 'wc': ['38.96071674051165', '-77.03155367248387'],
  105. 'ny': ['40.76774141099703', '-73.98439238945637']}
  106. city = jsobj['fname']
  107. print(city)
  108. Map_coordinates = dict({
  109. "latitude": float(city_map[f"{city}"][0]),
  110. "longitude": float(city_map[f"{city}"][1]),
  111. "accuracy": 100
  112. })
  113. # 芝加哥、邁阿密、紐約、華盛頓
  114. # driver.execute_cdp_cmd("Emulation.setGeolocationOverride", Map_coordinates)
  115. # driver.get('https://www.google.com?num=100&lr=lang_en')
  116. driver.get(url)
  117. # print(driver.current_url)
  118. # elmt = driver.find_element(By.XPATH, "//input[@name='q']")
  119. # time.sleep(1)
  120. # elmt.send_keys(kw)
  121. #
  122. # elmt.send_keys(Keys.ENTER)
  123. time.sleep(3)
  124. scrolling(driver, 10)
  125. time.sleep(20)
  126. elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']//a")
  127. numresults = len(elmts)
  128. print('搜尋結果數量', numresults)
  129. time.sleep(20)
  130. # driver.save_screenshot('C:\/Users\/s1301\/Pictures\/Saved Pictures\/angelo koo\/' +date +fname + ".png")
  131. if numresults == 0:
  132. send_msg('stop working...')
  133. sys.exit()
  134. datadict = {'搜尋詞': [], '結果標題': [], '結果網址': [], '結果名次': [], '結果說明': []}
  135. df = pd.DataFrame()
  136. idx = 1
  137. found = False
  138. test_lst = []
  139. txt_dict = {}
  140. for elmt in elmts:
  141. href = elmt.get_attribute('href')
  142. txt = elmt.text
  143. desc = None
  144. try:
  145. elmt2 = elmt.find_element(By.XPATH, "./../../..//div[@data-content-feature=1]")
  146. desc = elmt2.text
  147. except:
  148. desc = None
  149. # print(desc)
  150. table.insert(
  151. {'title': elmt.text, 'url': href, 'keyword': kw, 'dt': datetime.datetime.now(), 'ranking': idx,'description':fname})
  152. datadict['搜尋詞'].append(kw)
  153. datadict['結果標題'].append(txt)
  154. datadict['結果網址'].append(href)
  155. datadict['結果名次'].append(str(idx))
  156. datadict['結果說明'].append(desc)
  157. # if len(txt) > 10:
  158. # if domain is not None:
  159. # for d in domain:
  160. # if d in href:
  161. # print('found....')
  162. # print('clicked....')
  163. # print(href)
  164. # print(txt)
  165. # print("ranking", idx)
  166. # found = True
  167. # return
  168. # else:
  169. # ex = False
  170. # for ee in exclude:
  171. # if ee in href:
  172. # ex = True
  173. # if not ex:
  174. # test_lst.append(elmt)
  175. # txt_dict[elmt] = txt
  176. idx += 1
  177. # if exclude is not None:
  178. # print('exclude')
  179. # elmt = random.choice(test_lst[5:])
  180. # print(elmt)
  181. # print(txt_dict[elmt])
  182. #
  183. # webdriver.ActionChains(driver).move_to_element(elmt).perform()
  184. # # elmt.click()
  185. # webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
  186. # time.sleep(5)
  187. #
  188. # if not found:
  189. # table.insert(
  190. # {'ranking': -1, 'kw': kw, 'results': numresults, 'url': '', 'title': '未收錄', 'descrption': desc})
  191. df['搜尋詞'] = datadict['搜尋詞']
  192. df['結果標題'] = datadict['結果標題']
  193. df['結果網址'] = datadict['結果網址']
  194. df['結果名次'] = datadict['結果名次']
  195. df['結果說明'] = datadict['結果說明']
  196. print('/Users/mac/Documents/wk/'+date+fname+".csv")
  197. df.to_csv('/Users/mac/Documents/wk/'+date+fname+".csv")
  198. except:
  199. print('exception')
  200. traceback.print_exc()
  201. # time.sleep(9999)
  202. # driver.save_screenshot('c:/tmp/seo/'+kw+".png")
  203. driver.quit()
  204. sys.exit()
  205. d = {'ny':"https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIWTmV3IFlvcmssVW5pdGVkIFN0YXRlcw&gws_rd=cr",
  206. 'wc':"https://www.google.com/search?q=angelo%20koo&hl=en&gl=us&num=100&uule=w+CAIQICItV2FzaGluZ3RvbixEaXN0cmljdCBvZiBDb2x1bWJpYSxVbml0ZWQgU3RhdGVz&gws_rd=cr#gws_rd=cr&ip=1",
  207. 'miami':"https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIbTWlhbWksRmxvcmlkYSxVbml0ZWQgU3RhdGVz&gws_rd=cr",
  208. 'chicago':"https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIeQ2hpY2FnbyxJbGxpbm9pcyxVbml0ZWQgU3RhdGVz&gws_rd=cr"}
  209. location = 'chicago'
  210. run_once({'kw':'angelo koo','fname':location,'date':'0229','url':d[location]})
  211. ####手動截圖:須按右下角的設定選擇區域######
  212. ny="https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIWTmV3IFlvcmssVW5pdGVkIFN0YXRlcw&gws_rd=cr"
  213. wc="https://www.google.com/search?q=angelo%20koo&hl=en&gl=us&num=100&uule=w+CAIQICItV2FzaGluZ3RvbixEaXN0cmljdCBvZiBDb2x1bWJpYSxVbml0ZWQgU3RhdGVz&gws_rd=cr#gws_rd=cr&ip=1"
  214. miami="https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIbTWlhbWksRmxvcmlkYSxVbml0ZWQgU3RhdGVz&gws_rd=cr"
  215. chicago="https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIeQ2hpY2FnbyxJbGxpbm9pcyxVbml0ZWQgU3RhdGVz&gws_rd=cr"