# import redis
import time
import traceback
# import json
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import time
import os
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import dataset
from selenium.webdriver.common.keys import Keys
import json
import random
import time
import redis
import sys
import codecs
import pandas as pd
import random
import os
import time
import datetime
from selenium.webdriver.chrome.service import Service
import dataset
import pymysql
pymysql.install_as_MySQLdb()
from userAgentRandomizer import userAgents
import requests
driver = None
db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')
headers = {
"Authorization": "Bearer " + "6SDULL1Ebklduc6TFxa97AFto5Sj21kyJ30CxiLiSoi",
"Content-Type": "application/x-www-form-urlencoded"
}
def send_msg(kw):
params = {"message": kw}
r = requests.post("https://notify-api.line.me/api/notify", headers=headers, params=params)
def re_get_webdriver():
global port
global driver
result = []
if driver is not None:
print('closing....')
driver.quit()
os.system('killall chrome')
print('quit....')
driver = None
try:
ua = userAgents()
user_agent = ua.random()
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
# options.add_argument("--headless")
print(user_agent)
options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
options.add_argument("--incognito")
driver = None
try:
if os.name == 'nt':
driver = webdriver.Chrome(options=options)
else:
driver = webdriver.Chrome(executable_path=r'C:\Users\Administrator\Downloads\chromedriver_108\chromedriver', options=options)
except:
traceback.print_exc()
return
driver.delete_all_cookies()
driver.set_window_size(950, 20000)
return
except:
traceback.print_exc()
driver = None
return None
def scrolling(driver, pgnum):
ub = driver.find_element("css selector",'body')
for i in range(pgnum):
ub.send_keys(Keys.PAGE_DOWN)
if pgnum > 1:
time.sleep(0.3)
def run_once(jsobj):
table = db['google_rank']
date = jsobj['date']
print(jsobj)
global driver
i = 100
if driver is None:
time.sleep(8)
re_get_webdriver()
if driver is None:
return
try:
kw = jsobj['kw']
fname = jsobj['fname']
url = jsobj['url']
# if jsobj.get('domain') is None:
# exclude = jsobj['exclude']
# domain = None
# else:
# domain = jsobj['domain']
# exclude = None
city_map = {'chicago': ['42.04866173771369', '-87.68260072643513'],
'miami': ['25.764458843530548', '-80.19787522585152'],
'wc': ['38.96071674051165', '-77.03155367248387'],
'ny': ['40.76774141099703', '-73.98439238945637']}
city = jsobj['fname']
print(city)
Map_coordinates = dict({
"latitude": float(city_map[f"{city}"][0]),
"longitude": float(city_map[f"{city}"][1]),
"accuracy": 100
})
# 芝加哥、邁阿密、紐約、華盛頓
# driver.execute_cdp_cmd("Emulation.setGeolocationOverride", Map_coordinates)
# driver.get('https://www.google.com?num=100&lr=lang_en')
driver.get(url)
# print(driver.current_url)
# elmt = driver.find_element(By.XPATH, "//input[@name='q']")
# time.sleep(1)
# elmt.send_keys(kw)
#
# elmt.send_keys(Keys.ENTER)
time.sleep(3)
scrolling(driver, 10)
time.sleep(20)
elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']/a")
numresults = len(elmts)
print('搜尋結果數量', numresults)
time.sleep(20)
# driver.save_screenshot('C:\/Users\/s1301\/Pictures\/Saved Pictures\/angelo koo\/' +date +fname + ".png")
if numresults == 0:
send_msg('stop working...')
sys.exit()
datadict = {'搜尋詞': [], '結果標題': [], '結果網址': [], '結果名次': [], '結果說明': []}
df = pd.DataFrame()
idx = 1
found = False
test_lst = []
txt_dict = {}
for elmt in elmts:
href = elmt.get_attribute('href')
txt = elmt.text
desc = None
try:
elmt2 = elmt.find_element(By.XPATH, "./../../..//div[@data-content-feature=1]")
desc = elmt2.text
except:
desc = None
# print(desc)
table.insert(
{'title': elmt.text, 'url': href, 'keyword': kw, 'dt': datetime.datetime.now(), 'ranking': idx,'description':fname})
datadict['搜尋詞'].append(kw)
datadict['結果標題'].append(txt)
datadict['結果網址'].append(href)
datadict['結果名次'].append(str(idx))
datadict['結果說明'].append(desc)
# if len(txt) > 10:
# if domain is not None:
# for d in domain:
# if d in href:
# print('found....')
# print('clicked....')
# print(href)
# print(txt)
# print("ranking", idx)
# found = True
# return
# else:
# ex = False
# for ee in exclude:
# if ee in href:
# ex = True
# if not ex:
# test_lst.append(elmt)
# txt_dict[elmt] = txt
idx += 1
# if exclude is not None:
# print('exclude')
# elmt = random.choice(test_lst[5:])
# print(elmt)
# print(txt_dict[elmt])
#
# webdriver.ActionChains(driver).move_to_element(elmt).perform()
# # elmt.click()
# webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
# time.sleep(5)
#
# if not found:
# table.insert(
# {'ranking': -1, 'kw': kw, 'results': numresults, 'url': '', 'title': '未收錄', 'descrption': desc})
df['搜尋詞'] = datadict['搜尋詞']
df['結果標題'] = datadict['結果標題']
df['結果網址'] = datadict['結果網址']
df['結果名次'] = datadict['結果名次']
df['結果說明'] = datadict['結果說明']
df.to_excel('C:\/Users\/s1301\/Pictures\/Saved Pictures\/angelo koo\/'+date+fname+".xls")
except:
print('exception')
traceback.print_exc()
# time.sleep(9999)
# driver.save_screenshot('c:/tmp/seo/'+kw+".png")
driver.quit()
sys.exit()
d = {'ny':"https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIWTmV3IFlvcmssVW5pdGVkIFN0YXRlcw&gws_rd=cr",
'wc':"https://www.google.com/search?q=angelo%20koo&hl=en&gl=us&num=100&uule=w+CAIQICItV2FzaGluZ3RvbixEaXN0cmljdCBvZiBDb2x1bWJpYSxVbml0ZWQgU3RhdGVz&gws_rd=cr#gws_rd=cr&ip=1",
'miami':"https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIbTWlhbWksRmxvcmlkYSxVbml0ZWQgU3RhdGVz&gws_rd=cr",
'chicago':"https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIeQ2hpY2FnbyxJbGxpbm9pcyxVbml0ZWQgU3RhdGVz&gws_rd=cr"}
location = 'chicago'
run_once({'kw':'angelo koo','fname':location,'date':'0216','url':d[location]})
####手動截圖:須按右下角的設定選擇區域######
ny="https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIWTmV3IFlvcmssVW5pdGVkIFN0YXRlcw&gws_rd=cr"
wc="https://www.google.com/search?q=angelo%20koo&hl=en&gl=us&num=100&uule=w+CAIQICItV2FzaGluZ3RvbixEaXN0cmljdCBvZiBDb2x1bWJpYSxVbml0ZWQgU3RhdGVz&gws_rd=cr#gws_rd=cr&ip=1"
miami="https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIbTWlhbWksRmxvcmlkYSxVbml0ZWQgU3RhdGVz&gws_rd=cr"
chicago="https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIeQ2hpY2FnbyxJbGxpbm9pcyxVbml0ZWQgU3RhdGVz&gws_rd=cr"