# -*- coding: utf-8 -*- #from selenium import webdriver from seleniumwire import webdriver from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.common.by import By import selenium import traceback from bs4 import BeautifulSoup from utility import database_access as DA from utility.parseutils import * from utility.connect import * from datetime import datetime import pandas as pd import dataset import time import json import re import sys, os import socket import brotli chrome_window=False def brower_start(port): options = webdriver.ChromeOptions() if chrome_window: browser = webdriver.Chrome( desired_capabilities=options.to_capabilities() ) else: browser = webdriver.Remote( command_executor='http://127.0.0.1:'+str(port)+'/wd/hub', desired_capabilities=options.to_capabilities() ) return browser def page_down_(driver, xpath_css, time_): e = driver.find_element_by_css_selector('span[class="Jl2AFb"]') result_count = e.text.split('-')[1].replace(' 項結果','') print(result_count) if int(result_count) > 5: for i in range(time_): e = driver.find_elements_by_css_selector('div[class="TFQHme"]') action = webdriver.common.action_chains.ActionChains(driver) action.move_to_element_with_offset(e[-1], e[-1].size['width'] + 1 , 0) action.click() action.perform() time.sleep(0.5) # elmts = driver.find_elements_by_xpath(xpath_css) # print(elmts) # if len(elmts)>1: # elmt=elmts[1] # else: # elmt=elmts[0] # actions = ActionChains(driver) # actions.move_to_element(elmt).click().perform() # for i in range(time_): # try: # actions = ActionChains(driver) # actions.send_keys(Keys.PAGE_DOWN).perform() # except: # traceback.print_exc() # time.sleep(0.5) def get_url_list(driver): # for i in range(5, 43, 2): # try: # wait = WebDriverWait(driver, 60) # wait.until( # EC.element_to_be_clickable((By.XPATH, '//*[@id="pane"]/div/div[1]/div/div/div[2]/div[1]/div[{}]/div/a'.format(i))) # ) # driver.find_element(By.XPATH,'//*[@id="pane"]/div/div[1]/div/div/div[2]/div[1]/div[{}]/div/a'.format(i)).send_keys(Keys.DOWN) # time.sleep(0.5) # except: # pass # wait = WebDriverWait(driver, 30) # try:地圖移動時更新結果 # wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="ppdPk-Ej1Yeb-LgbsSe-tJiF1e"]'))) # except selenium.common.exceptions.TimeoutException: # traceback.print_exc() # return "EMPTY" page_down_(driver, '//div[@class="TFQHme"]', 8) url_soup = BeautifulSoup(driver.page_source, 'html.parser') url_list = [] for i in url_soup.find_all('a'): try: if i['href'].find('maps/place') != -1: url_list += [[i['href'], i['aria-label']]] except: pass # print(len(url_list)) return url_list def keyin_keyword(driver, keyword): button = driver.find_element_by_id("searchbox") driver.implicitly_wait(30) ActionChains(driver).move_to_element(button).send_keys(keyword).send_keys(Keys.RETURN).perform() time.sleep(3) # def get_crawler_list(db): # result = db.query('select keyword, count(*) from shop_item_list group by keyword') # result = pd.DataFrame([i for i in result]) # result.columns = ['keyword', 'count'] # result = result[result['count'] < 100] # keyword = result.sample(1).iloc[0]['keyword'] # num=0 # cursor=db.query('select num from progress_list2 where kw = "'+keyword+'"') # for c in cursor: # num=c['num'] # break # cursor=db.query('select * from lat_lon_loc where num >= '+str(num)) # # cursor=db.query('select * from lat_lon_loc') # lst=[] # for c in cursor: # lst.append({'num':c['num'],'loc':c['loc'],'lat':c['lat'],'lon':c['lon']}) # return keyword, lst def get_crawler_list(db): # result = db.query('select * from shop_item_list order by keyword') # result = pd.DataFrame([i for i in result]) # result = result[~result.keyword.str.contains('項')] # progress = db.query('select distinct(kw) from progress_list2 where num < 367') # progress = pd.DataFrame([i for i in progress]) # if len(progress) != 0: # keyword = result[~result['keyword'].isin(progress.kw.to_list())].iloc[0]['keyword'] # else: # keyword = result.iloc[0]['keyword'] # # return keyword return '滷味' cursor = db.query('select distinct(kw) from progress_list2 where num < 367 order by num asc limit 1') for c in cursor: return c['kw'] return None def get_lon_lat_list(db, keyword): num=0 cursor=db.query('select num from progress_list2 where kw = "'+keyword+'"') for c in cursor: num=c['num'] break cursor=db.query('select * from lat_lon_loc where num >= '+str(num)) lst=[] for c in cursor: lst.append({'num':c['num'],'loc':c['loc'],'lat':c['lat'],'lon':c['lon']}) return lst def parsing_js(orig): content="" lines=orig.split('\n') for l in lines: newl=l.replace('\\"','"') content+=newl result=re.search(r'\[\["',content) content_begin=result.start() result=re.search(r'\]\]"',content) content_end=result.end() jscontent=content[content_begin:content_end-1] jsobj=json.loads(jscontent) print() for x in jsobj[0][1][1:]: print(x[14][11]) print(x[14][10]) print(x[14][2]) print(x[14][78]) def main(): global chrome_window localip=socket.gethostbyname(socket.gethostname()) if localip=='192.168.1.108': chrome_window=True db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/google_poi?charset=utf8mb4') table = db['shop_item_list3'] table2 = db['progress_list2'] port=4447 if len(sys.argv) > 1 : port=int(sys.argv[1]) print('restart docker p{}'.format(port)) os.system('sudo docker container restart p'+str(port)) time.sleep(8) print('drvier start...') driver = brower_start(port) for i in range(10): try: keyword = get_crawler_list(db) print(keyword) lst = get_lon_lat_list(db, keyword) # print(lst) print(keyword, len(lst)) for r in lst: latitude = r['lat'] #緯度 longitude = r['lon'] #精度 area_num=r['num'] table2.upsert({'kw':keyword,'num':r['num']},['kw']) url = 'https://www.google.com.tw/maps/@{},{},15z?hl=zh-TW'.format(latitude, longitude) driver.get(url) keyin_keyword(driver, keyword) failcnt = 0 time.sleep(10) for request in driver.requests: if request.response: if 'https://www.google.com.tw/search?tbm=map' in request.url : print( request.url, request.response.status_code, request.response.headers['Content-Type'] ) print('parsing js:') resp = brotli.decompress(request.response.body) jstext=resp.decode('utf-8') parsing_js(jstext) # import codecs # fw=codecs.open('c:/tmp/ot.json','w','utf-8') # fw.write(jstext) # fw.close() # print(jstext) # time.sleep(9999) # jsobj=json.loads(jstext) # print(jsobj) # sys.exit() for page in range(10): print(keyword, latitude, longitude, page) url_list = get_url_list(driver) duplicate = 0 # shop_item_list_col = ['name','lon','lat','keyword','item_url','crawler_date'] for item in url_list: try: table.insert({'name':item[1],'lon':longitude, 'lat':latitude, \ 'keyword':keyword, 'item_url':item[0],'area_num':area_num,'crawler_date':datetime.today().strftime("%Y/%m/%d %H:%M")}) except: duplicate += 1 print(len(url_list), duplicate) # result = [item[1], longitude, latitude, keyword, item[0], datetime.today().strftime("%Y/%m/%d %H:%M")] # insert_sql = """INSERT IGNORE INTO {}{} VALUES {}"""\ # .format('shop_item_list', str(tuple(shop_item_list_col)).replace('\'',''), tuple(result)) # DA.mysql_insert_data(db, insert_sql) if page < 2 : element = driver.find_element_by_id('ppdPk-Ej1Yeb-LgbsSe-tJiF1e') if element.get_attribute('disabled'): break driver.implicitly_wait(30) ActionChains(driver).move_to_element(element).click(element).perform() except: pass if __name__ == '__main__': main()