|
|
@@ -20,14 +20,21 @@ import time
|
|
|
import json
|
|
|
import re
|
|
|
import sys, os
|
|
|
+import socket
|
|
|
+chrome_window=False
|
|
|
|
|
|
def brower_start(port):
|
|
|
options = webdriver.ChromeOptions()
|
|
|
- browser = webdriver.Remote(
|
|
|
- #command_executor='http://192.53.174.202:4444/wd/hub',
|
|
|
- command_executor='http://127.0.0.1:'+str(port)+'/wd/hub',
|
|
|
- desired_capabilities=options.to_capabilities()
|
|
|
- )
|
|
|
+ if chrome_window:
|
|
|
+ browser = webdriver.Chrome(
|
|
|
+ desired_capabilities=options.to_capabilities()
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ browser = webdriver.Remote(
|
|
|
+ command_executor='http://127.0.0.1:'+str(port)+'/wd/hub',
|
|
|
+ desired_capabilities=options.to_capabilities()
|
|
|
+ )
|
|
|
+
|
|
|
return browser
|
|
|
|
|
|
|
|
|
@@ -157,6 +164,10 @@ def get_lon_lat_list(db, keyword):
|
|
|
|
|
|
|
|
|
def main():
|
|
|
+ global chrome_window
|
|
|
+ localip=socket.gethostbyname(socket.gethostname())
|
|
|
+ if localip=='192.168.1.108':
|
|
|
+ chrome_window=True
|
|
|
db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/google_poi?charset=utf8mb4')
|
|
|
table = db['shop_item_list3']
|
|
|
table2 = db['progress_list2']
|
|
|
@@ -182,6 +193,7 @@ def main():
|
|
|
for r in lst:
|
|
|
latitude = r['lat'] #緯度
|
|
|
longitude = r['lon'] #精度
|
|
|
+ area_num=r['num']
|
|
|
table2.upsert({'kw':keyword,'num':r['num']},['kw'])
|
|
|
|
|
|
url = 'https://www.google.com.tw/maps/@{},{},15z?hl=zh-TW'.format(latitude, longitude)
|
|
|
@@ -196,7 +208,7 @@ def main():
|
|
|
for item in url_list:
|
|
|
try:
|
|
|
table.insert({'name':item[1],'lon':longitude, 'lat':latitude, \
|
|
|
- 'keyword':keyword, 'item_url':item[0],'crawler_date':datetime.today().strftime("%Y/%m/%d %H:%M")})
|
|
|
+ 'keyword':keyword, 'item_url':item[0],'area_num':area_num,'crawler_date':datetime.today().strftime("%Y/%m/%d %H:%M")})
|
|
|
except:
|
|
|
duplicate += 1
|
|
|
print(len(url_list), duplicate)
|
jared