|
|
@@ -238,9 +238,13 @@ def parsing_js(orig):
|
|
|
|
|
|
def save_js_to_db(jsobj,num,keyword):
|
|
|
global store_list_table
|
|
|
+ global iddict
|
|
|
for r in jsobj:
|
|
|
+ if iddict.get(r['place_id']) is not None:
|
|
|
+ continue
|
|
|
r['num']=num
|
|
|
r['keyword']=keyword
|
|
|
+
|
|
|
try:
|
|
|
store_list_table.insert(r)
|
|
|
|
|
|
@@ -250,7 +254,6 @@ def save_js_to_db(jsobj,num,keyword):
|
|
|
# store_list_table.upsert(r,keys=['place_id'])
|
|
|
|
|
|
def process_web_request(driver,area_num,keyword):
|
|
|
- global iddict
|
|
|
# query = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.XPATH, '//button[@vet="19128"]')))
|
|
|
time.sleep(0.8)
|
|
|
time.sleep(3)
|
|
|
@@ -268,8 +271,6 @@ def process_web_request(driver,area_num,keyword):
|
|
|
jstext=resp.decode('utf-8')
|
|
|
resultobj=parsing_js(jstext)
|
|
|
print("before",datetime.now())
|
|
|
- if iddict.get(resultobj['place_id']) is not None:
|
|
|
- continue
|
|
|
save_js_to_db(resultobj,area_num,keyword)
|
|
|
print("after",datetime.now())
|
|
|
|
jared