edit

swire_shop_review.py

@@ -155,7 +155,7 @@ def process_web_request(db, driver, fid):
                 jstext = resp.decode('utf-8')
                 result = parsing_js(jstext)
 
-                save_js_to_db(resultobj, fid)
+                save_js_to_db(result, fid)
                 time.sleep(1)
 
 
@@ -225,22 +225,26 @@ def main():
     job = get_next_job(db)
 
     for row, group in job.iterrows():
-        item_url = group['item_url']
-        reviews_cnt = group['reviews_cnt']
-        fid = group['fid']
-
-        print(reviews_cnt, item_url)
-        driver.get(item_url)
-        time.sleep(0.5)
-        shop_soup = BeautifulSoup(driver.page_source, 'html.parser')
-        tmp_value = shop_soup.find('span', {'jsaction':'pane.rating.moreReviews'})
-
-        if tmp_value:
-            get_reviews(driver, reviews_cnt)
-            process_web_request(db, driver, fid)
-            print(driver.current_url)
-
-        db['review_process'].insert({'fid':fid, 'dt':datetime.now()})
+        try:
+            item_url = group['item_url']
+            reviews_cnt = group['reviews_cnt']
+            fid = group['fid']
+
+            print(reviews_cnt, item_url)
+            driver.get(item_url)
+            time.sleep(0.5)
+            shop_soup = BeautifulSoup(driver.page_source, 'html.parser')
+            tmp_value = shop_soup.find('span', {'jsaction':'pane.rating.moreReviews'})
+
+            if tmp_value:
+                get_reviews(driver, reviews_cnt)
+                process_web_request(db, driver, fid)
+                print(driver.current_url)
+
+            db['review_process'].insert({'fid':fid, 'dt':datetime.now()})
+            
+        except:
+            traceback.print_exc()
 
 if __name__ == '__main__':
     main()