|
|
@@ -99,7 +99,7 @@ def get_next_job(db):
|
|
|
result = {}
|
|
|
# result = db.query('select * from error_list2 where check_=0 ORDER BY RAND() limit 100')
|
|
|
|
|
|
- result = db.query('SELECT * FROM swire_store_list a WHERE not exists (select 1 from error_list2 tei where tei.fid = a.fid limit 1 ) ORDER BY RAND() limit 100')
|
|
|
+ result = db.query('SELECT * FROM swire_store_list a WHERE not exists (select 1 from error_list2 tei where tei.fid = a.fid limit 1 ) ORDER BY RAND() limit 30')
|
|
|
|
|
|
|
|
|
url_pd = pd.DataFrame([dict(i) for i in result])
|
|
|
@@ -635,7 +635,7 @@ def main():
|
|
|
error_table3 = db['error_list3']
|
|
|
error_table3.insert({'name':name,'keyword':keyword,'item_url':item_url,'crawler_date':datetime.today().strftime("%Y/%m/%d %H:%M")})
|
|
|
traceback.print_exc()
|
|
|
- sys.exit()
|
|
|
+# sys.exit()
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
