add asyncio

similar_web/main.py

@@ -51,42 +51,7 @@ def fake_traffic(jsdict):
         return jsdict
 
 def get_domain_data(raw_domain):
-    url = "https://similarweb2.p.rapidapi.com/pageoverview"
-    domain_name=raw_domain
-    if 'http' not in domain_name:
-        domain_name='http://'+domain_name
-    domain_name = domain_name.replace('%3A',':')
-    domain_name = domain_name.replace('%2F','/')
-    print(domain_name)
-    querystring = {"website":domain_name}
-    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
     
-    statement = 'SELECT * FROM storage_similar_web where SiteName ="'+ raw_domain+'"'
-    jsdict = None
-    for row in db.query(statement):
-        jsdict = {'SiteName':row['SiteName'],'Description':row['Description'],'GlobalRank':row['GlobalRank']
-        ,'Title':row['Title'],'Category':row['Category'],'CountryRank':row['CountryRank'],'EstimatedMonthlyVisits':eval(row['EstimatedMonthlyVisits'])
-        ,'totalVisits':row['totalVisits']}
-    if jsdict==None:
-        headers = {"x-rapidapi-key": "6dd30886e0msh7aefc9a0a794398p1896f2jsn275c45475609",
-            "x-rapidapi-host": "similarweb2.p.rapidapi.com"}
-        response = requests.request("GET", url, headers=headers, params=querystring)
-        js=json.loads(response.text)
-        jsdict={'SiteName':js['name'],'Description':js['siteDescription'],'GlobalRank':js['globalRank'],'Title':js['name'],'Category':js['categoryRank']['taxonomy'],'CountryRank':js['countryRank']['rank']}
-        url = "https://similarweb2.p.rapidapi.com/trafficoverview"
-        querystring = {"website":domain_name}
-
-        response = requests.request("GET", url, headers=headers, params=querystring)
-        js2=json.loads(response.text)
-        jsdict['totalVisits'] = js2['engagement']['totalVisits']
-        jsdict['EstimatedMonthlyVisits']=js2['monthlyVisitsEstimate']
-        log_table = db['storage_similar_web']
-        log_table.insert({'SiteName':jsdict['SiteName'],'Description':jsdict['Description'],'GlobalRank':jsdict['GlobalRank']
-        ,'Title':jsdict['Title'],'Category':jsdict['Category'],'CountryRank':jsdict['CountryRank'],'EstimatedMonthlyVisits':jsdict['EstimatedMonthlyVisits']
-        ,'totalVisits':jsdict['totalVisits']})
-
-    if 'hhh' in domain_name:   
-        jsdict = fake_traffic(jsdict)
     return jsdict
 
 def domain_filter(url_array):
@@ -122,13 +87,84 @@ async def index():
 @app.post("/kw_dm")
 async def get_domain_by_keyword(req:kw_req):
     ls = domain_filter(search(req.keyword,num_results=20))
-    jsdict = get_domain_data(ls[0])
+    raw_domain = ls[0]
+    url = "https://similarweb2.p.rapidapi.com/pageoverview"
+    domain_name=raw_domain
+    if 'http' not in domain_name:
+        domain_name='http://'+domain_name
+    domain_name = domain_name.replace('%3A',':')
+    domain_name = domain_name.replace('%2F','/')
+    print(domain_name)
+    querystring = {"website":domain_name}
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+    
+    statement = 'SELECT * FROM storage_similar_web where SiteName ="'+ raw_domain+'"'
+    jsdict = None
+    for row in db.query(statement):
+        jsdict = {'SiteName':row['SiteName'],'Description':row['Description'],'GlobalRank':row['GlobalRank']
+        ,'Title':row['Title'],'Category':row['Category'],'CountryRank':row['CountryRank'],'EstimatedMonthlyVisits':eval(row['EstimatedMonthlyVisits'])
+        ,'totalVisits':row['totalVisits']}
+    if jsdict==None:
+        headers = {"x-rapidapi-key": "6dd30886e0msh7aefc9a0a794398p1896f2jsn275c45475609",
+            "x-rapidapi-host": "similarweb2.p.rapidapi.com"}
+        response = requests.request("GET", url, headers=headers, params=querystring)
+        js=json.loads(response.text)
+        jsdict={'SiteName':js['name'],'Description':js['siteDescription'],'GlobalRank':js['globalRank'],'Title':js['name'],'Category':js['categoryRank']['taxonomy'],'CountryRank':js['countryRank']['rank']}
+        url = "https://similarweb2.p.rapidapi.com/trafficoverview"
+        querystring = {"website":domain_name}
+
+        response = requests.request("GET", url, headers=headers, params=querystring)
+        js2=json.loads(response.text)
+        jsdict['totalVisits'] = js2['engagement']['totalVisits']
+        jsdict['EstimatedMonthlyVisits']=js2['monthlyVisitsEstimate']
+        log_table = db['storage_similar_web']
+        log_table.insert({'SiteName':jsdict['SiteName'],'Description':jsdict['Description'],'GlobalRank':jsdict['GlobalRank']
+        ,'Title':jsdict['Title'],'Category':jsdict['Category'],'CountryRank':jsdict['CountryRank'],'EstimatedMonthlyVisits':jsdict['EstimatedMonthlyVisits']
+        ,'totalVisits':jsdict['totalVisits']})
+
+    if 'hhh' in domain_name:   
+        jsdict = fake_traffic(jsdict)
     return JSONResponse(content=jsdict)
     
 @app.post("/dm")
 async def get_domain_data(req:q_req):
-    jsdict = get_domain_data(req.domain_name)
-    print(jsdict)
+    raw_domain=req.domain_name
+    url = "https://similarweb2.p.rapidapi.com/pageoverview"
+    domain_name=raw_domain
+    if 'http' not in domain_name:
+        domain_name='http://'+domain_name
+    domain_name = domain_name.replace('%3A',':')
+    domain_name = domain_name.replace('%2F','/')
+    print(domain_name)
+    querystring = {"website":domain_name}
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+    
+    statement = 'SELECT * FROM storage_similar_web where SiteName ="'+ raw_domain+'"'
+    jsdict = None
+    for row in db.query(statement):
+        jsdict = {'SiteName':row['SiteName'],'Description':row['Description'],'GlobalRank':row['GlobalRank']
+        ,'Title':row['Title'],'Category':row['Category'],'CountryRank':row['CountryRank'],'EstimatedMonthlyVisits':eval(row['EstimatedMonthlyVisits'])
+        ,'totalVisits':row['totalVisits']}
+    if jsdict==None:
+        headers = {"x-rapidapi-key": "6dd30886e0msh7aefc9a0a794398p1896f2jsn275c45475609",
+            "x-rapidapi-host": "similarweb2.p.rapidapi.com"}
+        response = requests.request("GET", url, headers=headers, params=querystring)
+        js=json.loads(response.text)
+        jsdict={'SiteName':js['name'],'Description':js['siteDescription'],'GlobalRank':js['globalRank'],'Title':js['name'],'Category':js['categoryRank']['taxonomy'],'CountryRank':js['countryRank']['rank']}
+        url = "https://similarweb2.p.rapidapi.com/trafficoverview"
+        querystring = {"website":domain_name}
+
+        response = requests.request("GET", url, headers=headers, params=querystring)
+        js2=json.loads(response.text)
+        jsdict['totalVisits'] = js2['engagement']['totalVisits']
+        jsdict['EstimatedMonthlyVisits']=js2['monthlyVisitsEstimate']
+        log_table = db['storage_similar_web']
+        log_table.insert({'SiteName':jsdict['SiteName'],'Description':jsdict['Description'],'GlobalRank':jsdict['GlobalRank']
+        ,'Title':jsdict['Title'],'Category':jsdict['Category'],'CountryRank':jsdict['CountryRank'],'EstimatedMonthlyVisits':jsdict['EstimatedMonthlyVisits']
+        ,'totalVisits':jsdict['totalVisits']})
+
+    if 'hhh' in domain_name:   
+        jsdict = fake_traffic(jsdict)
     return JSONResponse(content=jsdict)