kw_tools/similar_web/main.py

import rpyc
import time
from fastapi import FastAPI
import sys
import os
import dataset
import requests
import datetime
import json
import ast
from fastapi.responses import JSONResponse, FileResponse
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel


app = FastAPI()

origins = [
    "http://www.googo.org",
    "http://www.googo.org:8080",
    "http://0.0.0.0:8080",
    "http://googo.org:8080",
    "http://googo.org",
    "http://139.162.121.30"
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

class q_req(BaseModel):
    domain_name: str


@app.get("/index")
async def index():
    return FileResponse('index.html')
@app.get("/echarts.min.js")
async def index():
    return FileResponse('echarts.min.js')
@app.get("/reset.css")
async def index():
    return FileResponse('reset.css')
@app.get("/main.css")
async def index():
    return FileResponse('main.css')


@app.post("/dm")
async def get_domain(req:q_req):
    url = "https://similarweb2.p.rapidapi.com/pageoverview"
    domain_name=req.domain_name
    if 'http' not in domain_name:
        domain_name='http://'+domain_name
    domain_name = domain_name.replace('%3A',':')
    domain_name = domain_name.replace('%2F','/')
    print(domain_name)
    querystring = {"website":domain_name}
    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
    
    statement = 'SELECT * FROM storage_similar_web where SiteName ="'+ req.domain_name+'"'
    jsdict = None
    for row in db.query(statement):
        jsdict = {'SiteName':row['SiteName'],'Description':row['Description'],'GlobalRank':row['GlobalRank']
        ,'Title':row['Title'],'Category':row['Category'],'CountryRank':row['CountryRank'],'EstimatedMonthlyVisits':eval(row['EstimatedMonthlyVisits'])
        ,'totalVisits':row['totalVisits']}
    print(jsdict)
    if jsdict==None:
        headers = {
            "x-rapidapi-key": "6dd30886e0msh7aefc9a0a794398p1896f2jsn275c45475609",
            "x-rapidapi-host": "similarweb2.p.rapidapi.com"
        }

        
        js=''
        for i in range(0,50):
            print('Try'+str(i)+'times')
            while True:
                try:
                    response = requests.request("GET", url, headers=headers, params=querystring)
                    js=json.loads(response.text)
                except:
                    continue
                break

        jsdict={'SiteName':js['name'],'Description':js['siteDescription'],'GlobalRank':js['globalRank'],'Title':js['name'],'Category':js['categoryRank']['taxonomy'],'CountryRank':js['countryRank']['rank']}


        url = "https://similarweb2.p.rapidapi.com/trafficoverview"

        querystring = {"website":domain_name}

        response = requests.request("GET", url, headers=headers, params=querystring)

        js2=json.loads(response.text)
        #print(response.text)
        jsdict['totalVisits'] = js2['engagement']['totalVisits']
        jsdict['EstimatedMonthlyVisits']=js2['monthlyVisitsEstimate']
        log_table = db['storage_similar_web']
        log_table.insert({'SiteName':jsdict['SiteName'],'Description':jsdict['Description'],'GlobalRank':jsdict['GlobalRank']
        ,'Title':jsdict['Title'],'Category':jsdict['Category'],'CountryRank':jsdict['CountryRank'],'EstimatedMonthlyVisits':jsdict['EstimatedMonthlyVisits']
        ,'totalVisits':jsdict['totalVisits']})

    return JSONResponse(content=jsdict)