kw_tools/urlshortener/main.py

import rpyc
import time
from fastapi import FastAPI
import sys
import os
import dataset
import requests
import datetime
import json
import ast
from fastapi.responses import JSONResponse
from fastapi.middleware.cors import CORSMiddleware
db = dataset.connect('sqlite:///similar.db')

app = FastAPI()

origins = [
    "http://cmm.ai",
    "https://cmm.ai",
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

@app.post("/seturl/{full_url}")
async def get_domain(full_url):
    url = "https://similarweb2.p.rapidapi.com/pageoverview"

    if 'http' not in domain_name:
        domain_name='http://'+domain_name
    domain_name = domain_name.replace('%3A',':')
    domain_name = domain_name.replace('%2F','/')
    print(domain_name)
    querystring = {"website":domain_name}

    headers = {
    "x-rapidapi-key": "6dd30886e0msh7aefc9a0a794398p1896f2jsn275c45475609",
        "x-rapidapi-host": "similarweb2.p.rapidapi.com"
        }

    response = requests.request("GET", url, headers=headers, params=querystring)
    print(response.text)
    js=json.loads(response.text)
    print(response.text)


    jsdict={'SiteName':js['name'],'Description':js['siteDescription'],'GlobalRank':js['globalRank'],'Title':js['name'],'Category':js['categoryRank']['taxonomy'],'CountryRank':js['countryRank']['rank']}


    url = "https://similarweb2.p.rapidapi.com/trafficoverview"

    querystring = {"website":domain_name}

    response = requests.request("GET", url, headers=headers, params=querystring)

    js2=json.loads(response.text)
    print(response.text)

    jsdict['EstimatedMonthlyVisits']=js2['monthlyVisitsEstimate']
#    desc=js['siteDescription']
#    ustr=desc.encode('utf-8').decode('utf-8')
#    print(ustr)
    return JSONResponse(content=jsdict)



@app.get("/domain/{domain_name}")
async def read_item(domain_name):
    cursor=db.query('select domain_name,json,dt from similar where domain_name="'+domain_name+'"')
    for c in cursor:
        print(c['json'])
        print(c['dt'])
#        jsdict=ast.literal_eval(c['json'])
        jsdict=json.loads(c['json'])
        return JSONResponse(content=jsdict)

    print(domain_name)
#    conn = rpyc.connect("localhost",12345)
    conn = rpyc.connect("192.168.192.119",12345)

    #domain_name
    jsdata=conn.root.get_url('https://data.similarweb.com/api/v1/data?domain='+domain_name)
    print(jsdata)
#    jsdict=str(jsdata)[1:-1]
#    jsdict=json.loads(str(jsdata))
    jsdict=ast.literal_eval(str(jsdata))
#    jsdict=str(jsdata)
    table=db['similar']
    jstr=json.dumps(jsdict)
    table.insert({'domain_name':domain_name,'json':jstr,'dt':datetime.datetime.now()})
    db.commit()
    return JSONResponse(content=jsdict)
#    print(jsdata)
#    return {"item_id": domain_name}