Home Explore Help
Register Sign In
choozmo
/
kw_tools
2
0
Fork 1
Files Issues 0 Pull Requests 0 Wiki
Browse Source

w

Your Name 4 years ago
parent
299432396c
commit
e6fa497a15
4 changed files with 78 additions and 9 deletions
Split View Show Diff Stats
  1. 14 5
      similar_server/clienttest.py
  2. 36 0
      similar_server/main.py
  3. 9 2
      similar_server/sel_server.py
  4. 19 2
      similar_web/index.html

+ 14 - 5
similar_server/clienttest.py
View File 

@@ -1,9 +1,18 @@
 
 import rpyc
 
 import time
 
-conn = rpyc.connect("localhost",12345)
 
-conn.root.get_url('https://yahoo.com')
 
-time.sleep(10)
 
-conn.root.get_url('https://google.com')
 
+from fastapi import FastAPI
 
+import sys
 
+import os
 
+
 
+app = FastAPI()
 
+
 
+@app.get("/domain/{domain_name}")
 
+async def read_item(domain_name):
 
+    conn = rpyc.connect("localhost",12345)
 
+    jsdata=conn.root.get_url('http://headers.jsontest.com/')
 
+    return jsdata
 
+#    print(jsdata)
 
+#    return {"item_id": domain_name}
 
+
 
 
 
-#https://data.similarweb.com/api/v1/data?domain=cnn.com

+ 36 - 0
similar_server/main.py
View File 

@@ -0,0 +1,36 @@
 
+import rpyc
 
+import time
 
+from fastapi import FastAPI
 
+import sys
 
+import os
 
+import dataset
 
+import datetime
 
+import json
 
+
 
+db = dataset.connect('sqlite:///similar.db')
 
+
 
+app = FastAPI()
 
+
 
+@app.get("/domain/{domain_name}")
 
+async def read_item(domain_name):
 
+    cursor=db.query('select domain_name,json,dt from similar where domain_name="'+domain_name+'"')
 
+    for c in cursor:
 
+        print(c['json'])
 
+        print(c['dt'])
 
+        return c['json'][1:-1]
 
+
 
+    print(domain_name)
 
+    conn = rpyc.connect("localhost",12345)
 
+    #domain_name
 
+    jsdata=conn.root.get_url('https://data.similarweb.com/api/v1/data?domain='+domain_name)
 
+    print(jsdata)
 
+    jsdict=str(jsdata)[1:-1]
 
+    table=db['similar']
 
+    table.insert({'domain_name':domain_name,'json':jsdict,'dt':datetime.datetime.now()})
 
+    db.commit()
 
+    return jsdata
 
+#    print(jsdata)
 
+#    return {"item_id": domain_name}
 
+
 
+
 
+

+ 9 - 2
similar_server/sel_server.py
View File 

@@ -8,9 +8,10 @@ from selenium.webdriver.support import expected_conditions as EC
 
 from selenium.webdriver.common.keys import Keys
 
 from selenium.webdriver.remote.webdriver import WebDriver
 
 import time
 
-import json
 
 import rpyc
 
 from rpyc.utils.server import ThreadedServer # or ForkingServer
 
+from bs4 import BeautifulSoup
 
+import json
 
 
 
 
@@ -56,7 +57,13 @@ class MyService(rpyc.Service):
 
         print('add url.............')
 
         print(self.driver)
 
         self.driver.get(url)
 
-
 
+        
+#        txt=self.driver.find_element_by_xpath("//div[@id='json']").text
 
+        print(self.driver.page_source)
 
+        soup = BeautifulSoup(self.driver.page_source,features="lxml")
 
+        dict_from_json = json.loads(soup.find("body").text)
 
+        return dict_from_json
 
+#        return json.loads(self.driver.page_source)

+ 19 - 2
similar_web/index.html
View File 

@@ -20,6 +20,7 @@
 
   type="text/javascript"
 
   src="https://cdnjs.cloudflare.com/ajax/libs/mdb-ui-kit/3.3.0/mdb.min.js"
 
 ></script>    
+<script src="https://unpkg.com/axios/dist/axios.min.js"></script>
 
 </head>
 
 <body>
 
 
@@ -131,8 +132,8 @@
 
     <div class="col-sm-11 col-lg-11">
 
 
         <!-- Search form -->
 
-        <form class="d-flex input-group w-auto">
 
-          <input type="search" class="form-control" placeholder="Search" aria-label="Domain Name" />
 
+        <form id="searchform" class="d-flex input-group w-auto" onsubmit="check_form();">
 
+          <input type="search" class="form-control" placeholder="Search" aria-label="Domain Name" id="search_query"/>
 
           <button class="btn btn-primary" type="button" data-mdb-ripple-color="dark">
 
             Search
 
           </button>
 
@@ -145,6 +146,22 @@
 
 
 </div>
 
 
+
 
+<script>
 
+  function check_form(){
 
+    var qry=document.getElementById('search_query').value;
 
+    axios.get('http://127.0.0.1:8080/domain/w3.org')
 
+  .then((response) => {
 
+    console.log(response.data);
 
+    console.log(response.status);
 
+    console.log(response.statusText);
 
+    console.log(response.headers);
 
+    console.log(response.config);
 
+  });
 
+
 
+  }
 
+</script>
 
+
 
 </body>