update

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+docs

api/main.py

@@ -1,8 +1,11 @@
-import pandas as pd
 from fastapi import FastAPI, File, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
 import dataset
-from functools import reduce
+import zipfile
+import pandas as pd
+import glob
+
+
 
 app = FastAPI()
 app.add_middleware(
@@ -14,21 +17,47 @@ app.add_middleware(
 )
 
 
-@app.post("/write/")
-async def writecsv(file: UploadFile = File(...)):
-    # read csv
-    repls = (' Stats ', '_'), (' at ', '_'), ('-', '_'), \
-            (' ', ''), ('.csv', '')
-    filename = file.filename
-    filename = reduce(lambda a, kv: a.replace(*kv), repls, filename)
+@app.post("/upload/")
+async def upload( title:str, file: UploadFile = File(...)):
+
+    file = file.file
+    target = title
+    with zipfile.ZipFile(file, "r") as zip_ref:
+        zip_ref.extractall(target)
+    await write(target, title)
+
+
+@app.post("/write")
+async def write(dir, title):
+
+    path = dir
+    all_files = glob.glob(path + "/*.csv")
+
+    li = []
 
-    data = file.file
+    for filename in all_files:
+        df = pd.read_csv(filename, index_col=None, header=0)
+        li.append(df)
+
+    # pre-clean data
+    frame = pd.concat(li, axis=0, ignore_index=True)
+    data_name = "./"+dir+"/"+title+".csv"
+    data = frame.to_csv(data_name)
+    await todb(data_name, title)
+    print("data", data_name)
+
+
+@app.post("/todb")
+async def todb(file, title):
+    # read csv
+    filename = title
+    data = file
     df = pd.read_csv(data, sep=",",
                      skiprows=0, na_values='NULL')
 
     # db connect
     db = dataset.connect(
-        'mysql://choozmo:pAssw0rd@db.ptt.cx:3306/keywordweb?charset=utf8mb4'
+        'mysql://root:pAssw0rd@localhost:3306/keywordweb?charset=utf8mb4'
     )
 
     sql = "CREATE TABLE IF NOT EXISTS " \
@@ -48,10 +77,10 @@ async def writecsv(file: UploadFile = File(...)):
     # write to db
     table = db[filename]
     lines = df.shape
-    i = 0
 
-    rows = lines[0]
 
+    rows = lines[0]
+    i = 0
     for i in range(rows):
         row1 = df.iloc[i]
         k = row1[0]
@@ -71,3 +100,5 @@ async def writecsv(file: UploadFile = File(...)):
         table.insert(dbdata)
 
     db.close()
+
+