Merge remote-tracking branch 'origin/master' into front-dev

.env

@@ -1,4 +1,4 @@
-DOMAIN=localhost
+DOMAIN=cloud.choozmo.com:8080
 SERVER_ADDRESS=http://localhost
 # DOMAIN=local.dockertoolbox.tiangolo.com
 # DOMAIN=localhost.tiangolo.com
@@ -11,39 +11,20 @@ TRAEFIK_TAG=ai-anchor.com
 TRAEFIK_PUBLIC_TAG=traefik-public
 
 DOCKER_IMAGE_BACKEND=backend
-DOCKER_IMAGE_CELERYWORKER=celeryworker
+DOCKER_IMAGE_CELERYWORKER=gpuceleryworker
 DOCKER_IMAGE_FRONTEND=frontend
 
 # Backend
-BACKEND_CORS_ORIGINS=["https://cloud.choozmo:8080", 
-                      "http://cloud.choozmo.com:8080", 
-                      "https://cloud.choozmo.com", 
-                      "http://cloud.choozmo.com", 
-                      "http://local.ai-anchor.com:5173", 
-                      "http://local.ai-anchor.com:8080", 
-                      "http://localhost", 
-                      "http://localhost:4200", 
-                      "http://localhost:3000", 
-                      "http://localhost:5173", 
-                      "http://localhost:8080", 
-                      "https://localhost", 
-                      "https://localhost:4200", 
-                      "https://localhost:3000", 
-                      "https://localhost:8080", 
-                      "http://dev.ai-anchor.com:3000",
-                      "http://dev.ai-anchor.com:5173", 
-                      "http://dev.ai-anchor.com:8080", 
-                      "https://stag.ai-anchor.com", 
-                      "https://ai-anchor.com"]
+BACKEND_CORS_ORIGINS=["https://cloud.choozmo:8080", "http://cloud.choozmo.com:8080", "https://cloud.choozmo.com", "http://cloud.choozmo.com","http://172.105.219.42", "http://local.ai-anchor.com:5173", "http://local.ai-anchor.com:8080", "http://localhost", "http://localhost:4200", "http://localhost:3000", "http://localhost:5173", "http://localhost:8080", "https://localhost", "https://localhost:4200", "https://localhost:3000", "https://localhost:8080", "http://dev.ai-anchor.com:3000", "http://dev.ai-anchor.com:5173", "http://dev.ai-anchor.com:8080", "https://stag.ai-anchor.com", "https://ai-anchor.com", "http://local.dockertoolbox.tiangolo.com", "http://localhost.tiangolo.com"]
 PROJECT_NAME=AI anchor
 SECRET_KEY=1df1f2180c7b2550e76a8ccf5e67a76e5321d8c2d3fee4a725f8b80baf9a0c91
 FIRST_SUPERUSER=admin@ai-anchor.com
 FIRST_SUPERUSER_PASSWORD=password
 SMTP_TLS=True
 SMTP_PORT=587
-SMTP_HOST=
-SMTP_USER=
-SMTP_PASSWORD=
+SMTP_HOST=smtp.gmail.com
+SMTP_USER=verify@choozmo.com
+SMTP_PASSWORD=hlmaxzjnvpeaulhw
 EMAILS_FROM_EMAIL=info@ai-anchor.com
 
 USERS_OPEN_REGISTRATION=True
@@ -68,8 +49,8 @@ PGADMIN_DEFAULT_EMAIL=admin@ai-anchor.com
 PGADMIN_DEFAULT_PASSWORD=password
 
 # Initial data
-MEMBERSHIP_TYPES=["normal", "infinite"]
-PROGRESS_TYPES=["waiting", "processing", "completed"]
+MEMBERSHIP_STATUS=["normal", "infinite"]
+PROGRESS_STATE=["waiting", "processing", "completed"]
 
 # celery
 CELERY_ZIP_STORAGE=celery_storage/zips

backend/app/app/api/api_v1/endpoints/videos.py

@@ -14,6 +14,8 @@ from app.core.celery_app import celery_app
 from app.core.config import settings
 from pathlib import Path
 
+from app.core.celery_app import celery_app
+
 ZIP_STORAGE = Path("/app").joinpath(settings.BACKEND_ZIP_STORAGE)
 VIDEO_STORAGE = Path("/app").joinpath(settings.BACKEND_VIDEO_STORAGE)
 
@@ -64,6 +66,8 @@ def upload_plot(
         return {"error": str(e)}
     finally:
         upload_file.file.close()
+
+    celery_app.send_task("app.worker.make_video", args=[video.id, video.stored_file_name, current_user.id])
     return video
 
 @router.get("/{id}")

backend/app/app/core/celery_app.py

@@ -1,8 +1,7 @@
 from celery import Celery
 
-celery_app = Celery("worker", broker="amqp://guest@queue//")
+celery_app = Celery("worker", broker="redis://172.104.93.163:16379/0")
 
 
 
-celery_app.conf.task_routes = {"app.worker.test_celery": "main-queue", 
-                                  "app.worker.make_video": "main-queue"}
+celery_app.conf.task_routes = {"app.worker.make_video": "main-queue"}

backend/app/app/core/config.py

@@ -9,13 +9,13 @@ class Settings(BaseSettings):
     SECRET_KEY: str = secrets.token_urlsafe(32)
     # 60 minutes * 24 hours * 8 days = 8 days
     ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
-    SERVER_NAME: str 
-    SERVER_HOST: AnyHttpUrl 
+    SERVER_NAME: str  = "cloud.choozmo.com:8080"
+    SERVER_HOST: AnyHttpUrl  = "http://cloud.choozmo.com:8080"
     # BACKEND_CORS_ORIGINS is a JSON-formatted list of origins
     # e.g: '["http://localhost", "http://localhost:4200", "http://localhost:3000", \
     # "http://localhost:8080", "http://local.dockertoolbox.tiangolo.com"]'
-    BACKEND_CORS_ORIGINS: List[AnyHttpUrl] = ["*"]
-    '''
+    BACKEND_CORS_ORIGINS: List[AnyHttpUrl] = ["https://cloud.choozmo:8080", "http://cloud.choozmo.com:8080", "https://cloud.choozmo.com", "http://cloud.choozmo.com","http://172.105.219.42", "http://local.ai-anchor.com:5173", "http://local.ai-anchor.com:8080", "http://localhost", "http://localhost:4200", "http://localhost:3000", "http://localhost:5173", "http://localhost:8080", "https://localhost", "https://localhost:4200", "https://localhost:3000", "https://localhost:8080", "http://dev.ai-anchor.com:3000", "http://dev.ai-anchor.com:5173", "http://dev.ai-anchor.com:8080"]
+    
     @validator("BACKEND_CORS_ORIGINS", pre=True)
     def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str]:
         if isinstance(v, str) and not v.startswith("["):
@@ -23,21 +23,15 @@ class Settings(BaseSettings):
         elif isinstance(v, (list, str)):
             return v
         raise ValueError(v)
-    '''
+    
     PROJECT_NAME: str = "ai-anchor"
-    SENTRY_DSN: Optional[HttpUrl] = None
+    SENTRY_DSN: Optional[HttpUrl] = "http://SENTRY_DSN"
 
     @validator("SENTRY_DSN", pre=True)
     def sentry_dsn_can_be_blank(cls, v: str) -> Optional[str]:
         if len(v) == 0:
             return None
-        return v
-
-    POSTGRES_SERVER: str
-    POSTGRES_USER: str
-    POSTGRES_PASSWORD: str
-    POSTGRES_DB: str
-    SQLALCHEMY_DATABASE_URI: Optional[PostgresDsn] = None
+        return 
 
     '''
     @validator("SQLALCHEMY_DATABASE_URI", pre=True)
@@ -52,19 +46,6 @@ class Settings(BaseSettings):
             path=f"/{values.get('POSTGRES_DB') or ''}",
         )
     '''
-    '''
-    @validator("SQLALCHEMY_DATABASE_URI", pre=True)
-    def assemble_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any:
-        if isinstance(v, str):
-            return v
-        return PostgresDsn.build(
-            scheme="mysql",
-            user="choozmo",
-            password="pAssw0rd",
-            host="db.ptt.cx:3306",
-            path=f"/ai-anchor",
-        )
-    '''
 
     SMTP_TLS: bool = True
     SMTP_PORT: Optional[int] = None
@@ -93,20 +74,20 @@ class Settings(BaseSettings):
         )
 
     EMAIL_TEST_USER: EmailStr = "test@example.com"  # type: ignore
-    FIRST_SUPERUSER: EmailStr
-    FIRST_SUPERUSER_PASSWORD: str
+    FIRST_SUPERUSER: EmailStr = "admin@ai-anchor.com"
+    FIRST_SUPERUSER_PASSWORD: str = "password"
     USERS_OPEN_REGISTRATION: bool = False
 
-    MEMBERSHIP_TYPES: List[str]
-    PROGRESS_TYPES: List[str]
+    MEMBERSHIP_STATUS : List[str] = ['normal', 'infinite']
+    PROGRESS_STATE: List[str] = ['waiting', 'processing', 'completed']
 
-    SERVER_ADDRESS: AnyHttpUrl
+    SERVER_ADDRESS: AnyHttpUrl = "http://172.105.219.42:8080"
 
-    CELERY_ZIP_STORAGE: str
-    CELERY_VIDEO_STORAGE: str
+    CELERY_ZIP_STORAGE: str = "/celery_storage/zips"
+    CELERY_VIDEO_STORAGE: str = "/celery_storage/videos"
 
-    BACKEND_ZIP_STORAGE: str
-    BACKEND_VIDEO_STORAGE: str
+    BACKEND_ZIP_STORAGE: str = "/backend_storage/zips"
+    BACKEND_VIDEO_STORAGE: str = "/backend_storage/videos"
 
     class Config:
         case_sensitive = True

backend/app/app/db/init_db.py

@@ -26,16 +26,16 @@ def init_db(db: Session) -> None:
     engine = create_engine(f'mysql://{user}:{password}@{host}/{db_name}', pool_pre_ping=True)
     base.Base.metadata.create_all(bind=engine)
 
-    if settings.MEMBERSHIP_TYPES:
-      for TYPE in settings.MEMBERSHIP_TYPES:
-          if not db.query(Membership).filter(Membership.status == TYPE).first():
-                db.add(Membership(status=TYPE))
+    if settings.MEMBERSHIP_STATUS:
+      for STATUS in settings.MEMBERSHIP_STATUS:
+          if not db.query(Membership).filter(Membership.status == STATUS).first():
+                db.add(Membership(status=STATUS))
       db.commit()
 
-    if settings.PROGRESS_TYPES:
-        for TYPE in settings.PROGRESS_TYPES:
-            if not db.query(Progress).filter(Progress.state == TYPE).first():
-                db.add(Progress(state=TYPE))
+    if settings.PROGRESS_STATE:
+        for STATE in settings.PROGRESS_STATE:
+            if not db.query(Progress).filter(Progress.state == STATE).first():
+                db.add(Progress(state=STATE))
         db.commit()
     
     user = crud.user.get_by_email(db, email=settings.FIRST_SUPERUSER)
@@ -47,14 +47,3 @@ def init_db(db: Session) -> None:
         )
         user = crud.user.create(db, obj_in=user_in)  # noqa: F841
     
-    if settings.MEMBERSHIP_TYPES:
-        for TYPE in settings.MEMBERSHIP_TYPES:
-            if not db.query(Membership).filter(Membership.status == TYPE).first():
-                  db.add(Membership(status=TYPE))
-        db.commit()
-
-    if settings.PROGRESS_TYPES:
-        for TYPE in settings.PROGRESS_TYPES:
-            if not db.query(Progress).filter(Progress.state == TYPE).first():
-                db.add(Progress(state=TYPE))
-        db.commit()

backend/app/app/main.py

@@ -12,7 +12,7 @@ app = FastAPI(
 if settings.BACKEND_CORS_ORIGINS:
     app.add_middleware(
         CORSMiddleware,
-        allow_origins=[str(origin) for origin in settings.BACKEND_CORS_ORIGINS],
+        allow_origins=["*"],#str(origin) for origin in settings.BACKEND_CORS_ORIGINS],
         allow_credentials=True,
         allow_methods=["*"],
         allow_headers=["*"],

backend/app/app/worker.py

@@ -1,61 +1,34 @@
 from raven import Client
-
+import os
 from app.core.celery_app import celery_app
 from app.core.config import settings
 import requests
 from pathlib import Path
 from urllib.parse import urlparse, urljoin
 
-client_sentry = Client(settings.SENTRY_DSN)
-
-download_to_local_url = urljoin(settings.SERVER_ADDRESS, settings.API_V1_STR, "/videos/worker")
-upload_to_server_url = urljoin(settings.SERVER_ADDRESS, settings.API_V1_STR, "/videos/worker")
+#client_sentry = Client(settings.SENTRY_DSN)
 
-ZIP_STORAGE = Path("/app").joinpath(settings.CELERY_ZIP_STORAGE)
-VIDEO_STORAGE = Path("/app").joinpath(settings.CELERY_VIDEO_STORAGE)
+download_to_local_url = urljoin(settings.SERVER_HOST, settings.API_V1_STR+"/videos/worker")
+upload_to_server_url = urljoin(settings.SERVER_HOST, settings.API_V1_STR+"/videos/worker")
 
+ZIP_STORAGE = Path(settings.CELERY_ZIP_STORAGE) 
+VIDEO_STORAGE = Path(settings.CELERY_VIDEO_STORAGE)
 
-@celery_app.task(acks_late=True)
-def test_celery(word: str) -> str:
-    return f"test task return {word}"
-
-@celery_app.task(acks_late=True)
-def download_to_local(id:int, file_name:str) -> str:
-
-    zip_file = Path(file_name+"/.zip")
-    r = requests.get(download_to_local_url, stream=True)
-
-    with open(str(VIDEO_STORAGE/zip_file), 'wb') as f:
-        r.raise_for_status()
-        for chunk in r.iter_content(chunk_size=1024):
-            f.write(chunk)
-    return "complete"
 
 
 @celery_app.task(acks_late=True)
-def make_video(id:int, file_name) -> str:
+def make_video(video_id, zip_filename, user_id) -> str:
+    #video_id, zip_filename, user_id = args
     # download 
-    zip_file = Path(file_name+"/.zip")
     r = requests.get(download_to_local_url, stream=True)
-
-    with open(str(VIDEO_STORAGE/zip_file), 'wb') as f:
+    with open(str(VIDEO_STORAGE/zip_filename), 'wb') as f:
         r.raise_for_status()
         for chunk in r.iter_content(chunk_size=1024):
             f.write(chunk)
-
     # make video
 
 
-    video_file = Path(file_name+".mp4")
-    r = requests.post(urljoin(upload_to_server_url,str(id)))
-
-
+    
     return "complete"
 
 
-@celery_app.task(acks_late=True)
-def upload_to_server(id:int, file_name:str) -> str:
-
-    video_file = Path(file_name+".mp4")
-    r = requests.post(urljoin(upload_to_server_url,str(id)))
-    return "complete"

backend/app/pyproject.toml

@@ -14,6 +14,7 @@ python-multipart = "^0.0.5"
 email-validator = "^1.0.5"
 requests = "^2.23.0"
 celery = "^5.2.7"
+redis = "^4.5.0"
 passlib = {extras = ["bcrypt"], version = "^1.7.2"}
 tenacity = "^8.0.0"
 pydantic = "^1.10.4"

cloud-docker-compose.override.yml

@@ -4,7 +4,6 @@ services:
   proxy:
     ports:
       - "8080:80"
-      - "8090:8080"
     command:
       # Enable Docker in Traefik, so that it reads labels from Docker services
       - --providers.docker
@@ -27,12 +26,28 @@ services:
       - traefik.enable=true
       - traefik.http.routers.${STACK_NAME?Variable not set}-traefik-public-http.rule=Host(`${DOMAIN?Variable not set}`)
       - traefik.http.services.${STACK_NAME?Variable not set}-traefik-public.loadbalancer.server.port=80
-
-
-
-
-
-
+  
+  backend:
+    volumes:
+      - ./backend/app:/app
+      - /${BACKEND_ZIP_STORAGE}:/app/${BACKEND_ZIP_STORAGE}
+      - /${BACKEND_VIDEO_STORAGE}:/app/${BACKEND_VIDEO_STORAGE}
+    environment:
+      - SERVER_HOST=http://${DOMAIN?Variable not set}
+    build:
+      context: ./backend
+      dockerfile: backend.dockerfile
+      args:
+        INSTALL_DEV: ${INSTALL_DEV-true}
+        INSTALL_JUPYTER: ${INSTALL_JUPYTER-false}
+    # command: bash -c "while true; do sleep 1; done"  # Infinite loop to keep container live doing nothing
+    command: /start-reload.sh
+    labels:
+      - traefik.enable=true
+      - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
+      - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)
+      - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=80
+  
   frontend:
     build:
       context: ./frontend

cloud-docker-compose.yml

@@ -67,7 +67,27 @@ services:
         # Middleware to redirect www, and redirect HTTP to HTTPS
         # to disable www redirection remove the section: ${STACK_NAME?Variable not set}-www-redirect,
         - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.middlewares=${STACK_NAME?Variable not set}-www-redirect,${STACK_NAME?Variable not set}-https-redirect
-
+  
+  backend:
+    image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}'
+    env_file:
+      - .env
+    environment:
+      - SERVER_NAME=${DOMAIN?Variable not set}
+      - SERVER_HOST=https://${DOMAIN?Variable not set}
+      # Allow explicit env var override for tests
+      - SMTP_HOST=${SMTP_HOST}
+    build:
+      context: ./backend
+      dockerfile: backend.dockerfile
+      args:
+        INSTALL_DEV: ${INSTALL_DEV-false}
+    deploy:
+      labels:
+        - traefik.enable=true
+        - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
+        - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)
+        - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=80
   
   frontend:
     image: '${DOCKER_IMAGE_FRONTEND?Variable not set}:${TAG-latest}'

frontend/.env

@@ -1,9 +1,9 @@
-VITE_APP_DOMAIN_DEV=172.104.93.163:10000
+VITE_APP_DOMAIN_DEV=cloud.choozmo.com:8080
 # VUE_APP_DOMAIN_DEV=local.dockertoolbox.tiangolo.com
 # VUE_APP_DOMAIN_DEV=localhost.tiangolo.com
 # VUE_APP_DOMAIN_DEV=dev.ai-anchor.com
 VITE_APP_DOMAIN_STAG=stag.ai-anchor.com
-VITE_APP_DOMAIN_PROD=ai-anchor.com
+VITE_APP_DOMAIN_PROD=cloud.choozmo.com
 VITE_APP_NAME=AI anchor
 VITE_APP_ENV=development
 # VUE_APP_ENV=staging