modified backend

README.md

@@ -1,3 +1,95 @@
+RUN ln -s /usr/lib/python3/dist-packages/apt_pkg.cpython-36m-x86_64-linux-gnu.so 
+RUN ln -s /usr/lib/python3/dist-packages/gi/_gi.cpython-36m-x86_64-linux-gnu.so
+RUN apt-get install -y software-properties-common
+
+#RUN python3.6 /usr/bin/apt-add-repository ppa:mosquitto-dev/mosquitto-ppa
+RUN git clone https://github.com/OpenShot/libopenshot.git
+RUN git clone https://github.com/OpenShot/libopenshot-audio.git
+RUN python3.6 /usr/bin/add-apt-repository ppa:openshot.developers/libopenshot-daily
+
+RUN apt-get install -y \
+    cmake \
+    pkg-config \
+    libopenshot-audio-dev \
+    libx11-dev \
+    libfreetype6-dev \
+    libasound2-dev \
+    libavcodec-dev \
+    libavformat-dev \
+    libavutil-dev \
+    libswresample-dev \
+    libswscale-dev \
+    libpostproc-dev \
+    libfdk-aac-dev \
+    libjsoncpp-dev \
+    libzmq3-dev \
+    qtbase5-dev \
+    libqt5svg5-dev \
+    libbabl-dev \
+    libopencv-dev \
+    libprotobuf-dev \
+    protobuf-compiler \
+    python3-dev \
+    swig \
+    libmagick++-dev
+
+RUN ls
+RUN cd libopenshot-audio
+WORKDIR /app/libopenshot-audio
+RUN cmake -B build -S .
+RUN cmake --build build
+RUN cmake --install build
+
+RUN cd ../libopenshot
+WORKDIR /app/libopenshot
+RUN cmake -B build -S .
+RUN cmake --build build
+
+RUN cd ..
+WORKDIR /app/
+
+# Install Poetry
+RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python3 && \
+    cd /usr/local/bin && \
+    ln -s /opt/poetry/bin/poetry && \
+    poetry config virtualenvs.create false
+
+# Copy poetry.lock* in case it doesn't exist in the repo
+COPY ./app/gpuproject.toml /app/pyproject.toml
+# COPY ./app/poetry.lock* /app/
+
+# Allow installing dev dependencies to run tests
+ARG INSTALL_DEV=false
+RUN bash -c "if [ $INSTALL_DEV == 'true' ] ; then poetry install --no-root ; else poetry install --no-root --no-dev ; fi"
+
+# For development, Jupyter remote kernel, Hydrogen
+# Using inside the container:
+# jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.custom_display_url=http://127.0.0.1:8888
+ARG INSTALL_JUPYTER=false
+RUN bash -c "if [ $INSTALL_JUPYTER == 'true' ] ; then pip install jupyterlab ; fi"
+
+ENV C_FORCE_ROOT=1
+
+COPY ./app /app
+WORKDIR /app
+COPY /${CELERY_ZIP_STORAGE} /app/${CELERY_ZIP_STORAGE}
+COPY /${CELERY_ZIP_STORAGE} /app/${CELERY_ZIP_STORAGE}
+
+RUN apt-get install -y python3.10
+#ENV PYTHONPATH=/app
+
+
+RUN python --version
+RUN python3 --version
+RUN echo ${PYTHONPATH}
+
+
+COPY ./app/worker-start.sh /worker-start.sh
+
+RUN chmod +x /worker-start.sh
+
+CMD ["bash", "/worker-start.sh"]
+
 # AI anchor
 
 ## Backend Requirements

backend/app/app/api/api_v1/endpoints/videos.py

@@ -1,5 +1,5 @@
 from typing import Any, List, Optional
-
+import subprocess
 from fastapi import UploadFile, File, Form
 from fastapi.responses import FileResponse
 from fastapi import APIRouter, Depends, HTTPException
@@ -66,7 +66,9 @@ def upload_plot(
         return {"error": str(e)}
     finally:
         upload_file.file.close()
-
+    zip_filename = video.stored_file_name+".zip"
+    print(str(ZIP_STORAGE/zip_filename))
+    subprocess.run(f'sshpass -p "choozmo9" scp {str(ZIP_STORAGE/zip_filename)}  root@172.104.93.163:{str(ZIP_STORAGE/zip_filename)}')
     celery_app.send_task("app.worker.make_video", args=[video.id, video.stored_file_name, current_user.id])
     return video
 
@@ -93,40 +95,3 @@ def download_video(
     else:
         raise HTTPException(status_code=404, detail="Error occurs")
 
-
-@router.get("/worker/{id}")
-def download_plot(
-  *,
-  db: Session = Depends(deps.get_db),
-  id: int
-) -> Any:
-    video = db.query(models.Video).filter(db=db, id=id).first()
-    if not video:
-        raise HTTPException(status_code=404, detail="Video not found")
-    
-    file_path = Path(ZIP_STORAGE).joinpath(video.stored_file_name+".zip")
-    return FileResponse(path=str(file_path))
-
-@router.post("/worker/{id}")
-def upload_complete_video(
-    *,
-    db: Session = Depends(deps.get_db),
-    id: int,
-    upload_video: Optional[UploadFile]=None,
-    result:int=Form(...)
-) -> Any:
-    
-    video = db.query(models.Video).filter(db=db, id=id).first()
-    if not video:
-        raise HTTPException(status_code=404, detail="Video not found")
-    
-    try:
-        with open(str(Path(VIDEO_STORAGE).joinpath(video.stored_file_name+".mp4")), 'wb') as f:
-            while contents := upload_video.file.read(1024 * 1024):
-                f.write(contents)
-    except Exception as e:
-        print(e, type(e))
-        return {"error": str(e)}
-    finally:
-        upload_video.file.close()
-    return video

backend/app/app/worker.py

@@ -5,7 +5,7 @@ from app.core.config import settings
 import requests
 from pathlib import Path
 from urllib.parse import urlparse, urljoin
-
+import os
 #client_sentry = Client(settings.SENTRY_DSN)
 
 download_to_local_url = urljoin(settings.SERVER_HOST, settings.API_V1_STR+"/videos/worker")
@@ -20,11 +20,14 @@ VIDEO_STORAGE = Path(settings.CELERY_VIDEO_STORAGE)
 def make_video(video_id, zip_filename, user_id) -> str:
     #video_id, zip_filename, user_id = args
     # download 
+    '''
     r = requests.get(download_to_local_url, stream=True)
     with open(str(VIDEO_STORAGE/zip_filename), 'wb') as f:
         r.raise_for_status()
         for chunk in r.iter_content(chunk_size=1024):
             f.write(chunk)
+    '''
+    print((ZIP_STORAGE/zip_filename).exists())
     # make video
 
 

backend/app/gpuproject.toml

@@ -0,0 +1,39 @@
+[tool.poetry]
+name = "app"
+version = "0.1.0"
+description = ""
+authors = ["tomoya <tomoya@choozmo.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.8.0"
+Pillow = "^9.4.0"
+pydantic = "^1.10.4"
+mysql-connector-python = "^8.0.32"
+mysqlclient = "^2.1.1"
+sqlalchemy = "^1.4.0"
+opencv-python = "^4.7.0"
+requests = "^2.23.0"
+celery = "^5.2.7"
+tenacity = "^8.0.0"
+pytest = "^7.2.0"
+torch = "^1.13.0"
+torchvision = "^0.14.0"
+torchaudio = "^0.13.0"
+
+[tool.poetry.dev-dependencies]
+mypy = "^0.991"
+black = "^19.10b0"
+isort = "^5.11.0"
+autoflake = "^1.3.1"
+pytest = "^7.2.0"
+pytest-cov = "^4.0.0"
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+line_length = 88
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"
+

backend/app/worker-start.sh

@@ -1,16 +1,9 @@
 #! /usr/bin/env bash
+export PYTHONPATH=$PYTHONPATH:/usr/lib/python3.8/site-packages/:/app
+
 set -e
 
-python /app/app/celeryworker_pre_start.py
+#python /app/app/celeryworker_pre_start.py
 
 celery -A app.worker worker -l info -Q main-queue -c 1
 
-celery -A app.wo
-
-if [ ! -e ZIP_ZIP_STORAGE ]; then
-    mkdir ZIP_ZIP_STORAGE
-fi
-
-if [ ! -e ZIP_VIDEOS_STORAGE ]; then
-    mkdir ZIP_VIDEOS_STORAGE
-fi

backend/backend.dockerfile

@@ -2,6 +2,9 @@ FROM tiangolo/uvicorn-gunicorn:python3.10
 
 WORKDIR /app/
 
+RUN atp update
+RUN atp-get install -y sshpass
+
 # Install Poetry
 RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python3 && \
     cd /usr/local/bin && \

backend/gpuceleryworker.dockerfile

@@ -0,0 +1,162 @@
+FROM nvidia/cuda:11.7.0-cudnn8-runtime-ubuntu20.04
+
+# ensure local python is preferred over distribution python
+ENV PATH /usr/local/bin:$PATH
+
+# http://bugs.python.org/issue19846
+# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
+ENV LANG C.UTF-8
+
+RUN apt-get update
+
+# runtime dependencies
+RUN set -eux; \
+  	apt-get install -y  \
+		ca-certificates \
+		tzdata \
+	;
+
+ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D
+ENV PYTHON_VERSION 3.10.10
+
+RUN apt-get update && \
+    set -eux; \
+	  \
+	  apt-get install -y \
+    build-essential\
+    wget \
+    curl \
+		tar \
+    xz-utils\
+		make \
+    cmake \
+    zlib1g-dev \
+    libssl-dev \
+    libssl1.1 || sudo apt install libssl1.0 \
+    software-properties-common \
+    libgl1-mesa-glx libsm6 libxrender1 libxext-dev\
+	  ; \
+	\
+	wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \
+	wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \
+	GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \
+	gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \
+	gpg --batch --verify python.tar.xz.asc python.tar.xz; \
+	command -v gpgconf > /dev/null && gpgconf --kill all || :; \
+	rm -rf "$GNUPGHOME" python.tar.xz.asc; \
+	mkdir -p /usr/src/python; \
+	tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \
+	rm python.tar.xz; \
+	\
+	cd /usr/src/python; \
+	# gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \
+	./configure \
+		#--build="$gnuArch" \
+		--enable-loadable-sqlite-extensions \
+		--enable-optimizations \
+		--enable-option-checking=fatal \
+		--enable-shared \
+		--with-lto \
+		--with-system-expat \
+		--without-ensurepip \
+	; \
+	nproc="$(nproc)"; \
+# set thread stack size to 1MB so we don't segfault before we hit sys.getrecursionlimit()
+# https://github.com/alpinelinux/aports/commit/2026e1259422d4e0cf92391ca2d3844356c649d0
+	EXTRA_CFLAGS="-DTHREAD_STACK_SIZE=0x100000"; \
+	LDFLAGS="-Wl,--strip-all"; \
+	make -j "$nproc" \
+		"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
+		"LDFLAGS=${LDFLAGS:-}" \
+		"PROFILE_TASK=${PROFILE_TASK:-}" \
+	; \
+# https://github.com/docker-library/python/issues/784
+# prevent accidental usage of a system installed libpython of the same version
+	rm python; \
+	make -j "$nproc" \
+		"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" \
+		"LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" \
+		"PROFILE_TASK=${PROFILE_TASK:-}" \
+		python \
+	; \
+	make install; \
+	\
+	cd /; \
+	rm -rf /usr/src/python; \
+	\
+	find /usr/local -depth \
+		\( \
+			\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+			-o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \
+		\) -exec rm -rf '{}' + \
+	; \
+	\
+	find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec scanelf --needed --nobanner --format '%n#p' '{}' ';' \
+		| tr ',' '\n' \
+		| sort -u \
+		| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
+		| xargs -rt apk add --no-network --virtual .python-rundeps \
+	; \
+	# apk del --no-network .build-deps; \
+	\
+	python3 --version
+
+# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends)
+RUN set -eux; \
+	for src in idle3 pydoc3 python3 python3-config; do \
+		dst="$(echo "$src" | tr -d 3)"; \
+		[ -s "/usr/local/bin/$src" ]; \
+		[ ! -e "/usr/local/bin/$dst" ]; \
+		ln -svT "$src" "/usr/local/bin/$dst"; \
+	done
+
+# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
+ENV PYTHON_PIP_VERSION 22.3.1
+# https://github.com/docker-library/python/issues/365
+ENV PYTHON_SETUPTOOLS_VERSION 65.5.1
+# https://github.com/pypa/get-pip
+ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/1a96dc5acd0303c4700e02655aefd3bc68c78958/public/get-pip.py
+ENV PYTHON_GET_PIP_SHA256 d1d09b0f9e745610657a528689ba3ea44a73bd19c60f4c954271b790c71c2653
+
+RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py && \
+    rm get-pip.py \  
+  \
+	pip --version
+
+
+
+RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' ~/.bashrc && \
+    # above enable color prompt of docker in terminal
+    # set alias
+    echo "alias nv='nvidia-smi'" >> ~/.bash_aliases && \
+    echo "alias wnv='watch -n 1 nvidia-smi'" >> ~/.bash_aliases && \
+    echo "alias wwnv='watch -n 0.1 nvidia-smi'" >> ~/.bash_aliases
+    
+RUN python3 --version
+
+WORKDIR /app/
+
+RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python3 && \
+    cd /usr/local/bin && \
+    ln -s /opt/poetry/bin/poetry && \
+    poetry config virtualenvs.create false
+
+COPY ./app/gpuproject.toml /app/pyproject.toml
+
+ARG INSTALL_DEV=false
+RUN bash -c "if [ $INSTALL_DEV == 'true' ] ; then poetry install --no-root ; else poetry install --no-root --no-dev ; fi"
+
+ENV C_FORCE_ROOT=1
+
+COPY ./app /app
+WORKDIR /app
+COPY /${CELERY_ZIP_STORAGE} /app/${CELERY_ZIP_STORAGE}
+COPY /${CELERY_ZIP_STORAGE} /app/${CELERY_ZIP_STORAGE}
+
+
+COPY ./app/worker-start.sh /worker-start.sh
+
+RUN chmod +x /worker-start.sh
+
+CMD ["bash", "/worker-start.sh"]

frontend/src/interfaces/index.ts

@@ -42,5 +42,6 @@ export interface MainState {
 export interface Video {
   id: number;
   title: string;
+  stored_file_name: string;
   progress_state: string;
 }

local-docker-compose.override.yml

@@ -43,7 +43,7 @@ services:
       dockerfile: backend.dockerfile
       args:
         INSTALL_DEV: ${INSTALL_DEV-true}
-        INSTALL_JUPYTER: ${INSTALL_JUPYTER-true}
+        INSTALL_JUPYTER: ${INSTALL_JUPYTER-false}
     # command: bash -c "while true; do sleep 1; done"  # Infinite loop to keep container live doing nothing
     command: /start-reload.sh
     labels:
@@ -52,21 +52,19 @@ services:
       - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)
       - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=80
 
-  celeryworker:
+  gpuceleryworker:
     volumes:
       - ./backend/app:/app
-      - /${CELERY_ZIP_STORAGE}:/app/${CELERY_ZIP_STORAGE}
-      - /${CELERY_VIDEO_STORAGE}:/app${CELERY_VIDEO_STORAGE}
+      - /${CELERY_ZIP_STORAGE}:/${CELERY_ZIP_STORAGE}
+      - /${CELERY_VIDEO_STORAGE}:/${CELERY_VIDEO_STORAGE}
     environment:
       - RUN=celery worker -A app.worker -l info -Q main-queue -c 1
-      - JUPYTER=jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.custom_display_url=http://127.0.0.1:8888
-      - SERVER_HOST=http://${DOMAIN?Variable not set}
     build:
       context: ./backend
-      dockerfile: celeryworker.dockerfile
+      dockerfile: gpuceleryworker.dockerfile
       args:
         INSTALL_DEV: ${INSTALL_DEV-true}
-        INSTALL_JUPYTER: ${INSTALL_JUPYTER-true}
+        INSTALL_JUPYTER: ${INSTALL_JUPYTER-false}
 
 
 networks:

local-docker-compose.yml

@@ -124,7 +124,7 @@ services:
         - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)
         - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=80
   
-  celeryworker:
+  gpuceleryworker:
     image: '${DOCKER_IMAGE_CELERYWORKER?Variable not set}:${TAG-latest}'
     depends_on:
       - queue
@@ -137,9 +137,11 @@ services:
       - SMTP_HOST=${SMTP_HOST?Variable not set}
     build:
       context: ./backend
-      dockerfile: celeryworker.dockerfile
+      dockerfile: gpuceleryworker.dockerfile
       args:
         INSTALL_DEV: ${INSTALL_DEV-false}
+
+
   
 
 

start_local_docker.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker-compose -f local-docker-compose.yml -f local-docker-compose.override.yml up