video-maker/backend/app/app/worker.py

from raven import Client
import os, subprocess
from app.core.celery_app import celery_app
from app.core.config import settings
import requests
from pathlib import Path
from urllib.parse import urlparse, urljoin
from .aianchor import  make_video_from_zip
import gc
#client_sentry = Client(settings.SENTRY_DSN)
import dataset
from app.db.session import SessionLocal
from app.models import video
from app import crud
import gc
download_to_local_url = urljoin(settings.SERVER_HOST, settings.API_V1_STR+"/videos/worker")
upload_to_server_url = urljoin(settings.SERVER_HOST, settings.API_V1_STR+"/videos/worker")

LOCAL_ZIP_STORAGE = Path("/").joinpath(settings.LOCAL_ZIP_STORAGE)
CELERY_ZIP_STORAGE = Path("/").joinpath(settings.CELERY_ZIP_STORAGE) 
CELERY_VIDEO_STORAGE = Path("/").joinpath(settings.LOCAL_VIDEO_STORAGE)
LOCAL_VIDEO_STORAGE = Path("/").joinpath(settings.LOCAL_VIDEO_STORAGE)

STORAGE_IP = '192.168.192.252'#os.getenv('STORAGE_IP')
BACKEND_IP = '172.'
if not STORAGE_IP:
    raise Exception

@celery_app.task(acks_late=True, bind=True, track_started=True)
def make_video_test(self, title=None, anchor_id=None, lang_id=None)->int:
   print(title, anchor_id, lang_id)
   return 0

@celery_app.task(acks_late=True, bind=True, track_started=True)
def make_video(self, *, video_id, stored_filename=None, user_id=None, anchor_id=None, membership=None, available_time=None, **others) -> int:
    
    #video_id, zip_filename, user_id = args
    # download 
    '''
    r = requests.get(download_to_local_url, stream=True)
    with open(str(VIDEO_STORAGE/zip_filename), 'wb') as f:
        r.raise_for_status()
        for chunk in r.iter_content(chunk_size=1024):
            f.write(chunk)
    '''
    # db = SessionLocal()
    # db.execute("SELECT 1")
    zip_filename = stored_filename + ".zip"
    r = subprocess.run(["sshpass", "-p", "choozmo9", 
                        "scp", "-o", "StrictHostKeyChecking=no", f"root@{STORAGE_IP}:{str(LOCAL_ZIP_STORAGE/zip_filename)}", f"{str(CELERY_ZIP_STORAGE/zip_filename)}"])
    print(f'get from local storage: {r.returncode}')
    print(f"video_id: {video_id}, file name: {stored_filename}")
    
    # db.execute(f"UPDATE video SET progress_state='processing' where id={video_id}")
    # db.commit()
    # make video
    watermark_path='medias/logo_watermark.jpg'
    content_time = 0
    try:
      if membership=="infinite":
         watermark_path=None
      content_time = make_video_from_zip(working_dir=CELERY_ZIP_STORAGE,style=Path("app/style/choozmo"),  
                                         inputfile=zip_filename,
                                         opening=False, 
                                         ending=False, 
                                         watermark_path=watermark_path, 
                                         available_time=available_time)
    except Exception as e:
      print(f"error:{e}")
      # db.execute(f"UPDATE video SET progress_state='failed' where id={video_id}")
      # db.commit()
    else:
      # 
      video_filename = stored_filename + ".mp4"
      r = subprocess.run(["sshpass", "-p", "choozmo9", 
                          "scp", "-o", "StrictHostKeyChecking=no", f"{str(CELERY_ZIP_STORAGE/'output.mp4')}", f"root@{STORAGE_IP}:{'/var/www/videos/'+video_filename}"])
      print(f"return to local storage: {r.returncode}")
      print(f"video_id: {video_id}, file name: {stored_filename}")

      # db.execute(f"UPDATE video SET progress_state='completed', length={int(content_time)} where id={video_id}")
      # db.commit()

      
      gc_num = gc.collect()
      print(f"gc_num: {gc_num}")
      return int(content_time)

@celery_app.task(acks_late=True, bind=True, track_started=True)
def super_resolution(self, dirname, filenames):
   
    r = subprocess.run(["sshpass", "-p", "choozmo9", 
                        "scp", "-o", "StrictHostKeyChecking=no", "-r", f"root@{STORAGE_IP}:{str(LOCAL_ZIP_STORAGE/dirname)}", f"{str(CELERY_ZIP_STORAGE)}"])
   
    r = subprocess.run(["python", "/root/github/GFPGAN/inference_gfpgan.py", "-i", f"{str(CELERY_ZIP_STORAGE/dirname)}", "-o", f"{str(CELERY_ZIP_STORAGE/dirname)}", "-v", "1.4", "-s", "2"])
   
    r = subprocess.run(["sshpass", "-p", "choozmo9", 
                        "scp", "-o", "StrictHostKeyChecking=no", "-r", f"{str(CELERY_ZIP_STORAGE/dirname)}", f"root@{STORAGE_IP}:{str(LOCAL_ZIP_STORAGE)}"])