Merge branch 'master' of http://git.choozmo.com:3000/choozmo/kw_tools

docker/python-crontab/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.9
+RUN apt-get update && apt-get -y install cron vim
+WORKDIR /app
+COPY crontab /etc/cron.d/crontab
+COPY hello.py /app/hello.py
+RUN chmod 0644 /etc/cron.d/crontab
+RUN /usr/bin/crontab /etc/cron.d/crontab
+
+# run crond as main process of container
+CMD ["cron", "-f"]

docker/python-crontab/crontab

@@ -0,0 +1,2 @@
+#run python script every minutes
+* * * * * python /app/hello.py > /proc/1/fd/1 2>/proc/1/fd/2

docker/python-crontab/hello.py

@@ -0,0 +1,3 @@
+#!/usr/bin/python3
+print ("hello world!")
+print ("Welcome to python cron job")

docker/python37/Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.7-slim
+WORKDIR /code
+RUN apt-get update \
+    && apt-get --yes --no-install-recommends install \
+        python3 python3-dev \
+        build-essential cmake \
+        portaudio19-dev python3-pyaudio \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install numpy
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+COPY src/ .
+CMD [ "/bin/bash" ]
+
+#CMD [ "python3", "./server.py" ]
+
+#RUN apt-get update && apt-get -y install cron vim
+#WORKDIR /app
+#COPY crontab /etc/cron.d/crontab
+#COPY hello.py /app/hello.py
+#RUN chmod 0644 /etc/cron.d/crontab
+#RUN /usr/bin/crontab /etc/cron.d/crontab
+
+# run crond as main process of container
+#CMD ["cron", "-f"]
+
+# copy the dependencies file to the working directory
+
+# install dependencies
+
+# copy the content of the local src directory to the working directory
+
+# command to run on container start
+#CMD [ "python", "./server.py" ]
+
+CMD ["python3"]

docker/python37/requirements.txt

@@ -0,0 +1,11 @@
+webrtcvad
+pydub
+lws
+scipy
+numpy
+librosa
+pyworld
+tensorflow<=1.13.1
+pyaudio
+sounddevice
+dotmap

docker/python37/src/README.md

@@ -0,0 +1,236 @@
+
+
+![aukit](aukit.png "aukit")
+
+
+
+## aukit
+
+audio toolkit: 语音和频谱处理的工具箱。
+
+
+
+### 安装
+
+
+
+```
+
+pip install -U aukit
+
+```
+
+
+
+- 注意
+
+    * 可能需另外安装的依赖包：tensorflow, pyaudio, sounddevice。
+
+    * tensorflow<=1.13.1
+
+    * pyaudio暂不支持python37以上版本直接pip安装，需要下载whl文件安装，下载路径：https://www.lfd.uci.edu/~gohlke/pythonlibs/#pyaudio
+
+    * sounddevice依赖pyaudio。
+
+    * aukit的默认音频采样率为16k。
+
+
+
+### 版本
+
+v1.4.4
+
+
+
+### audio_cli
+
+命令行，播放音频，去除背景噪声，音频格式转换。
+
+支持递归处理文件夹内的全部音频。
+
+
+
+#### 命令行
+
+
+
+##### **说明**
+
+
+
+- 用位置参数来控制。
+
+- 名称说明
+
+    * inpath：输入音频路径或目录。
+
+    * outpath：输出音频路径或目录，如果为目录，则输出的子目录按照inpath的子目录格式输出。
+
+    * sr：音频采样率，默认16000或自动识别采样率。
+
+    * in_format：输入音频格式，主要用以限制为指定后缀名的文件，如果不设置，则处理目录的全部文件。
+
+    * out_format：输出音频格式，主要用以音频格式转换，设置输出音频的后缀名。
+
+- 中括号【[]】里面的是可选参数。
+
+
+
+#### **工具**
+
+- auplay: 播放音频
+
+
+
+```
+
+auplay inpath [sr] [in_format]
+
+```
+
+
+
+- aunoise: 语音降噪
+
+
+
+```
+
+aunoise inpath outpath [in_format]
+
+```
+
+
+
+
+
+- auformat: 音频格式转换
+
+
+
+```
+
+auformat inpath outpath out_format [in_format]
+
+```
+
+
+
+
+
+
+
+
+
+### audio_changer
+
+变声器，变高低音，变语速，变萝莉音，回声。
+
+
+
+### audio_editor
+
+语音编辑，切分音频，去除语音中的较长静音，去除语音首尾静音，设置采样率，设置通道数。
+
+音频格式相互转换，例如wav格式转为mp3格式。
+
+切分音频，去除静音，去除首尾静音输入输出都支持wav格式。
+
+语音编辑功能基于pydub的方法，增加了数据格式支持。
+
+
+
+### audio_griffinlim
+
+griffinlim声码器，线性频谱转语音，梅尔频谱转语音，TensorFlow版本转语音，梅尔频谱和线性频谱相互转换。
+
+
+
+### audio_io
+
+语音IO，语音保存、读取，支持wav和mp3格式，语音形式转换（np.array,bytes,io.BytesIO），支持【.】操作符的字典。
+
+
+
+### audio_noise_remover
+
+语音降噪，降低环境噪声。
+
+
+
+### audio_normalizer
+
+语音正则化，去除音量低的音频段（去除静音），调节音量。
+
+语音正则化方法基于VAD的方法。
+
+
+
+### audio_player
+
+语音播放，传入文件名播放，播放wave数据，播放bytes数据。
+
+
+
+### audio_spectrogram
+
+语音频谱，语音转线性频谱，语音转梅尔频谱。
+
+
+
+### audio_tuner
+
+语音调整，调整语速，调整音高。
+
+
+
+### audio_world
+
+world声码器，提取语音的基频、频谱包络和非周期信号，频谱转为语音。调音高，调机器人音。
+
+
+
+### 历史版本
+
+
+
+#### v1.4.4
+
+- Dict2Obj方法增加parse方法。
+
+
+
+#### v1.4.3
+
+- 修正Dict2Obj的get方法达不到预期的bug。
+
+- 修正world声码器变声静音报错的bug。
+
+
+
+#### v1.4.1
+
+- 修正安装依赖报错的bugs。
+
+- set系列改为convert系列，如set_sample_rate改为convert_sample_rate。
+
+- Dict2Obj用dotmap模块的DotMap代替。
+
+
+
+#### v1.4.0
+
+- 增加音频格式转换方法。
+
+- 命令行批量处理语音，支持音频播放、语音降噪、音频格式转换。
+
+- 增加命令行工具：auplay, aunoise, auformat。
+
+- 调整日志输出方法。
+
+
+
+#### v1.3.12
+
+- 减少不必要的依赖，最低依赖只保留pydub,  scipy, numpy, librosa。
+

docker/python37/src/aukit/__init__.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# @author: KDD
+# @time: 2018-11-10
+"""
+![aukit](aukit.png "aukit")
+
+## aukit
+audio toolkit: 语音和频谱处理的工具箱。
+
+### 安装
+
+```
+pip install -U aukit
+```
+
+- 注意
+    * 可能需另外安装的依赖包：tensorflow, pyaudio, sounddevice。
+    * tensorflow<=1.13.1
+    * pyaudio暂不支持python37以上版本直接pip安装，需要下载whl文件安装，下载路径：https://www.lfd.uci.edu/~gohlke/pythonlibs/#pyaudio
+    * sounddevice依赖pyaudio。
+    * aukit的默认音频采样率为16k。
+"""
+
+__version__ = '1.4.4'
+
+from .audio_io import load_wav, save_wav, anything2bytesio, anything2wav, anything2bytes, Dict2Obj, _sr
+from .audio_editor import strip_audio, remove_silence_audio, split_audio, convert_sample_rate
+from .audio_editor import strip_silence_wave, remove_silence_wave, split_silence_wave
+from .audio_tuner import tune_pitch, tune_speed
+from .audio_player import play_audio, play_sound
+from .audio_noise_remover import remove_noise
+from .audio_normalizer import preprocess_wav, remove_silence, tune_volume
+from .audio_spectrogram import linear_spectrogram, mel_spectrogram
+from .audio_spectrogram import default_hparams as hparams_spectrogram
+from .audio_spectrogram import linear2mel_spectrogram, mel2linear_spectrogram
+from .audio_griffinlim import inv_linear_spectrogram, inv_linear_spectrogram_tf, inv_mel_spectrogram
+from .audio_griffinlim import default_hparams as hparams_griffinlim
+from .audio_changer import change_male, change_pitch, change_attention, change_stretch, change_vague, change_pitchspeed
+from .audio_changer import change_reback, change_sample, change_speed
+from .audio_world import world_spectrogram, inv_world_spectrogram, change_voice
+from .audio_world import world_spectrogram_default, inv_world_spectrogram_default
+
+from .audio_io import __doc__ as io_doc
+from .audio_editor import __doc__ as editor_doc
+from .audio_tuner import __doc__ as tuner_doc
+from .audio_player import __doc__ as player_doc
+from .audio_noise_remover import __doc__ as noise_remover_doc
+from .audio_normalizer import __doc__ as normalizer_doc
+from .audio_spectrogram import __doc__ as spectrogram_doc
+from .audio_griffinlim import __doc__ as griffinlim_doc
+from .audio_changer import __doc__ as changer_doc
+from .audio_cli import __doc__ as cli_doc
+from .audio_world import __doc__ as world_doc
+
+version_doc = """
+### 版本
+v{}
+""".format(__version__)
+
+history_doc = """
+### 历史版本
+
+#### v1.4.4
+- Dict2Obj方法增加parse方法。
+
+#### v1.4.3
+- 修正Dict2Obj的get方法达不到预期的bug。
+- 修正world声码器变声静音报错的bug。
+
+#### v1.4.1
+- 修正安装依赖报错的bugs。
+- set系列改为convert系列，如set_sample_rate改为convert_sample_rate。
+- Dict2Obj用dotmap模块的DotMap代替。
+
+#### v1.4.0
+- 增加音频格式转换方法。
+- 命令行批量处理语音，支持音频播放、语音降噪、音频格式转换。
+- 增加命令行工具：auplay, aunoise, auformat。
+- 调整日志输出方法。
+
+#### v1.3.12
+- 减少不必要的依赖，最低依赖只保留pydub,  scipy, numpy, librosa。
+"""
+
+readme_docs = [__doc__, version_doc, cli_doc, changer_doc, editor_doc, griffinlim_doc, io_doc, noise_remover_doc,
+               normalizer_doc, player_doc, spectrogram_doc, tuner_doc, world_doc, history_doc]
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/aukit/achanger.py

@@ -0,0 +1,263 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/12/22
+"""
+### audio_changer
+变声器，变高低音，变语速，变萝莉音，回声。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+import numpy as np
+import librosa
+
+from .audio_io import _sr
+
+
+def change_pitch(wav, sr=_sr, rate=0.):
+    """
+    调音高。
+    :param rate:-20~20,float，0:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    return librosa.effects.pitch_shift(wav, sr=sr, n_steps=rate)
+
+
+def change_speed(wav, sr=_sr, rate=0.):
+    """
+    调语速。
+    :param rate:0~5,float，0:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    return librosa.effects.time_stretch(wav, rate)
+
+
+def change_sample(wav, sr=_sr, rate=1):
+    """
+    调采样率，语速和音高同时改变。
+    :param rate:0~5,float，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    return librosa.resample(wav, orig_sr=sr, target_sr=int(sr * rate))
+
+
+def change_reback(wav, sr=_sr, rate=1):
+    """
+    回声。
+    :param rate:1~10,int，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = pool(D, size=(1, rate))
+    D = repeat(D, rate)
+    return librosa.istft(D)
+
+
+def change_pitchspeed(wav, sr=_sr, rate=1):
+    """
+    音高和语速同时变化。
+    :param rate:0~10,float，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    n = int(D.shape[0] * rate)
+    if n <= D.shape[0]:
+        D = drop(D, D.shape[0] - n, mode="r")
+    else:
+        D = rewardshape(D, (n, D.shape[1]))
+    return librosa.istft(D)
+
+
+def change_attention(wav, sr=_sr, rate=0):
+    """
+    突出高音或低音段。
+    :param rate:-100~100,int，0:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = roll(D, rate)
+    return librosa.istft(D)
+
+
+def change_male(wav, sr=_sr, rate=0):
+    """
+    变男声。
+    :param rate:0~1025,int，0,1,1025:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = pool_step(D, rate)
+    return librosa.istft(D)
+
+
+def change_stretch(wav, sr=_sr, rate=1):
+    """
+    成倍拉伸延长。
+    :param rate:1~10,int，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = spread(D, rate)
+    return librosa.istft(D)
+
+
+def change_vague(wav, sr=_sr, rate=1):
+    """
+    模糊。
+    :param rate:1~10,int，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = pool(D, (1, rate))
+    D = spread(D, (1, rate))
+    return librosa.istft(D)
+
+
+class CheckStep(object):
+    def __init__(self, step):
+        self.step = step
+        self.index = 0
+
+    def __call__(self, *args):
+        self.index += 1
+        return self.index % self.step != 0
+
+
+def spread(D, size=(3, 3)):
+    """传播，重复每个数据点。"""
+    if isinstance(size, tuple):
+        if size[0] > 1:
+            D = np.repeat(D, size[0], axis=0)
+        if size[1] > 1:
+            D = np.repeat(D, size[1], axis=1)
+        if size[0] * size[1] > 1:
+            D = D / (size[0] * size[1])
+    elif isinstance(size, int):
+        D = np.repeat(D, size, axis=1)
+    return D
+
+
+def drop(D, n, mode="l"):
+    """丢弃,mode:left,right,side,center"""
+    if n == 0:
+        return D
+    if mode == "l":
+        return D[n:]
+    elif mode == "r":
+        return D[:-n]
+    elif mode == "s":
+        return D[n // 2:-(n // 2)]
+    elif mode == "c":
+        if n < len(D):
+            return np.vstack((D[:n // 2], D[-(n // 2):]))
+        else:
+            return ()
+    else:
+        raise AssertionError
+
+
+def repeat(D, n, axis=0):
+    """重复"""
+    return np.repeat(D, n, axis=axis)
+
+
+def roll(D, n):
+    """循环移动"""
+    return np.roll(D, n, axis=0)
+
+
+def rewardshape(D, shape):
+    """填充"""
+    x = shape[0] - D.shape[0]
+    y = shape[1] - D.shape[1]
+    if x > 0:
+        bottomlist = np.zeros([x, D.shape[1]])
+        D = np.r_[D, bottomlist]
+    if y > 0:
+        rightlist = np.zeros([D.shape[0], y])
+        D = np.c_[D, rightlist]
+    return D
+
+
+def pool_step(D, step):
+    """步长池化"""
+    _shape = D.shape
+    if step < 2:
+        return D
+    cs = CheckStep(step)
+    return rewardshape(np.array(list(filter(cs, D))), _shape)
+
+
+def pool(D, size=(3, 3), shapeed=False):
+    """池化"""
+    _shape = D.shape
+    if isinstance(size, tuple):
+        if size[1] > 1:
+            D = _pool(D, size[1])
+        if size[0] > 1:
+            D = _pool(D.T, size[0]).T
+    elif isinstance(size, int):
+        D = _pool(D.T, size).T
+    if shapeed:
+        D = rewardshape(D, _shape)
+    return D
+
+
+def _pool(D, poolsize):
+    """池化方法"""
+    x = D.shape[1] // poolsize
+    restsize = D.shape[1] % poolsize
+    if restsize > 0:
+        x += 1
+        rightlist = np.zeros([D.shape[0], poolsize - restsize])
+        D = np.c_[D, rightlist]
+    D = D.reshape((-1, poolsize))
+    D = D.sum(axis=1).reshape(-1, x)
+    return D
+
+
+if __name__ == '__main__':
+    from aukit.audio_player import play_audio
+
+#    path = r"C:/tmp/aukit-master/hello.wav"
+    path = r'C:/Users/jared/Downloads/hi.mp3'
+    y, sr = librosa.load(path)
+
+    # y = change_vague(3, y, sr)
+    # y = change_pitch(-6, y, sr)
+    # y = change_speed(0.5, y, sr)
+    # y = change_sample(0.8, y, sr)
+    # y = change_reback(3, y, sr)
+    # y = change_pitchspeed(2, y, sr)
+    # y = change_attention(50, y, sr)
+    # y = change_male(5, y, sr)
+    # y = change_vague(6, y, sr)
+
+    """童声"""
+#    y = change_pitch(2, y, sr)
+#    y = change_male(20, y, sr)
+    y = change_male(0, y, sr)
+
+    play_audio(y, sr=sr)

docker/python37/src/aukit/audio_changer.py

@@ -0,0 +1,261 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/12/22
+"""
+### audio_changer
+变声器，变高低音，变语速，变萝莉音，回声。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+import numpy as np
+import librosa
+
+from .audio_io import _sr
+
+
+def change_pitch(wav, sr=_sr, rate=0.):
+    """
+    调音高。
+    :param rate:-20~20,float，0:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    return librosa.effects.pitch_shift(wav, sr=sr, n_steps=rate)
+
+
+def change_speed(wav, sr=_sr, rate=0.):
+    """
+    调语速。
+    :param rate:0~5,float，0:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    return librosa.effects.time_stretch(wav, rate)
+
+
+def change_sample(wav, sr=_sr, rate=1):
+    """
+    调采样率，语速和音高同时改变。
+    :param rate:0~5,float，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    return librosa.resample(wav, orig_sr=sr, target_sr=int(sr * rate))
+
+
+def change_reback(wav, sr=_sr, rate=1):
+    """
+    回声。
+    :param rate:1~10,int，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = pool(D, size=(1, rate))
+    D = repeat(D, rate)
+    return librosa.istft(D)
+
+
+def change_pitchspeed(wav, sr=_sr, rate=1):
+    """
+    音高和语速同时变化。
+    :param rate:0~10,float，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    n = int(D.shape[0] * rate)
+    if n <= D.shape[0]:
+        D = drop(D, D.shape[0] - n, mode="r")
+    else:
+        D = rewardshape(D, (n, D.shape[1]))
+    return librosa.istft(D)
+
+
+def change_attention(wav, sr=_sr, rate=0):
+    """
+    突出高音或低音段。
+    :param rate:-100~100,int，0:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = roll(D, rate)
+    return librosa.istft(D)
+
+
+def change_male(wav, sr=_sr, rate=0):
+    """
+    变男声。
+    :param rate:0~1025,int，0,1,1025:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = pool_step(D, rate)
+    return librosa.istft(D)
+
+
+def change_stretch(wav, sr=_sr, rate=1):
+    """
+    成倍拉伸延长。
+    :param rate:1~10,int，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = spread(D, rate)
+    return librosa.istft(D)
+
+
+def change_vague(wav, sr=_sr, rate=1):
+    """
+    模糊。
+    :param rate:1~10,int，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = pool(D, (1, rate))
+    D = spread(D, (1, rate))
+    return librosa.istft(D)
+
+
+class CheckStep(object):
+    def __init__(self, step):
+        self.step = step
+        self.index = 0
+
+    def __call__(self, *args):
+        self.index += 1
+        return self.index % self.step != 0
+
+
+def spread(D, size=(3, 3)):
+    """传播，重复每个数据点。"""
+    if isinstance(size, tuple):
+        if size[0] > 1:
+            D = np.repeat(D, size[0], axis=0)
+        if size[1] > 1:
+            D = np.repeat(D, size[1], axis=1)
+        if size[0] * size[1] > 1:
+            D = D / (size[0] * size[1])
+    elif isinstance(size, int):
+        D = np.repeat(D, size, axis=1)
+    return D
+
+
+def drop(D, n, mode="l"):
+    """丢弃,mode:left,right,side,center"""
+    if n == 0:
+        return D
+    if mode == "l":
+        return D[n:]
+    elif mode == "r":
+        return D[:-n]
+    elif mode == "s":
+        return D[n // 2:-(n // 2)]
+    elif mode == "c":
+        if n < len(D):
+            return np.vstack((D[:n // 2], D[-(n // 2):]))
+        else:
+            return ()
+    else:
+        raise AssertionError
+
+
+def repeat(D, n, axis=0):
+    """重复"""
+    return np.repeat(D, n, axis=axis)
+
+
+def roll(D, n):
+    """循环移动"""
+    return np.roll(D, n, axis=0)
+
+
+def rewardshape(D, shape):
+    """填充"""
+    x = shape[0] - D.shape[0]
+    y = shape[1] - D.shape[1]
+    if x > 0:
+        bottomlist = np.zeros([x, D.shape[1]])
+        D = np.r_[D, bottomlist]
+    if y > 0:
+        rightlist = np.zeros([D.shape[0], y])
+        D = np.c_[D, rightlist]
+    return D
+
+
+def pool_step(D, step):
+    """步长池化"""
+    _shape = D.shape
+    if step < 2:
+        return D
+    cs = CheckStep(step)
+    return rewardshape(np.array(list(filter(cs, D))), _shape)
+
+
+def pool(D, size=(3, 3), shapeed=False):
+    """池化"""
+    _shape = D.shape
+    if isinstance(size, tuple):
+        if size[1] > 1:
+            D = _pool(D, size[1])
+        if size[0] > 1:
+            D = _pool(D.T, size[0]).T
+    elif isinstance(size, int):
+        D = _pool(D.T, size).T
+    if shapeed:
+        D = rewardshape(D, _shape)
+    return D
+
+
+def _pool(D, poolsize):
+    """池化方法"""
+    x = D.shape[1] // poolsize
+    restsize = D.shape[1] % poolsize
+    if restsize > 0:
+        x += 1
+        rightlist = np.zeros([D.shape[0], poolsize - restsize])
+        D = np.c_[D, rightlist]
+    D = D.reshape((-1, poolsize))
+    D = D.sum(axis=1).reshape(-1, x)
+    return D
+
+
+if __name__ == '__main__':
+    from aukit.audio_player import play_audio
+
+    path = r"E:/data/temp/01.wav"
+    y, sr = librosa.load(path)
+
+    # y = change_vague(3, y, sr)
+    # y = change_pitch(-6, y, sr)
+    # y = change_speed(0.5, y, sr)
+    # y = change_sample(0.8, y, sr)
+    # y = change_reback(3, y, sr)
+    # y = change_pitchspeed(2, y, sr)
+    # y = change_attention(50, y, sr)
+    # y = change_male(5, y, sr)
+    # y = change_vague(6, y, sr)
+
+    """童声"""
+    y = change_pitch(6, y, sr)
+    y = change_male(20, y, sr)
+
+    play_audio(y, sr=sr)

docker/python37/src/aukit/audio_cli.py

@@ -0,0 +1,212 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2020/1/5
+"""
+### audio_cli
+命令行，播放音频，去除背景噪声，音频格式转换。
+支持递归处理文件夹内的全部音频。
+
+#### 命令行
+
+##### **说明**
+
+- 用位置参数来控制。
+- 名称说明
+    * inpath：输入音频路径或目录。
+    * outpath：输出音频路径或目录，如果为目录，则输出的子目录按照inpath的子目录格式输出。
+    * sr：音频采样率，默认16000或自动识别采样率。
+    * in_format：输入音频格式，主要用以限制为指定后缀名的文件，如果不设置，则处理目录的全部文件。
+    * out_format：输出音频格式，主要用以音频格式转换，设置输出音频的后缀名。
+- 中括号【[]】里面的是可选参数。
+
+#### **工具**
+- auplay: 播放音频
+
+```
+auplay inpath [sr] [in_format]
+```
+
+- aunoise: 语音降噪
+
+```
+aunoise inpath outpath [in_format]
+```
+
+
+- auformat: 音频格式转换
+
+```
+auformat inpath outpath out_format [in_format]
+```
+
+
+
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+import sys
+from functools import partial
+from multiprocessing import Pool
+from tqdm import tqdm
+from pathlib import Path
+import traceback
+import multiprocessing as mp
+
+from .audio_io import _sr
+from .audio_player import play_audio
+from .audio_noise_remover import remove_noise_os
+from .audio_editor import convert_format_os
+
+_n_process = max(1, mp.cpu_count() - 2)
+
+
+def play_audio_one(inpath, sr):
+    try:
+        play_audio(inpath, sr=sr)
+    except Exception as e:
+        logger.info("PlayAudioFailed: {}".format(inpath))
+        traceback.print_exc()
+
+
+def play_audio_cli():
+    fpath = Path(sys.argv[1])
+    if len(sys.argv) >= 3:
+        sr = int(sys.argv[2])
+    else:
+        sr = _sr
+    if len(sys.argv) >= 4:
+        in_format = sys.argv[3]
+    else:
+        in_format = None
+
+    if fpath.is_file():
+        play_audio_one(fpath, sr=sr)
+    elif fpath.is_dir():
+        tmp = "**/*" if in_format is None else "**/*.{}".format(in_format)
+        for inpath in sorted(fpath.glob(tmp)):
+            if not inpath.is_file():
+                continue
+            try:
+                play_audio_one(inpath, sr=sr)
+            except Exception as e:
+                logger.info("PlayAudioFailed: {}".format(inpath))
+                traceback.print_exc()
+    else:
+        assert fpath.exists()
+
+
+def remove_noise_one(x):
+    outpath = Path(x["outpath"])
+    outpath.parent.mkdir(exist_ok=True, parents=True)
+    try:
+        out = remove_noise_os(**x)
+    except Exception as e:
+        inpath = str(x["inpath"])
+        logger.info("RemoveNoiseFailed: {}".format(inpath))
+        traceback.print_exc()
+        out = None
+    return out
+
+
+def remove_noise_cli():
+    inpath = Path(sys.argv[1])
+    outpath = Path(sys.argv[2])
+    if len(sys.argv) >= 4:
+        in_format = sys.argv[3]
+    else:
+        in_format = None
+
+    if inpath.is_file():
+        remove_noise_one(dict(inpath=inpath, outpath=outpath))
+    elif inpath.is_dir():
+        indir = inpath
+        outdir = outpath
+        kw_lst = []
+        tmp = "**/*" if in_format is None else "**/*.{}".format(in_format)
+        for inpath in indir.glob(tmp):
+            if not inpath.is_file():
+                continue
+            parts = inpath.relative_to(indir).parts
+            outpath = outdir.joinpath(*parts)
+            kw = dict(inpath=str(inpath), outpath=str(outpath))
+            kw_lst.append(kw)
+
+        pool_jobs(func=remove_noise_one, n_process=_n_process, kwargs_list=kw_lst, tqdm_desc='remove_noise')
+    else:
+        assert inpath.exists()
+
+
+def convert_format_one(x):
+    outpath = Path(x["outpath"])
+    outpath.parent.mkdir(exist_ok=True, parents=True)
+    try:
+        out = convert_format_os(**x)
+    except Exception as e:
+        inpath = str(x["inpath"])
+        logger.info("ConvertFormatFailed: {}".format(inpath))
+        traceback.print_exc()
+        out = None
+    return out
+
+
+def convert_format_cli():
+    inpath = Path(sys.argv[1])
+    outpath = Path(sys.argv[2])
+    out_format = sys.argv[3]
+    if len(sys.argv) >= 5:
+        in_format = sys.argv[4]
+    else:
+        in_format = None
+
+    if inpath.is_file():
+        convert_format_one(dict(inpath=inpath, outpath=outpath, in_format=in_format, out_format=out_format))
+    elif inpath.is_dir():
+        indir = inpath
+        outdir = outpath
+        kw_lst = []
+        tmp = "**/*" if in_format is None else "**/*.{}".format(in_format)
+        for inpath in indir.glob(tmp):
+            if not inpath.is_file():
+                continue
+            parts = inpath.parent.relative_to(indir).parts
+            name = "{}.{}".format(inpath.stem, out_format)
+            outpath = outdir.joinpath(*parts, name)
+            kw = dict(inpath=str(inpath), outpath=str(outpath), in_format=in_format, out_format=out_format)
+            kw_lst.append(kw)
+
+        pool_jobs(func=convert_format_one, n_process=_n_process, kwargs_list=kw_lst, tqdm_desc='convert_format')
+    else:
+        assert inpath.exists()
+
+
+def pool_jobs(func, n_process=_n_process, kwargs_list=(), post_func=None, tqdm_desc="job"):
+    """
+    多进程执行任务。
+    :param func: 第一个参数为变化参数，如果多个参数组合变化，请用：【def _func(x): return func(**x)】的方式处理函数。
+    :param n_process:
+    :param kwargs_list:
+    :param post_func:
+    :param tqdm_desc:
+    :return:
+    """
+    if post_func is None:
+        post_func = lambda x: x
+    _tqdm = lambda x: tqdm(x, desc=tqdm_desc, total=len(kwargs_list), ncols=80, mininterval=1)
+    if n_process == 0 or n_process == 1:
+        for kw in _tqdm(kwargs_list):
+            out = func(kw)
+            post_func(out)
+    else:
+        partial_func = partial(func)
+        job = Pool(n_process).imap(partial_func, kwargs_list)
+        for out in list(_tqdm(job)):
+            post_func(out)
+
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/aukit/audio_editor.py

@@ -0,0 +1,203 @@
+# author: kdd
+# date: 
+"""
+### audio_editor
+语音编辑，切分音频，去除语音中的较长静音，去除语音首尾静音，设置采样率，设置通道数。
+音频格式相互转换，例如wav格式转为mp3格式。
+切分音频，去除静音，去除首尾静音输入输出都支持wav格式。
+语音编辑功能基于pydub的方法，增加了数据格式支持。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+from pydub import AudioSegment
+from pydub.silence import detect_nonsilent
+
+from .audio_io import anything2bytesio, _sr, _int16_max
+import numpy as np
+import io
+
+
+def convert_channels(wav, sr=_sr, value=1):
+    aud = wav2audiosegment(wav, sr=sr)
+    aud = aud.set_channels(channels=value)
+    wav = audiosegment2wav(aud)
+    return wav
+
+
+def convert_sample_rate(wav, sr=_sr, value=_sr):
+    aud = wav2audiosegment(wav, sr=sr)
+    aud = aud.set_frame_rate(frame_rate=value)
+    wav = audiosegment2wav(aud)
+    return wav
+
+
+def convert_sample_width(wav, sr=_sr, value=4):
+    aud = wav2audiosegment(wav, sr=sr)
+    aud = aud.set_sample_width(sample_width=value)
+    wav = audiosegment2wav(aud)
+    return wav
+
+
+def convert_format(wav, sr=_sr, format='mp3'):
+    """
+    语音信号转为指定音频格式的bytes。
+    :param wav:
+    :param sr:
+    :param format:
+    :return:
+    """
+    aud = wav2audiosegment(wav, sr=sr)
+    out = io.BytesIO()
+    aud.export(out, format=format)
+    return out.getvalue()
+
+
+def convert_format_os(inpath, outpath, out_format='mp3', in_format=None):
+    """
+    音频格式转换。
+    :param inpath:
+    :param outpath:
+    :param in_format:
+    :param out_format:
+    :return:
+    """
+    src = AudioSegment.from_file(inpath, format=in_format)
+    src.export(outpath, format=out_format)
+
+
+def audiosegment2wav(data: AudioSegment):
+    """
+    pydub.AudioSegment格式转为音频信号wav。
+    :param data:
+    :return:
+    """
+    wav = np.array(data.get_array_of_samples()) / _int16_max
+    return wav
+
+
+def wav2audiosegment(wav: np.ndarray, sr):
+    """
+    音频信号wav转为pydub.AudioSegment格式。
+    :param wav:
+    :param sr:
+    :return:
+    """
+    tmp = anything2bytesio(wav, sr=sr)
+    out = AudioSegment.from_wav(tmp)
+    return out
+
+
+def strip_silence_wave(wav: np.ndarray, sr=_sr, keep_silence_len=20, min_silence_len=100, silence_thresh=-32, **kwargs):
+    """
+    去除语音前后静音。
+    :param wav:
+    :param sr:
+    :param keep_silence_len:
+    :param min_silence_len:
+    :param silence_thresh:
+    :param kwargs:
+    :return:
+    """
+    data = wav2audiosegment(wav, sr=sr)
+    out = strip_audio(data,
+                      keep_silence_len=keep_silence_len,
+                      min_silence_len=min_silence_len,
+                      silence_thresh=silence_thresh,
+                      **kwargs)
+    out = audiosegment2wav(out)
+    return out
+
+
+def strip_audio(data: AudioSegment, keep_silence_len=20, min_silence_len=100, silence_thresh=-32, **kwargs):
+    nsils = detect_nonsilent(data, min_silence_len=min_silence_len, silence_thresh=silence_thresh)
+    if len(nsils) >= 1:
+        return data[max(0, nsils[0][0] - keep_silence_len): min(len(data), nsils[-1][1] + keep_silence_len)]
+    else:
+        return AudioSegment.empty()
+
+
+def strip_audio_os(inpath, outpath, **kwargs):
+    try:
+        data = AudioSegment.from_file(inpath, kwargs.get('format', 'wav'))
+        out = strip_audio(data, **kwargs)
+        out.export(outpath, kwargs.get('format', 'wav'))
+    except Exception as e:
+        logger.info('Error path: {}'.format(inpath))
+        logger.info('Error info: {}'.format(e))
+
+
+def split_silence_wave(wav, sr=_sr, keep_silence_len=20, min_silence_len=100, silence_thresh=-32, **kwargs):
+    """
+    根据静音切分音频。
+    :param wav:
+    :param sr:
+    :param keep_silence_len:
+    :param min_silence_len:
+    :param silence_thresh:
+    :param kwargs:
+    :return:
+    """
+    data = wav2audiosegment(wav, sr=sr)
+    outs = split_audio(data,
+                       keep_silence_len=keep_silence_len,
+                       min_silence_len=min_silence_len,
+                       silence_thresh=silence_thresh,
+                       **kwargs)
+    out_wavs = []
+    for out in outs:
+        wav = audiosegment2wav(out)
+        out_wavs.append(wav)
+    return out_wavs
+
+
+def split_audio(data: AudioSegment, keep_silence_len=20, min_silence_len=100, silence_thresh=-32, **kwargs):
+    nsils = detect_nonsilent(data, min_silence_len=min_silence_len, silence_thresh=silence_thresh)
+    if len(nsils) >= 1:
+        outs = []
+        for ab in nsils:
+            out = data[max(0, ab[0] - keep_silence_len): min(len(data), ab[1] + keep_silence_len)]
+            outs.append(out)
+    else:
+        outs = [AudioSegment.empty()]
+    return outs
+
+
+def remove_silence_wave(wav, sr=_sr, keep_silence_len=20, min_silence_len=100, silence_thresh=-32, **kwargs):
+    """
+    去除音频中的静音段。
+    :param wav:
+    :param sr:
+    :param keep_silence_len:
+    :param min_silence_len:
+    :param silence_thresh:
+    :param kwargs:
+    :return:
+    """
+    data = wav2audiosegment(wav, sr=sr)
+    out = remove_silence_audio(data,
+                               keep_silence_len=keep_silence_len,
+                               min_silence_len=min_silence_len,
+                               silence_thresh=silence_thresh,
+                               **kwargs)
+    out = audiosegment2wav(out)
+    return out
+
+
+def remove_silence_audio(data: AudioSegment, keep_silence_len=20, min_silence_len=100, silence_thresh=-32, **kwargs):
+    nsils = detect_nonsilent(data, min_silence_len=min_silence_len, silence_thresh=silence_thresh)
+    out = AudioSegment.empty()
+    sf = 0
+    for i, ab in enumerate(nsils):
+        si = max(ab[0] - keep_silence_len, sf)
+        ei = ab[1] + keep_silence_len
+        out = out + data[si: ei]
+        sf = ei
+    return out
+
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/aukit/audio_griffinlim.py

@@ -0,0 +1,422 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/11/30
+"""
+### audio_griffinlim
+griffinlim声码器，线性频谱转语音，梅尔频谱转语音，TensorFlow版本转语音，梅尔频谱和线性频谱相互转换。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+import librosa
+import librosa.filters
+import numpy as np
+from scipy import signal
+from scipy.io import wavfile
+from .audio_spectrogram import default_hparams
+from .audio_io import Dict2Obj
+
+# try:
+#     import tensorflow as tf
+# except ImportError as e:
+#     logger.info("ImportError: {}".format(e))
+
+tmp = dict([('use_lws', False), ('frame_shift_ms', None), ('silence_threshold', 2), ('griffin_lim_iters', 30)])
+default_hparams.update(tmp)
+
+
+def hparams_debug_string(hparams=None):
+    hparams = hparams or default_hparams
+    values = hparams.values()
+    hp = ["  %s: %s" % (name, values[name]) for name in sorted(values) if name != "sentences"]
+    return "Hyperparameters:\n" + "\n".join(hp)
+
+
+def inv_linear_spectrogram(linear_spectrogram, hparams=None):
+    """Converts linear spectrogram to waveform using librosa"""
+    hparams = hparams or default_hparams
+    if hparams.signal_normalization:
+        D = _denormalize(linear_spectrogram, hparams)
+    else:
+        D = linear_spectrogram
+
+    S = _db_to_amp(D + hparams.ref_level_db)  # Convert back to linear
+
+    if hparams.use_lws:
+        processor = _lws_processor(hparams)
+        D = processor.run_lws(S.astype(np.float64).T ** hparams.power)
+        y = processor.istft(D).astype(np.float32)
+        return inv_preemphasis(y, hparams.preemphasis, hparams.preemphasize)
+    else:
+        return inv_preemphasis(_griffin_lim(S ** hparams.power, hparams), hparams.preemphasis, hparams.preemphasize)
+
+
+def inv_mel_spectrogram(mel_spectrogram, hparams=None):
+    """Converts mel spectrogram to waveform using librosa"""
+    hparams = hparams or default_hparams
+    if hparams.signal_normalization:
+        D = _denormalize(mel_spectrogram, hparams)
+    else:
+        D = mel_spectrogram
+
+    S = _mel_to_linear(_db_to_amp(D + hparams.ref_level_db), hparams)  # Convert back to linear
+
+    if hparams.use_lws:
+        processor = _lws_processor(hparams)
+        D = processor.run_lws(S.astype(np.float64).T ** hparams.power)
+        y = processor.istft(D).astype(np.float32)
+        return inv_preemphasis(y, hparams.preemphasis, hparams.preemphasize)
+    else:
+        return inv_preemphasis(_griffin_lim(S ** hparams.power, hparams), hparams.preemphasis, hparams.preemphasize)
+
+
+def inv_linear_spectrogram_tensorflow(linear_spectrogram, hparams=None):
+    '''Builds computational graph to convert spectrogram to waveform using TensorFlow.
+    Unlike inv_spectrogram, this does NOT invert the preemphasis. The caller should call
+    inv_preemphasis on the output after running the graph.
+    linear_spectrogram.shape[1] = n_fft
+    '''
+    import tensorflow as tf
+    hparams = hparams or default_hparams
+    S = _db_to_amp_tensorflow(_denormalize_tensorflow(linear_spectrogram, hparams) + hparams.ref_level_db)
+    return _griffin_lim_tensorflow(tf.pow(S, hparams.power), hparams)
+
+
+def inv_linear_spectrogram_tf(linear_spectrogram, hparams=None):
+    """
+    返回wav语音信号。
+    linear_spectrogram.shape[1] = num_freq = (n_fft / 2) + 1
+    """
+    import tensorflow as tf
+    hparams = hparams or default_hparams
+    _shape = linear_spectrogram.shape
+    tmp = np.concatenate(
+        (linear_spectrogram, np.zeros((_shape[0], (hparams.n_fft // 2) + 1 - _shape[1]), dtype=np.float32)), axis=1)
+    wav_tf = inv_linear_spectrogram_tensorflow(tmp, hparams)
+    with tf.Session() as sess:
+        return sess.run(wav_tf)
+
+
+# 以下模块后续版本可能删除
+
+def load_wav(path, sr):
+    return librosa.core.load(path, sr=sr)[0]
+
+
+def save_wav(wav, path, sr):
+    out = wav * 32767 / max(0.01, np.max(np.abs(wav)))
+    # proposed by @dsmiller
+    wavfile.write(path, sr, out.astype(np.int16))
+
+
+def save_wavenet_wav(wav, path, sr):
+    librosa.output.write_wav(path, wav, sr=sr)
+
+
+def preemphasis(wav, k, preemphasize=True):
+    if preemphasize:
+        return signal.lfilter([1, -k], [1], wav)
+    return wav
+
+
+def inv_preemphasis(wav, k, inv_preemphasize=True):
+    if inv_preemphasize:
+        return signal.lfilter([1], [1, -k], wav)
+    return wav
+
+
+# From https://github.com/r9y9/wavenet_vocoder/blob/master/audio.py
+def start_and_end_indices(quantized, silence_threshold=2):
+    for start in range(quantized.size):
+        if abs(quantized[start] - 127) > silence_threshold:
+            break
+    for end in range(quantized.size - 1, 1, -1):
+        if abs(quantized[end] - 127) > silence_threshold:
+            break
+
+    assert abs(quantized[start] - 127) > silence_threshold
+    assert abs(quantized[end] - 127) > silence_threshold
+
+    return start, end
+
+
+def get_hop_size(hparams=None):
+    hparams = hparams or default_hparams
+    hop_size = hparams.hop_size
+    if hop_size is None:
+        assert hparams.frame_shift_ms is not None
+        hop_size = int(hparams.frame_shift_ms / 1000 * hparams.sample_rate)
+    return hop_size
+
+
+def linear_spectrogram(wav, hparams=None):
+    hparams = hparams or default_hparams
+    D = _stft(preemphasis(wav, hparams.preemphasis, hparams.preemphasize), hparams)
+    S = _amp_to_db(np.abs(D), hparams) - hparams.ref_level_db
+
+    if hparams.signal_normalization:
+        return _normalize(S, hparams)
+    return S
+
+
+def mel_spectrogram(wav, hparams=None):
+    hparams = hparams or default_hparams
+    D = _stft(preemphasis(wav, hparams.preemphasis, hparams.preemphasize), hparams)
+    S = _amp_to_db(_linear_to_mel(np.abs(D), hparams), hparams) - hparams.ref_level_db
+
+    if hparams.signal_normalization:
+        return _normalize(S, hparams)
+    return S
+
+
+def mel_spectrogram_feature(wav, hparams=None):
+    """
+    Derives a mel spectrogram ready to be used by the encoder from a preprocessed audio waveform.
+    Note: this not a log-mel spectrogram.
+    """
+    hparams = hparams or default_hparams
+    frames = librosa.feature.melspectrogram(
+        wav,
+        hparams.sample_rate,
+        n_fft=hparams.n_fft,
+        hop_length=hparams.hop_size,
+        n_mels=hparams.num_mels
+    )
+    return _amp_to_db(frames.astype(np.float32))
+
+
+def linear2mel_spectrogram(linear_spectrogram, hparams=None):
+    """Converts linear spectrogram to mel spectrogram"""
+    hparams = hparams or default_hparams
+
+    if hparams.signal_normalization:
+        D = _denormalize(linear_spectrogram, hparams)
+    else:
+        D = linear_spectrogram
+
+    D = _db_to_amp(D + hparams.ref_level_db)  # Convert back to linear
+
+    S = _amp_to_db(_linear_to_mel(np.abs(D), hparams), hparams) - hparams.ref_level_db
+
+    if hparams.signal_normalization:
+        return _normalize(S, hparams)
+    return S
+
+
+def mel2linear_spectrogram(mel_spectrogram, hparams=None):
+    """Converts mel spectrogram to linear spectrogram"""
+    hparams = hparams or default_hparams
+
+    if hparams.signal_normalization:
+        D = _denormalize(mel_spectrogram, hparams)
+    else:
+        D = mel_spectrogram
+
+    D = _mel_to_linear(_db_to_amp(D - hparams.ref_level_db), hparams)  # Convert back to linear
+    S = _amp_to_db(np.abs(D), hparams) - hparams.ref_level_db
+
+    if hparams.signal_normalization:
+        return _normalize(S, hparams)
+    return S
+
+
+def _lws_processor(hparams=None):
+    hparams = hparams or default_hparams
+    import lws
+    return lws.lws(hparams.n_fft, get_hop_size(hparams), fftsize=hparams.win_size, mode="speech")
+
+
+def find_endpoint(wav, threshold_db=-40, min_silence_sec=0.8, hparams=None):
+    hparams = hparams or default_hparams
+    window_length = int(hparams.sample_rate * min_silence_sec)
+    hop_length = int(window_length / 4)
+    threshold = _db_to_amp(threshold_db)
+    for x in range(hop_length, len(wav) - window_length, hop_length):
+        if np.max(wav[x:x + window_length]) < threshold:
+            return x + hop_length
+    return len(wav)
+
+
+def _griffin_lim(S, hparams=None):
+    """librosa implementation of Griffin-Lim
+    Based on https://github.com/librosa/librosa/issues/434
+    """
+    hparams = hparams or default_hparams
+    angles = np.exp(2j * np.pi * np.random.rand(*S.shape))
+    S_complex = np.abs(S).astype(np.complex)
+    y = _istft(S_complex * angles, hparams)
+    for i in range(hparams.griffin_lim_iters):
+        angles = np.exp(1j * np.angle(_stft(y, hparams)))
+        y = _istft(S_complex * angles, hparams)
+    return y
+
+
+def _griffin_lim_tensorflow(S, hparams=None):
+    '''TensorFlow implementation of Griffin-Lim
+    Based on https://github.com/Kyubyong/tensorflow-exercises/blob/master/Audio_Processing.ipynb
+    '''
+    import tensorflow as tf
+    hparams = hparams or default_hparams
+    with tf.variable_scope('griffinlim'):
+        # TensorFlow's stft and istft operate on a batch of spectrograms; create batch of size 1
+        S = tf.expand_dims(S, 0)
+        S_complex = tf.identity(tf.cast(S, dtype=tf.complex64))
+        y = _istft_tensorflow(S_complex, hparams)
+        for i in range(hparams.griffin_lim_iters):
+            est = _stft_tensorflow(y, hparams)
+            angles = est / tf.cast(tf.maximum(1e-8, tf.abs(est)), tf.complex64)
+            y = _istft_tensorflow(S_complex * angles, hparams)
+        return tf.squeeze(y, 0)
+
+
+def _stft(y, hparams=None):
+    hparams = hparams or default_hparams
+    if hparams.use_lws:
+        return _lws_processor(hparams).stft(y).T
+    else:
+        return librosa.stft(y=y, n_fft=hparams.n_fft, hop_length=get_hop_size(hparams), win_length=hparams.win_size,
+                            center=hparams.center)
+
+
+def _stft_tensorflow(signals, hparams=None):
+    import tensorflow as tf
+    hparams = hparams or default_hparams
+    n_fft, hop_length, win_length = _stft_parameters(hparams)
+    return tf.contrib.signal.stft(signals, win_length, hop_length, n_fft, pad_end=False)
+
+
+def _istft(y, hparams=None):
+    hparams = hparams or default_hparams
+    return librosa.istft(y, hop_length=get_hop_size(hparams), win_length=hparams.win_size, center=hparams.center)
+
+
+def _istft_tensorflow(stfts, hparams=None):
+    import tensorflow as tf
+    hparams = hparams or default_hparams
+    n_fft, hop_length, win_length = _stft_parameters(hparams)
+    return tf.contrib.signal.inverse_stft(stfts, win_length, hop_length, n_fft)
+
+
+def _stft_parameters(hparams=None):
+    hparams = hparams or default_hparams
+    n_fft = hparams.n_fft  # (hparams.num_freq - 1) * 2
+    hop_length = hparams.hop_size  # int(hparams.frame_shift_ms / 1000 * hparams.sample_rate)
+    win_length = hparams.win_size  # int(hparams.frame_length_ms / 1000 * hparams.sample_rate)
+    return n_fft, hop_length, win_length
+
+
+##########################################################
+# Those are only correct when using lws!!! (This was messing with Wavenet quality for a long time!)
+def num_frames(length, fsize, fshift):
+    """Compute number of time frames of spectrogram
+    """
+    pad = (fsize - fshift)
+    if length % fshift == 0:
+        M = (length + pad * 2 - fsize) // fshift + 1
+    else:
+        M = (length + pad * 2 - fsize) // fshift + 2
+    return M
+
+
+def pad_lr(x, fsize, fshift):
+    """Compute left and right padding
+    """
+    M = num_frames(len(x), fsize, fshift)
+    pad = (fsize - fshift)
+    T = len(x) + 2 * pad
+    r = (M - 1) * fshift + fsize - T
+    return pad, pad + r
+
+
+##########################################################
+# Librosa correct padding
+def librosa_pad_lr(x, fsize, fshift):
+    return 0, (x.shape[0] // fshift + 1) * fshift - x.shape[0]
+
+
+def _linear_to_mel(spectogram, hparams=None):
+    hparams = hparams or default_hparams
+    if hparams.mel_basis is None:
+        _mel_basis = _build_mel_basis(hparams)
+    else:
+        _mel_basis = hparams.mel_basis
+    return np.dot(_mel_basis, spectogram)
+
+
+def _mel_to_linear(mel_spectrogram, hparams=None):
+    hparams = hparams or default_hparams
+    if hparams.inv_mel_basis is None:
+        _inv_mel_basis = np.linalg.pinv(_build_mel_basis(hparams))
+    else:
+        _inv_mel_basis = hparams.inv_mel_basis
+    return np.maximum(1e-10, np.dot(_inv_mel_basis, mel_spectrogram))
+
+
+def _build_mel_basis(hparams=None):
+    hparams = hparams or default_hparams
+    assert hparams.fmax <= hparams.sample_rate // 2
+    return librosa.filters.mel(hparams.sample_rate, hparams.n_fft, n_mels=hparams.num_mels,
+                               fmin=hparams.fmin, fmax=hparams.fmax)
+
+
+def _amp_to_db(x, hparams=None):
+    hparams = hparams or default_hparams
+    min_level = np.exp(hparams.min_level_db / 20 * np.log(10))
+    return 20 * np.log10(np.maximum(min_level, x))
+
+
+def _db_to_amp(x):
+    return np.power(10.0, (x) * 0.05)
+
+
+def _db_to_amp_tensorflow(x):
+    import tensorflow as tf
+    return tf.pow(tf.ones(tf.shape(x)) * 10.0, x * 0.05)
+
+
+def _normalize(S, hparams=None):
+    hparams = hparams or default_hparams
+    ma = hparams.max_abs_value
+    mi = hparams.min_level_db
+    if hparams.allow_clipping_in_normalization:
+        if hparams.symmetric_mels:
+            return np.clip((2 * ma) * ((S - mi) / (-mi)) - ma, -ma, ma)
+        else:
+            return np.clip(ma * ((S - mi) / (-mi)), 0, ma)
+    else:
+        assert S.max() <= 0 and S.min() - mi >= 0
+        if hparams.symmetric_mels:
+            return (2 * ma) * ((S - mi) / (-mi)) - ma
+        else:
+            return ma * ((S - mi) / (-mi))
+
+
+def _denormalize(D, hparams=None):
+    hparams = hparams or default_hparams
+    ma = hparams.max_abs_value
+    mi = hparams.min_level_db
+    if hparams.allow_clipping_in_normalization:
+        if hparams.symmetric_mels:
+            return ((np.clip(D, -ma, ma) + ma) * -mi / (2 * ma)) + mi
+        else:
+            return (np.clip(D, 0, ma) * -mi / ma) + mi
+    else:
+        if hparams.symmetric_mels:
+            return ((D + ma) * -mi / (2 * ma)) + mi
+        else:
+            return (D * -mi / ma) + mi
+
+
+def _denormalize_tensorflow(S, hparams=None):
+    import tensorflow as tf
+    hparams = hparams or default_hparams
+    mi = hparams.min_level_db
+    return (tf.clip_by_value(S, 0, 1) * -mi) + mi
+
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/aukit/audio_io.py

@@ -0,0 +1,120 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/12/1
+"""
+### audio_io
+语音IO，语音保存、读取，支持wav和mp3格式，语音形式转换（np.array,bytes,io.BytesIO），支持【.】操作符的字典。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+from scipy.io import wavfile
+from pathlib import Path
+import numpy as np
+import librosa
+import io
+import json
+from dotmap import DotMap
+
+_sr = 16000
+_int16_max = 2 ** 15 - 1
+
+
+class Dict2Obj(DotMap):
+    """
+    修正DotMap的get方法生成DotMap对象的bug。
+    Dict2Obj的get方法和dict的get功能相同。
+    """
+
+    def __getitem__(self, k):
+        if k not in self._map:
+            return None
+        else:
+            return self._map[k]
+
+    def parse(self, json_string):
+        if json_string.strip():
+            _hp = json.loads(json_string)
+            for k, v in _hp.items():
+                self[k] = v
+        return self
+
+
+def load_wav(path, sr=None, with_sr=False):
+    """
+    导入语音信号。支持wav和mp3格式。
+    :param path: 文件路径。
+    :param sr: 采样率，None: 自动识别采样率。
+    :param with_sr: 是否返回采样率。
+    :return: np.ndarray
+    """
+    return load_wav_librosa(path, sr=sr, with_sr=with_sr)
+
+
+def save_wav(wav, path, sr=_sr):
+    save_wav_wavfile(wav, path=path, sr=sr)
+
+
+def load_wav_librosa(path, sr=_sr, with_sr=False):
+    wav, sr = librosa.core.load(path, sr=sr)
+    return (wav, sr) if with_sr else wav
+
+
+def load_wav_wavfile(path, sr=None, with_sr=False):
+    sr, wav = wavfile.read(path)
+    wav = wav / np.max(np.abs(wav))
+    return (wav, sr) if with_sr else wav
+
+
+def save_wav_librosa(wav, path, sr=_sr):
+    librosa.output.write_wav(path, wav, sr=sr)
+
+
+def save_wav_wavfile(wav, path, sr=_sr, volume=1.):
+    out = wav * _int16_max * volume / max(0.01, np.max(np.abs(wav)))
+    # proposed by @dsmiller
+    wavfile.write(path, sr, out.astype(np.int16))
+
+
+def anything2bytesio(src, sr=_sr, volume=1.):
+    if isinstance(src, (str, Path)):
+        src = load_wav(src, sr=sr)
+    if isinstance(src, (list, np.ndarray, np.matrix)):
+        out_io = io.BytesIO()
+        save_wav_wavfile(src, out_io, sr=sr, volume=volume)
+    elif isinstance(src, bytes):
+        out_io = io.BytesIO(src)
+    elif isinstance(src, io.BytesIO):
+        out_io = src
+    else:
+        raise TypeError
+    return out_io
+
+
+def anything2wav(src, sr=_sr, volume=1.):
+    if isinstance(src, (list, np.ndarray, np.matrix)):
+        return np.array(src)
+    else:
+        bysio = anything2bytesio(src, sr=sr, volume=volume)
+        return load_wav_wavfile(bysio, sr=sr)
+
+
+def anything2bytes(src, sr=_sr, volume=1.):
+    if isinstance(src, bytes):
+        return src
+    else:
+        bysio = anything2bytesio(src, sr=sr, volume=volume)
+        return bysio.getvalue()
+
+
+if __name__ == "__main__":
+    print(__file__)
+    inpath = r"../hello.wav"
+    bys = anything2bytesio(inpath, sr=16000)
+    print(bys)
+    wav = anything2wav(bys, sr=16000)
+    print(wav)

docker/python37/src/aukit/audio_noise_remover.py

@@ -0,0 +1,192 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/11/23
+"""
+### audio_noise_remover
+语音降噪，降低环境噪声。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+import numpy as np
+import ctypes as ct
+from .audio_io import load_wav, save_wav
+from .audio_io import _sr
+import traceback
+
+
+def remove_noise(wav: np.array, sr=_sr, **kwargs):
+    """
+    谱减法去除背景噪声。
+    :param wav: 语音信号
+    :param sr: 采样率
+    :param kwargs:
+    :return: np.ndarray
+    """
+    x = wav
+    noise_span = kwargs.get("noise_span", (0, 100))
+    noise_wav = kwargs.get("noise_wav", None)
+    threshold = kwargs.get("threshold", 3)
+    beta = kwargs.get("beta", 0.002)
+    # 计算参数
+    unit_ = 20  # 每帧时长，单位ms
+    len_ = unit_ * sr // 1000  # 样本中帧的大小
+    PERC = 50  # 窗口重叠占帧的百分比
+    len1 = len_ * PERC // 100  # 重叠窗口
+    len2 = len_ - len1  # 非重叠窗口
+    # 设置默认参数
+    Thres = threshold
+    Expnt = 2.0
+    beta = beta
+    G = 0.9
+    # 初始化汉明窗
+    win = np.hamming(len_)
+    # normalization gain for overlap+add with 50% overlap
+    winGain = len2 / sum(win)
+
+    # Noise magnitude calculations - assuming that the first 5 frames is noise/silence
+    nFFT = 2 * 2 ** (nextpow2(len_))
+    noise_mean = np.zeros(nFFT)
+    if noise_wav is None:
+        sidx = noise_span[0] // unit_
+        eidx = noise_span[1] // unit_
+        for k in range(sidx, eidx):
+            noise_mean = noise_mean + abs(np.fft.fft(win * x[k * len_:(k + 1) * len_], nFFT))
+        noise_mu = noise_mean / (eidx - sidx)
+    else:
+        if "noise_span" in kwargs:
+            sidx = noise_span[0] // unit_
+            eidx = noise_span[1] // unit_
+        else:
+            sidx = 0
+            eidx = len(noise_wav) // unit_
+        for k in range(sidx, eidx):
+            noise_mean = noise_mean + abs(np.fft.fft(win * x[k * len_:(k + 1) * len_], nFFT))
+        noise_mu = noise_mean / (eidx - sidx)
+    # --- allocate memory and initialize various variables
+    k = 1
+    img = 1j
+    x_old = np.zeros(len1)
+    Nframes = len(x) // len2 - 1
+    xfinal = np.zeros(Nframes * len2)
+
+    # =========================    Start Processing   ===============================
+    for n in range(0, Nframes):
+        # Windowing
+        insign = win * x[k - 1:k + len_ - 1]
+        # compute fourier transform of a frame
+        spec = np.fft.fft(insign, nFFT)
+        # compute the magnitude
+        sig = abs(spec)
+
+        # save the noisy phase information
+        theta = np.angle(spec)
+        SNRseg = 10 * np.log10(np.linalg.norm(sig, 2) ** 2 / np.linalg.norm(noise_mu, 2) ** 2)
+
+        if Expnt == 1:  # 幅度谱
+            alpha = berouti1(SNRseg)
+        else:  # 功率谱
+            alpha = berouti(SNRseg)
+        #############
+        sub_speech = sig ** Expnt - alpha * noise_mu ** Expnt
+        # 当纯净信号小于噪声信号的功率时
+        diffw = sub_speech - beta * noise_mu ** Expnt
+
+        # beta negative components
+
+        z = find_index(diffw)
+        if len(z) > 0:
+            # 用估计出来的噪声信号表示下限值
+            sub_speech[z] = beta * noise_mu[z] ** Expnt
+            # --- implement a simple VAD detector --------------
+        if SNRseg < Thres:  # Update noise spectrum
+            noise_temp = G * noise_mu ** Expnt + (1 - G) * sig ** Expnt  # 平滑处理噪声功率谱
+            noise_mu = noise_temp ** (1 / Expnt)  # 新的噪声幅度谱
+        # flipud函数实现矩阵的上下翻转，是以矩阵的“水平中线”为对称轴
+        # 交换上下对称元素
+        sub_speech[nFFT // 2 + 1:nFFT] = np.flipud(sub_speech[1:nFFT // 2])
+        x_phase = (sub_speech ** (1 / Expnt)) * (
+                np.array([np.cos(x) for x in theta]) + img * (np.array([np.sin(x) for x in theta])))
+        # take the IFFT
+
+        xi = np.fft.ifft(x_phase).real
+        # --- Overlap and add ---------------
+        xfinal[k - 1:k + len2 - 1] = x_old + xi[0:len1]
+        x_old = xi[0 + len1:len_]
+        k = k + len2
+    return winGain * xfinal
+
+
+def remove_noise_os(inpath, outpath, **kwargs):
+    try:
+        wav, sr = load_wav(inpath, with_sr=True)
+        out = remove_noise(wav, sr, **kwargs)
+        save_wav(out, outpath, sr)
+    except Exception as e:
+        logger.info('Error path: {}'.format(inpath))
+        logger.info('Error info: {}'.format(e))
+        traceback.print_exc()
+
+
+class FloatBits(ct.Structure):
+    _fields_ = [
+        ('M', ct.c_uint, 23),
+        ('E', ct.c_uint, 8),
+        ('S', ct.c_uint, 1)
+    ]
+
+
+class Float(ct.Union):
+    _anonymous_ = ('bits',)
+    _fields_ = [
+        ('value', ct.c_float),
+        ('bits', FloatBits)
+    ]
+
+
+def nextpow2(x):
+    if x < 0:
+        x = -x
+    if x == 0:
+        return 0
+    d = Float()
+    d.value = x
+    if d.M == 0:
+        return d.E - 127
+    return d.E - 127 + 1
+
+
+def berouti(SNR):
+    if -5.0 <= SNR <= 20.0:
+        a = 4 - SNR * 3 / 20
+    elif SNR < -5.0:
+        a = 5
+    else:
+        a = 1
+    return a
+
+
+def berouti1(SNR):
+    if -5.0 <= SNR <= 20.0:
+        a = 3 - SNR * 2 / 20
+    elif SNR < -5.0:
+        a = 4
+    else:
+        a = 1
+    return a
+
+
+def find_index(x_list):
+    index_list = []
+    for i in range(len(x_list)):
+        if x_list[i] < 0:
+            index_list.append(i)
+    return index_list
+
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/aukit/audio_normalizer.py

@@ -0,0 +1,166 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/11/30
+"""
+### audio_normalizer
+语音正则化，去除音量低的音频段（去除静音），调节音量。
+语音正则化方法基于VAD的方法。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+from scipy.ndimage.morphology import binary_dilation
+from pathlib import Path
+from typing import Optional, Union
+import numpy as np
+import librosa
+import struct
+
+from .audio_io import Dict2Obj, _sr, _int16_max
+
+try:
+    import webrtcvad
+except ImportError as e:
+    logger.info("ImportError: {}".format(e))
+
+
+# Default hyperparameters
+default_hparams = Dict2Obj(dict(
+    int16_max=(2 ** 15) - 1,
+    ## Mel-filterbank
+    mel_window_length=25,  # In milliseconds
+    mel_window_step=10,  # In milliseconds
+    mel_n_channels=40,
+
+    ## Audio
+    sample_rate=16000,  # sampling_rate
+    # Number of spectrogram frames in a partial utterance
+    partials_n_frames=160,  # 1600 ms
+    # Number of spectrogram frames at inference
+    inference_n_frames=80,  # 800 ms
+
+    ## Voice Activation Detection
+    # Window size of the VAD. Must be either 10, 20 or 30 milliseconds.
+    # This sets the granularity of the VAD. Should not need to be changed.
+    vad_window_length=30,  # In milliseconds
+    # Number of frames to average together when performing the moving average smoothing.
+    # The larger this value, the larger the VAD variations must be to not get smoothed out.
+    vad_moving_average_width=8,
+    # Maximum number of consecutive silent frames a segment can have.
+    vad_max_silence_length=6,
+
+    ## Audio volume normalization
+    audio_norm_target_dBFS=-30,
+))
+
+
+def remove_silence(wav, sr=_sr, max_silence_ms=20):
+    """
+    去除语音中的静音。
+    :param wav:
+    :param sr:
+    :param max_silence_ms: 单位ms
+    :return:
+    """
+    # Compute the voice detection window size
+    wav = librosa.resample(wav, orig_sr=sr, target_sr=_sr)
+
+    vad_window_length = 20
+    vad_moving_average_width = 10
+
+    samples_per_window = (vad_window_length * _sr) // 1000
+
+    # Trim the end of the audio to have a multiple of the window size
+    wav = wav[:len(wav) - (len(wav) % samples_per_window)]
+
+    # Convert the float waveform to 16-bit mono PCM
+    pcm_wave = struct.pack("%dh" % len(wav), *(np.round(wav * _int16_max)).astype(np.int16))
+
+    # Perform voice activation detection
+    voice_flags = []
+    vad = webrtcvad.Vad(mode=3)
+    for window_start in range(0, len(wav), samples_per_window):
+        window_end = window_start + samples_per_window
+        voice_flags.append(vad.is_speech(pcm_wave[window_start * 2:window_end * 2], sample_rate=_sr))
+    voice_flags = np.array(voice_flags)
+
+    audio_mask = moving_average(voice_flags, vad_moving_average_width)
+    audio_mask = np.round(audio_mask).astype(np.bool)
+
+    # Dilate the voiced regions
+    audio_mask = binary_dilation(audio_mask, np.ones(max_silence_ms + 1))
+    audio_mask = np.repeat(audio_mask, samples_per_window)
+    out = wav[audio_mask == True]
+    out = librosa.resample(out, orig_sr=_sr, target_sr=sr)
+    return out
+
+
+def tune_volume(wav, target_dBFS, increase_only=False, decrease_only=False):
+    """
+    调节音量大小。
+    :param wav:
+    :param target_dBFS: 目标音量。
+    :param increase_only: 是否只是增加音量。
+    :param decrease_only: 是否只是降低音量。
+    :return:
+    """
+    if increase_only and decrease_only:
+        raise ValueError("Both increase only and decrease only are set")
+    rms = np.sqrt(np.mean((wav * _int16_max) ** 2))
+    wave_dBFS = 20 * np.log10(rms / _int16_max)
+    dBFS_change = target_dBFS - wave_dBFS
+    if dBFS_change < 0 and increase_only or dBFS_change > 0 and decrease_only:
+        return wav
+    return wav * (10 ** (dBFS_change / 20))
+
+
+def preprocess_wav(fpath_or_wav: Union[str, Path, np.ndarray], source_sr: Optional[int] = None, hparams=None):
+    """
+    预处理语音，去除静音和设置音量。
+    :param fpath_or_wav:
+    :param source_sr:
+    :param hparams:
+    :return:
+    """
+    hparams = hparams or default_hparams
+    # Load the wav from disk if needed
+    if isinstance(fpath_or_wav, str) or isinstance(fpath_or_wav, Path):
+        wav, source_sr = librosa.load(fpath_or_wav, sr=None)
+    else:
+        wav = fpath_or_wav
+
+    # Resample the wav if needed
+    if source_sr is not None and source_sr != hparams.sample_rate:
+        wav = librosa.resample(wav, source_sr, hparams.sample_rate)
+
+    # Apply the preprocessing: normalize volume and shorten long silences
+    wav = tune_volume(wav, hparams.audio_norm_target_dBFS, increase_only=True)
+    wav = trim_long_silences(wav, hparams=hparams)
+
+    return wav
+
+
+# Smooth the voice detection with a moving average
+def moving_average(array, width):
+    array_padded = np.concatenate((np.zeros((width - 1) // 2), array, np.zeros(width // 2)))
+    ret = np.cumsum(array_padded, dtype=float)
+    ret[width:] = ret[width:] - ret[:-width]
+    return ret[width - 1:] / width
+
+
+def trim_long_silences(wav, hparams):
+    """去除语音中的静音。(兼容历史版本)"""
+    hparams = hparams or default_hparams
+
+    wav = remove_silence(wav,
+                         sr=hparams.sample_rate,
+                         max_silence_ms=hparams.vad_max_silence_length)
+    return wav
+
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/aukit/audio_player.py

@@ -0,0 +1,59 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/11/23
+"""
+### audio_player
+语音播放，传入文件名播放，播放wave数据，播放bytes数据。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+import sys
+import time
+
+from .audio_io import anything2bytesio, anything2wav
+from .audio_io import _sr
+
+try:
+    import wave
+    from pyaudio import PyAudio
+except ImportError as e:
+    logger.info("ImportError: {}".format(e))
+
+
+def play_audio(src=None, sr=_sr, volume=1.):
+    chunk = 1024  # 2014kb
+    bytesio = anything2bytesio(src, sr=sr, volume=volume)
+    wf = wave.open(bytesio, "rb")
+    p = PyAudio()
+    stream = p.open(format=p.get_format_from_width(wf.getsampwidth()), channels=wf.getnchannels(),
+                    rate=wf.getframerate(), output=True)
+    t0 = time.time()
+    while True:
+        data = wf.readframes(chunk)
+        if data == b"":
+            break
+        stream.write(data)
+    stream.stop_stream()  # 停止数据流
+    stream.close()
+    p.terminate()  # 关闭 PyAudio
+    t = time.time() - t0
+    logger.info("play audio done, playing {:.2f} seconds.".format(t))
+
+
+def play_sound(src, sr=_sr, **kwargs):
+    import sounddevice as sd
+    data = anything2wav(src, sr=sr)
+    t0 = time.time()
+    sd.play(data, sr, **kwargs)
+    sd.wait()
+    t = time.time() - t0
+    logger.info("play sound done, playing {:.2f} seconds.".format(t))
+
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/aukit/audio_spectrogram.py

@@ -0,0 +1,261 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/11/30
+"""
+### audio_spectrogram
+语音频谱，语音转线性频谱，语音转梅尔频谱。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+from scipy.signal import lfilter
+import math
+import numpy as np
+import librosa
+
+from .audio_io import Dict2Obj
+
+# Default hyperparameters
+default_hparams = Dict2Obj(dict(
+    # Audio
+    mel_basis=None,
+    inv_mel_basis=None,
+    num_mels=80,  # Number of mel-spectrogram channels and local conditioning dimensionality
+
+    # Mel spectrogram
+    n_fft=800,  # 800,  # Extra window size is filled with 0 paddings to match this parameter
+    hop_size=200,  # For 16000Hz, 200 = 12.5 ms (0.0125 * sample_rate)
+    win_size=800,  # For 16000Hz, 800 = 50 ms (If None, win_size = n_fft) (0.05 * sample_rate)
+    sample_rate=16000,  # 16000Hz (corresponding to librispeech) (sox --i <filename>)
+    preemphasize=True,  # whether to apply filter
+    preemphasis=0.97,  # filter coefficient.
+    center=True,
+
+    # Whether to normalize mel spectrograms to some predefined range (following below parameters)
+    signal_normalization=True,
+    allow_clipping_in_normalization=True,  # True,  # Only relevant if mel_normalization = True
+    symmetric_mels=True,  # True,
+    max_abs_value=4.,  # 4.,
+
+    # Limits
+    min_level_db=-100,
+    ref_level_db=20,
+    fmin=55,
+    # Set this to 55 if your speaker is male! if female, 95 should help taking off noise. (To
+    # test depending on dataset. Pitch info: male~[65, 260], female~[100, 525])
+    fmax=7600,  # To be increased/reduced depending on data.
+
+    # Griffin Lim
+    power=1.5,
+))
+
+
+def linear_spectrogram(y, hparams=None):
+    hparams = hparams or default_hparams
+    D = stft(pre_emphasis(y, hparams), hparams)
+    S = amp_to_db(np.abs(D)) - hparams.ref_level_db
+    if hparams.signal_normalization:
+        return normalize(S, hparams)
+    else:
+        return S
+
+
+def mel_spectrogram(y, hparams=None):
+    hparams = hparams or default_hparams
+    D = stft(pre_emphasis(y, hparams), hparams)
+    S = amp_to_db(linear_to_mel(np.abs(D), hparams))
+    if hparams.signal_normalization:
+        return normalize(S, hparams)
+    return S
+
+
+def mel_spectrogram_feature(wav, hparams=None):
+    hparams = hparams or default_hparams
+    """
+    Derives a mel spectrogram ready to be used by the encoder from a preprocessed audio waveform.
+    Note: this not a log-mel spectrogram.
+    """
+    frames = librosa.feature.melspectrogram(
+        wav,
+        hparams.sample_rate,
+        n_fft=hparams.n_fft,
+        hop_length=hparams.hop_size,
+        n_mels=hparams.num_mels
+    )
+    return amp_to_db(frames.astype(np.float32))
+
+
+def mel2linear_spectrogram(mel_spectrogram, hparams=None):
+    """Converts mel spectrogram to linear spectrogram"""
+    hparams = hparams or default_hparams
+
+    if hparams.signal_normalization:
+        D = denormalize(mel_spectrogram, hparams)
+    else:
+        D = mel_spectrogram
+
+    D = mel_to_linear(db_to_amp(D - hparams.ref_level_db), hparams)  # Convert back to linear
+    S = amp_to_db(np.abs(D)) - hparams.ref_level_db
+
+    if hparams.signal_normalization:
+        return normalize(S, hparams)
+    return S
+
+
+def linear2mel_spectrogram(linear_spectrogram, hparams=None):
+    """Converts linear spectrogram to mel spectrogram"""
+    hparams = hparams or default_hparams
+
+    if hparams.signal_normalization:
+        D = denormalize(linear_spectrogram, hparams)
+    else:
+        D = linear_spectrogram
+
+    D = db_to_amp(D + hparams.ref_level_db)  # Convert back to linear
+
+    S = amp_to_db(linear_to_mel(np.abs(D), hparams)) - hparams.ref_level_db
+
+    if hparams.signal_normalization:
+        return normalize(S, hparams)
+    return S
+
+
+def label_2_float(x, bits):
+    return 2 * x / (2 ** bits - 1.) - 1.
+
+
+def float_2_label(x, bits):
+    assert abs(x).max() <= 1.0
+    x = (x + 1.) * (2 ** bits - 1) / 2
+    return x.clip(0, 2 ** bits - 1)
+
+
+def load_wav(path, sr):
+    return librosa.load(path, sr=sr)[0]
+
+
+def save_wav(x, path, sr):
+    librosa.output.write_wav(path, x.astype(np.float32), sr=sr)
+
+
+def split_signal(x):
+    unsigned = x + 2 ** 15
+    coarse = unsigned // 256
+    fine = unsigned % 256
+    return coarse, fine
+
+
+def combine_signal(coarse, fine):
+    return coarse * 256 + fine - 2 ** 15
+
+
+def encode_16bits(x):
+    return np.clip(x * 2 ** 15, -2 ** 15, 2 ** 15 - 1).astype(np.int16)
+
+
+def linear_to_mel(spectrogram, hparams=None):
+    hparams = hparams or default_hparams
+    if hparams.mel_basis is None:
+        mel_basis = build_mel_basis(hparams)
+    else:
+        mel_basis = hparams.mel_basis
+    return np.dot(mel_basis, spectrogram)
+
+
+def mel_to_linear(mel_spectrogram, hparams=None):
+    hparams = hparams or default_hparams
+    if hparams.inv_mel_basis is None:
+        _inv_mel_basis = np.linalg.pinv(build_mel_basis(hparams))
+    else:
+        _inv_mel_basis = hparams.inv_mel_basis
+    return np.maximum(1e-10, np.dot(_inv_mel_basis, mel_spectrogram))
+
+
+def build_mel_basis(hparams=None):
+    hparams = hparams or default_hparams
+    return librosa.filters.mel(hparams.sample_rate, hparams.n_fft, n_mels=hparams.num_mels, fmin=hparams.fmin)
+
+
+def normalize(S, hparams=None):
+    hparams = hparams or default_hparams
+    ma = hparams.max_abs_value
+    mi = hparams.min_level_db
+    if hparams.allow_clipping_in_normalization:
+        if hparams.symmetric_mels:
+            return np.clip((2 * ma) * ((S - mi) / (-mi)) - ma, -ma, ma)
+        else:
+            return np.clip(ma * ((S - mi) / (-mi)), 0, ma)
+    else:
+        assert S.max() <= 0 and S.min() - mi >= 0
+        if hparams.symmetric_mels:
+            return (2 * ma) * ((S - mi) / (-mi)) - ma
+        else:
+            return ma * ((S - mi) / (-mi))
+
+
+def denormalize(D, hparams=None):
+    hparams = hparams or default_hparams
+    ma = hparams.max_abs_value
+    mi = hparams.min_level_db
+    if hparams.allow_clipping_in_normalization:
+        if hparams.symmetric_mels:
+            return ((np.clip(D, -ma, ma) + ma) * -mi / (2 * ma)) + mi
+        else:
+            return (np.clip(D, 0, ma) * -mi / ma) + mi
+    else:
+        if hparams.symmetric_mels:
+            return ((D + ma) * -mi / (2 * ma)) + mi
+        else:
+            return (D * -mi / ma) + mi
+
+
+def amp_to_db(x):
+    return 20 * np.log10(np.maximum(1e-5, x))
+
+
+def db_to_amp(x):
+    return np.power(10.0, x * 0.05)
+
+
+def stft(y, hparams=None):
+    hparams = hparams or default_hparams
+    return librosa.stft(y=y, n_fft=hparams.n_fft, hop_length=hparams.hop_size, win_length=hparams.win_size,
+                        center=hparams.center)
+
+
+def pre_emphasis(x, hparams=None):
+    hparams = hparams or default_hparams
+    if hparams.preemphasize:
+        return lfilter([1, -hparams.preemphasis], [1], x)
+    else:
+        return x
+
+
+def de_emphasis(x, hparams=None):
+    hparams = hparams or default_hparams
+    if hparams.preemphasize:
+        return lfilter([1], [1, -hparams.preemphasis], x)
+    else:
+        return x
+
+
+def encode_mu_law(x, mu):
+    mu = mu - 1
+    fx = np.sign(x) * np.log(1 + mu * np.abs(x)) / np.log(1 + mu)
+    return np.floor((fx + 1) / 2 * mu + 0.5)
+
+
+def decode_mu_law(y, mu, from_labels=True):
+    if from_labels:
+        y = label_2_float(y, math.log2(mu))
+    mu = mu - 1
+    x = np.sign(y) / mu * ((1 + mu) ** np.abs(y) - 1)
+    return x
+
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/aukit/audio_tuner.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @author: KDD
+# @time: 2018-11-10
+"""
+### audio_tuner
+语音调整，调整语速，调整音高。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+from pydub import AudioSegment
+from scipy.io import wavfile
+import numpy as np
+import io
+from .audio_io import anything2bytesio, anything2wav
+from .audio_io import _sr
+
+
+def tune_speed(src=None, sr=_sr, rate=1., out_type=np.ndarray):
+    """
+    变语速
+    rate = win / (bar - cro)
+    :param src:
+    :param rate:
+    :return:
+    """
+    song = AudioSegment.from_wav(anything2bytesio(src, sr=sr))
+    n_song = len(song)
+    win = 50
+    bar = 100
+    cro = int(bar - win / rate)
+
+    segs = []
+    for i in range(0, n_song - bar, win):
+        segs.append(song[i: i + bar])
+
+    out_song = segs[0]
+    for seg in segs[1:]:
+        out_song = out_song.append(seg, cro)
+
+    io_out = io.BytesIO()
+    out_song.export(io_out, format="wav")
+
+    if out_type is np.ndarray:
+        return anything2wav(io_out.getvalue(), sr=sr)
+    else:
+        return anything2bytesio(io_out.getvalue(), sr=sr)
+
+
+def tune_pitch(src=None, sr=_sr, rate=1., out_type=np.ndarray):
+    """
+    变音调
+    :param io_in:
+    :param rate:
+    :return:
+    """
+    frate, wavdata = wavfile.read(anything2bytesio(src, sr=sr))
+
+    cho_ids = [int(w) for w in np.arange(0, len(wavdata), rate)]
+    out_wavdata = wavdata[cho_ids]
+
+    io_out = io.BytesIO()
+    wavfile.write(io_out, frate, out_wavdata)
+    io_out = tune_speed(io_out, rate=1 / rate, out_type=io.BytesIO)
+    if out_type is np.ndarray:
+        return anything2wav(io_out.getvalue(), sr=sr)
+    else:
+        return anything2bytesio(io_out.getvalue(), sr=sr)
+
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/aukit/audio_world.py

@@ -0,0 +1,170 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/12/15
+"""
+### audio_world
+world声码器，提取语音的基频、频谱包络和非周期信号，频谱转为语音。调音高，调机器人音。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+import numpy as np
+from .audio_io import _sr
+
+try:
+    import pyworld as pw
+except ImportError as e:
+    logger.info("ImportError: {}".format(e))
+
+
+def world_spectrogram_default(wav, sr=_sr):
+    """默认参数的world声码器语音转为特征频谱。"""
+    # f0 : ndarray
+    #     F0 contour. 基频等高线
+    # sp : ndarray
+    #     Spectral envelope. 频谱包络
+    # ap : ndarray
+    #     Aperiodicity. 非周期性
+    f0, sp, ap = pw.wav2world(wav.astype(np.double), sr)  # use default options
+    return f0, sp, ap
+
+
+def inv_world_spectrogram_default(f0, sp, ap, sr=_sr):
+    """默认参数的world声码器特征频谱转为语音。"""
+    y = pw.synthesize(f0, sp, ap, sr)
+    return y
+
+
+def world_spectrogram(wav, sr=_sr, dim_num=32, **kwargs):
+    """world声码器语音转为频谱。"""
+    # 分布提取参数
+    frame_period = kwargs.get("frame_period", pw.default_frame_period)
+    f0_floor = kwargs.get("f0_floor", pw.default_f0_floor)
+    f0_ceil = kwargs.get("f0_ceil", pw.default_f0_ceil)
+    fft_size = kwargs.get("fft_size", pw.get_cheaptrick_fft_size(sr, f0_floor))
+    ap_threshold = kwargs.get("ap_threshold", 0.85)
+    f0_extractor = kwargs.get("f0_extractor", "dio")
+    x = wav.astype(np.double)
+    if f0_extractor == "dio":
+        # 使用DIO算法计算音频的基频F0
+        f0, t = pw.dio(x, sr, f0_floor=f0_floor, f0_ceil=f0_ceil)
+    elif f0_extractor == "harvest":
+        f0, t = pw.harvest(x, sr, f0_floor=f0_floor, f0_ceil=f0_ceil, frame_period=frame_period)
+    else:
+        f0, t = f0_extractor(x, sr, f0_floor=f0_floor, f0_ceil=f0_ceil, frame_period=frame_period)
+
+    # 使用CheapTrick算法计算音频的频谱包络
+    sp = pw.cheaptrick(x, f0, t, sr, f0_floor=f0_floor, fft_size=fft_size)
+    # SP降维
+    sp_enc = pw.code_spectral_envelope(sp, sr, number_of_dimensions=dim_num)
+
+    # 计算aperiodic参数
+    ap = pw.d4c(x, f0, t, sr, threshold=ap_threshold, fft_size=fft_size)
+    # AP降维
+    ap_enc = pw.code_aperiodicity(ap, sr)
+    return f0, sp_enc, ap_enc
+
+
+def inv_world_spectrogram(f0, sp, ap, sr=_sr, **kwargs):
+    """world声码器频谱转为语音。"""
+    frame_period = kwargs.get("frame_period", pw.default_frame_period)
+    f0_floor = kwargs.get("f0_floor", pw.default_f0_floor)
+    fft_size = kwargs.get("fft_size", pw.get_cheaptrick_fft_size(sr, f0_floor))
+    sp_dec = pw.decode_spectral_envelope(sp, sr, fft_size=fft_size)
+    ap_dec = pw.decode_aperiodicity(ap, sr, fft_size=fft_size)
+    y = pw.synthesize(f0, sp_dec, ap_dec, sr, frame_period=frame_period)
+    return y
+
+
+def change_voice(wav, sr=_sr, mode="tune_pitch", alpha=1., fix=True):
+    """
+    变声。
+    :param wav:
+    :param sr:
+    :param mode:
+    :param alpha:
+    :param fix:
+    :return:
+    """
+    x = wav.astype(np.double)
+    s = world_spectrogram_default(x, sr=sr)
+    if mode == "tune_pitch":
+        s = tune_pitch(*s, rate=alpha, fix=fix)
+    elif mode == "tune_robot":
+        s = tune_robot(*s, rate=alpha, fix=fix)
+    elif mode == "assign_pitch":
+        s = assign_pitch(*s, base=alpha, fix=fix)
+    elif mode == "assign_robot":
+        s = assign_robot(*s, base=alpha, fix=fix)
+    else:
+        logger.info("ModeError: mode={}".format(mode))
+        raise TypeError
+    y = inv_world_spectrogram_default(*s, sr=sr)
+    return y
+
+
+def tune_pitch(f0, sp, ap, rate=1., fix=False):
+    """调音高"""
+    f0_out = f0 * rate
+    if fix:
+        sp = fix_sp(sp, rate)
+    return f0_out, sp, ap
+
+
+def tune_robot(f0, sp, ap, rate=1., fix=False):
+    """调机器人音"""
+    tmp = f0[f0 > 0]
+    if len(tmp) >= 1:
+        m = np.percentile(tmp, 61.8)
+    else:
+        m = 1
+    f0_out = np.ones_like(f0) * m * rate
+    if fix:
+        sp = fix_sp(sp, rate)
+    return f0_out, sp, ap
+
+
+def assign_pitch(f0, sp, ap, base=250, fix=False):
+    """指定音高"""
+    tmp = f0[f0 > 0]
+    if len(tmp) >= 1:
+        m = np.percentile(tmp, 61.8)
+    else:
+        m = 1
+    rate = base / m
+    f0_out = f0 * rate
+    if fix:
+        sp = fix_sp(sp, rate)
+    return f0_out, sp, ap
+
+
+def assign_robot(f0, sp, ap, base=250, fix=False):
+    """指定音高的机器人音"""
+    tmp = f0[f0 > 0]
+    if len(tmp) >= 1:
+        m = np.percentile(tmp, 61.8)
+    else:
+        m = 1
+    rate = base / m
+    f0_out = np.ones_like(f0) * m * rate
+    if fix:
+        sp = fix_sp(sp, rate)
+    return f0_out, sp, ap
+
+
+def fix_sp(sp, rate=1.):
+    """修调频谱包络"""
+    sp_dim = sp.shape[1]
+    sp_out = np.zeros_like(sp)
+    for f in range(sp_dim):
+        f2 = min(sp_dim - 1, int(f / rate))
+        sp_out[:, f] = sp[:, f2]
+    return sp_out
+
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/requirements.txt

@@ -0,0 +1,11 @@
+webrtcvad
+pydub
+lws
+scipy
+numpy
+librosa
+pyworld
+tensorflow<=1.13.1
+pyaudio
+sounddevice
+dotmap

docker/python37/src/run_local.py

@@ -0,0 +1,212 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/12/1
+"""
+local
+"""
+from pathlib import Path
+from functools import partial
+from multiprocessing.pool import Pool
+from matplotlib import pyplot as plt
+# from tqdm import tqdm
+import collections as clt
+import os
+import re
+import json
+import numpy as np
+import shutil
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+
+def run_spectrogram():
+    from aukit import audio_spectrogram as asp
+    from aukit import audio_griffinlim as agf
+    from aukit import audio_io as aio
+    from aukit.audio_player import play_audio
+    inpath = r"E:/data/temp/01.wav"
+    wav, sr = aio.load_wav(inpath, with_sr=True)
+    print(wav.shape, sr)
+    play_audio(wav, sr)
+
+    lin_gf = agf.linear_spectrogram(wav)
+    wav_gf = agf.inv_linear_spectrogram(lin_gf)
+    play_audio(wav_gf, sr)
+
+    mel_sp = asp.mel_spectrogram(wav)
+    mel_sp = asp.mel2linear_spectrogram(mel_sp)
+    wav_sp = agf.inv_linear_spectrogram(mel_sp)
+    play_audio(wav_sp, sr)
+
+    linear_gf = agf.linear_spectrogram(wav)
+    mel_lin = agf.linear2mel_spectrogram(linear_gf)
+    linear_mel = agf.mel2linear_spectrogram(mel_lin)
+    wav_2 = agf.inv_linear_spectrogram(linear_mel)
+
+    mel_sp = asp.mel_spectrogram(wav)
+    mel_fea = asp.mel_spectrogram_feature(wav)
+
+    # plt.figure()
+    # plt.subplot("311")
+    # plt.pcolor(linear)
+    # plt.subplot("312")
+    # plt.pcolor(linear2)
+    # plt.subplot("313")
+    # plt.pcolor(mel_fea)
+    # plt.show()
+
+    wav_ms = agf.inv_mel_spectrogram(mel_sp)
+    wav_mf = agf.inv_mel_spectrogram(mel_fea)
+    play_audio(wav_ms, sr)
+    play_audio(wav_mf, sr)
+
+
+def run_world():
+    from aukit import audio_world as awd
+    from aukit import audio_player as apr
+    from aukit import audio_io as aio
+    inpath = r"E:/data/temp/01.wav"
+    # sr, x = wavfile.read(inpath)
+    x, sr = aio.load_wav(inpath, with_sr=True)
+    f0, sp, ap = awd.world_spectrogram(x, sr)
+    y = awd.inv_world_spectrogram(f0, sp, ap, sr)
+
+    apr.play_audio(x, sr)
+    apr.play_audio(y, sr)
+
+
+def create_readme():
+    from aukit import readme_docs
+    docs = []
+    with open("README.md", "wt", encoding="utf8") as fout:
+        for doc in readme_docs:
+            fout.write(doc)
+            docs.append(doc)
+    return "".join(docs)
+
+
+def run_tuner():
+    import aukit
+    from aukit.audio_tuner import tune_speed, tune_pitch
+    inpath = r"hello.wav"
+    aukit.anything2bytes(inpath)
+    aukit.anything2wav(inpath)
+    aukit.anything2bytesio(inpath)
+    bys = tune_speed(inpath, sr=16000, rate=0.5, out_type=None)
+    print(bys)
+    wav = tune_pitch(bys, sr=16000, rate=1, out_type=None)
+    print(wav)
+    aukit.play_audio(wav)
+
+
+def run_noise_remover():
+    import aukit
+    inpath = r"hello.wav"
+    wav = aukit.load_wav(inpath)
+    out = aukit.remove_noise(wav)
+    aukit.play_audio(out)
+
+
+def run_player():
+    import aukit
+    inpath = Path(r"E:\data\aliaudio\examples\ali_Aibao_000001.wav")
+    wav = aukit.load_wav(inpath, sr=16000)
+    wav = aukit.change_voice(wav, mode="assign_pitch", alpha=200)
+    aukit.play_audio(wav, volume=0.5)
+
+
+def run_aukit():
+    import time
+    t0 = time.time()
+    from aukit.audio_io import __doc__ as io_doc
+    from aukit.audio_editor import __doc__ as editor_doc
+    from aukit.audio_tuner import __doc__ as tuner_doc
+    from aukit.audio_player import __doc__ as player_doc
+    from aukit.audio_noise_remover import __doc__ as noise_remover_doc
+    from aukit.audio_normalizer import __doc__ as normalizer_doc
+    from aukit.audio_spectrogram import __doc__ as spectrogram_doc
+    from aukit.audio_griffinlim import __doc__ as griffinlim_doc
+    from aukit.audio_changer import __doc__ as changer_doc
+    from aukit.audio_cli import __doc__ as cli_doc
+    from aukit.audio_world import __doc__ as world_doc
+    t1 = time.time()
+    print(t1 - t0)
+
+
+def compare_hparams():
+    from aukit.audio_griffinlim import default_hparams as gfhp
+    from aukit.audio_spectrogram import default_hparams as sphp
+    a = set(gfhp.items()) - set(sphp.items())
+    b = set(sphp.items()) - set(gfhp.items())
+    print(a)
+    print(b)
+
+
+def run_normalizer():
+    import aukit
+    from aukit.audio_player import play_sound
+    from aukit import audio_normalizer as ano
+    inpath = r"hello.wav"
+    wav, sr = aukit.load_wav(inpath, with_sr=True)
+    out = ano.remove_silence(wav)
+    out = ano.tune_volume(wav, target_dBFS=-10)
+    play_sound(out, sr)
+
+
+def run_editor():
+    import aukit
+    from aukit.audio_player import play_sound, play_audio
+    from aukit import audio_editor as aed
+    inpath = r"hello.wav"
+    wav, sr = aukit.load_wav(inpath, with_sr=True)
+    aud = aed.wav2audiosegment(wav, sr)
+    out = aed.strip_audio(aud)
+    wav = aed.audiosegment2wav(out)
+
+    out = aed.remove_silence_wave(wav, sr=sr)
+    out = aed.strip_silence_wave(out, sr=sr)
+
+    print(len(wav), len(out))
+    play_audio(out, sr)
+
+from aukit.audio_editor import convert_format_os
+def convert_format(x):
+    return convert_format_os(**x)
+
+def run_cli():
+    from aukit.audio_cli import pool_jobs
+
+    from pathlib import Path
+
+
+    indir = Path(r"E:\lab\zhrtvc\data\samples\aishell")
+    outdir = Path(r"E:\lab\zhrtvc\data\samples_wav\aishell")
+
+    kw_lst = []
+    for inpath in indir.glob("**/*.mp3"):
+        parts = inpath.parent.relative_to(indir).parts
+        name = "{}.{}".format(inpath.stem, 'wav')
+        outpath = outdir.joinpath(*parts, name)
+        outpath.parent.mkdir(exist_ok=True, parents=True)
+        kw = dict(inpath=str(inpath), outpath=str(outpath), in_format=None, out_format='wav')
+        kw_lst.append(kw)
+
+    pool_jobs(func=convert_format, n_process=14, kwargs_list=kw_lst, tqdm_desc='convert_format')
+
+
+
+if __name__ == "__main__":
+    print(__file__)
+    # run_spectrogram()
+    # run_world()
+    # create_readme()
+    # run_tuner()
+    # run_noise_remover()
+    # run_player()
+    # run_aukit()
+    # compare_hparams()
+    # run_normalizer()
+    # run_editor()
+    # run_cli()

docker/python37/src/setup.py

@@ -0,0 +1,91 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/12/15
+"""
+语音处理工具箱。
+生成whl格式安装包：python setup.py bdist_wheel
+
+直接上传pypi：python setup.py sdist upload
+
+用twine上传pypi：
+生成安装包：python setup.py sdist
+上传安装包：twine upload [package path]
+
+注意：需要在home目录下建立.pypirc配置文件，文件内容格式：
+[distutils]
+index-servers=pypi
+
+[pypi]
+repository = https://upload.pypi.org/legacy/
+username: admin
+password: admin
+"""
+
+from setuptools import setup, find_packages
+import os
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(os.path.splitext(os.path.basename(__name__))[0])
+install_requires = ['librosa', 'pydub', 'scipy', 'numpy', 'dotmap']
+requires = ['tensorflow<=1.15.2', 'pyaudio', 'webrtcvad', 'lws', 'sounddevice', 'pyworld']
+
+
+def create_readme():
+    from aukit import readme_docs
+    docs = []
+    with open("README.md", "wt", encoding="utf8") as fout:
+        for doc in readme_docs:
+            fout.write(doc.replace("\n", "\n\n"))
+            docs.append(doc)
+    return "".join(docs)
+
+
+def pip_install():
+    for pkg in install_requires + requires:
+        try:
+            os.system("pip install {}".format(pkg))
+        except Exception as e:
+            logger.info("pip install {} failed".format(pkg))
+
+
+pip_install()
+aukit_doc = create_readme()
+from aukit import __version__ as aukit_version
+
+setup(
+    name="aukit",
+    version=aukit_version,
+    author="kuangdd",
+    author_email="kuangdd@foxmail.com",
+    description="audio toolkit",
+    long_description=aukit_doc,
+    long_description_content_type="text/markdown",
+    url="https://github.com/KuangDD/aukit",
+    packages=find_packages(exclude=['contrib', 'docs', 'test*']),
+    install_requires=install_requires,  # 指定项目最低限度需要运行的依赖项
+    python_requires='>=3.5',  # python的依赖关系
+    package_data={
+        'info': ['README.md', 'requirements.txt'],
+    },  # 包数据，通常是与软件包实现密切相关的数据
+    classifiers=[
+        'Intended Audience :: Developers',
+        'Topic :: Software Development :: Build Tools',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        "Operating System :: OS Independent",
+    ],
+    entry_points={
+        'console_scripts': [
+            'auplay = aukit.audio_cli:play_audio_cli',
+            'aunoise = aukit.audio_cli:remove_noise_cli',
+            'auformat = aukit.audio_cli:convert_format_cli'
+        ]
+    }
+)
+
+if __name__ == "__main__":
+    print(__file__)

docker/python37/src/test.py

@@ -0,0 +1,205 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2020/2/23
+"""
+"""
+import time
+import numpy as np
+from aukit import _sr
+
+# 普
+_wav = np.array([
+    0.05, 0.04, 0.03, 0.02, 0.01, 0.0, 0.0, 0.0, 0.0, -0.01, -0.01, -0.01, -0.01, 0.0, 0.0, 0.0, 0.01, 0.02, 0.03, 0.04,
+    0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.11, 0.11, 0.11, 0.1, 0.09, 0.08, 0.06, 0.04, 0.02, 0.0, -0.03, -0.06, -0.09,
+    -0.11, -0.14, -0.18, -0.2, -0.23, -0.26, -0.28, -0.3, -0.32, -0.33, -0.34, -0.34, -0.33, -0.32, -0.3, -0.28, -0.24,
+    -0.21, -0.18, -0.14, -0.09, -0.04, 0.0, 0.03, 0.07, 0.1, 0.13, 0.15, 0.17, 0.19, 0.2, 0.21, 0.21, 0.2, 0.19, 0.18,
+    0.17, 0.16, 0.15, 0.14, 0.13, 0.12, 0.11, 0.1, 0.09, 0.08, 0.08, 0.07, 0.07, 0.06, 0.05, 0.05, 0.04, 0.03, 0.03,
+    0.02, 0.01, 0.01, 0.0, 0.0, 0.0, 0.0, -0.01, -0.01, 0.0, 0.0, 0.0, 0.0, 0.01, 0.02, 0.04, 0.05, 0.06, 0.07, 0.09,
+    0.1, 0.11, 0.12, 0.12, 0.12, 0.12, 0.11, 0.11, 0.1, 0.09, 0.07, 0.05, 0.03, 0.01, -0.01, -0.03, -0.05, -0.08, -0.1,
+    -0.12, -0.15, -0.17, -0.2, -0.22, -0.24, -0.27, -0.29, -0.31, -0.32, -0.33, -0.34, -0.34, -0.33, -0.32, -0.3, -0.27,
+    -0.24, -0.2, -0.16, -0.12, -0.07, -0.02, 0.01, 0.05, 0.09, 0.13, 0.15, 0.18, 0.2, 0.21, 0.22, 0.22, 0.22, 0.22,
+    0.21, 0.19, 0.18, 0.17, 0.16, 0.15, 0.13, 0.12, 0.11, 0.1, 0.1, 0.09, 0.08, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03,
+    0.02, 0.01, 0.0, -0.01, -0.02, -0.02, -0.03, -0.04, -0.05, -0.05, -0.04, -0.04, -0.03, -0.02, -0.01, 0.0, 0.02,
+    0.03, 0.05, 0.08, 0.09, 0.11, 0.13, 0.14, 0.15, 0.16, 0.16, 0.16, 0.15, 0.14, 0.13, 0.11, 0.09, 0.07, 0.04, 0.02,
+    0.0, -0.02, -0.04, -0.07, -0.09, -0.11, -0.13, -0.15, -0.18, -0.2, -0.22, -0.25, -0.28, -0.3, -0.32, -0.34, -0.35,
+    -0.36, -0.36, -0.35, -0.34, -0.33, -0.31, -0.27, -0.24, -0.19, -0.15, -0.1, -0.04, 0.0, 0.04, 0.09, 0.13, 0.16,
+    0.19, 0.21, 0.23, 0.25, 0.25, 0.25, 0.25, 0.25, 0.23, 0.22, 0.2, 0.19, 0.17, 0.15, 0.13, 0.12, 0.11, 0.1, 0.08,
+    0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01, 0.0, -0.01, -0.03, -0.04, -0.05, -0.06, -0.07, -0.08, -0.08, -0.07, -0.07,
+    -0.06, -0.05, -0.03, -0.01, 0.0, 0.02, 0.05, 0.08, 0.1, 0.12, 0.15, 0.17, 0.18, 0.19, 0.2, 0.2, 0.19, 0.18, 0.17,
+    0.16, 0.14, 0.11, 0.09, 0.06, 0.04, 0.02, 0.0, -0.02, -0.04, -0.06, -0.08, -0.1, -0.12, -0.13, -0.15, -0.18, -0.21,
+    -0.22, -0.24, -0.27, -0.31, -0.33, -0.34, -0.36, -0.38, -0.39, -0.39, -0.38, -0.37, -0.35, -0.31, -0.27, -0.22,
+    -0.17, -0.12, -0.06, 0.0, 0.04, 0.09, 0.14, 0.19, 0.23, 0.25, 0.27, 0.29, 0.3, 0.29, 0.28, 0.28, 0.26, 0.24, 0.22,
+    0.2, 0.18, 0.15, 0.13, 0.11, 0.1, 0.08, 0.07, 0.05, 0.04, 0.04, 0.03, 0.02, 0.0, 0.0, 0.0, -0.02, -0.03, -0.04,
+    -0.05, -0.07, -0.08, -0.08, -0.09, -0.09, -0.08, -0.07, -0.06, -0.05, -0.03, 0.0, 0.02, 0.05, 0.07, 0.1, 0.13, 0.16,
+    0.18, 0.2, 0.22, 0.23, 0.23, 0.22, 0.21, 0.21, 0.18, 0.16, 0.14, 0.11, 0.08, 0.06, 0.03, 0.01, -0.01, -0.03, -0.05,
+    -0.06, -0.08, -0.09, -0.11, -0.13, -0.14, -0.16, -0.17, -0.21, -0.22, -0.25, -0.28, -0.31, -0.34, -0.36, -0.38,
+    -0.4, -0.42, -0.42, -0.4, -0.38, -0.36, -0.34, -0.29, -0.23, -0.18, -0.12, -0.06, 0.0, 0.06, 0.11, 0.16, 0.21, 0.25,
+    0.28, 0.3, 0.32, 0.33, 0.33, 0.32, 0.3, 0.28, 0.26, 0.23, 0.2, 0.18, 0.15, 0.13, 0.1, 0.08, 0.07, 0.06, 0.04, 0.03,
+    0.03, 0.02, 0.01, 0.0, 0.0, -0.01, -0.02, -0.03, -0.05, -0.06, -0.07, -0.09, -0.1, -0.11, -0.11, -0.12, -0.11, -0.1,
+    -0.08, -0.06, -0.05, -0.02, 0.0, 0.05, 0.07, 0.1, 0.14, 0.18, 0.21, 0.23, 0.24, 0.26, 0.27, 0.26, 0.24, 0.23, 0.21,
+    0.19, 0.16, 0.13, 0.1, 0.07, 0.04, 0.01, 0.0, -0.02, -0.04, -0.06, -0.08, -0.08, -0.09, -0.11, -0.12, -0.13, -0.14,
+    -0.17, -0.19, -0.21, -0.23, -0.25, -0.3, -0.34, -0.35, -0.37, -0.39, -0.41, -0.42, -0.41, -0.39, -0.39, -0.36,
+    -0.31, -0.25, -0.19, -0.14, -0.08, 0.0, 0.06, 0.11, 0.16, 0.22, 0.27, 0.3, 0.32, 0.34, 0.35, 0.34, 0.33, 0.31, 0.3,
+    0.27, 0.24, 0.2, 0.18, 0.16, 0.13, 0.09, 0.07, 0.06, 0.05, 0.03, 0.01, 0.01, 0.01, 0.0, -0.01, -0.02, -0.02, -0.03,
+    -0.04, -0.06, -0.07, -0.08, -0.1, -0.12, -0.13, -0.13, -0.14, -0.14, -0.14, -0.12, -0.1, -0.08, -0.05, -0.02, 0.01,
+    0.05, 0.09, 0.13, 0.18, 0.21, 0.24, 0.27, 0.29, 0.3, 0.31, 0.31, 0.29, 0.27, 0.24, 0.21, 0.17, 0.14, 0.1, 0.07,
+    0.04, 0.0, -0.01, -0.03, -0.05, -0.07, -0.08, -0.09, -0.1, -0.11, -0.12, -0.12, -0.13, -0.15, -0.17, -0.19, -0.21,
+    -0.24, -0.27, -0.31, -0.34, -0.36, -0.39, -0.42, -0.44, -0.44, -0.43, -0.41, -0.39, -0.36, -0.3, -0.24, -0.18,
+    -0.13, -0.06, 0.02, 0.09, 0.14, 0.19, 0.25, 0.3, 0.33, 0.35, 0.36, 0.36, 0.35, 0.33, 0.32, 0.29, 0.25, 0.21, 0.19,
+    0.16, 0.13, 0.09, 0.06, 0.04, 0.03, 0.01, 0.0, -0.01, -0.01, -0.01, -0.01, -0.02, -0.03, -0.03, -0.03, -0.04, -0.06,
+    -0.07, -0.08, -0.09, -0.11, -0.12, -0.13, -0.14, -0.14, -0.14, -0.12, -0.1, -0.09, -0.07, -0.02, 0.02, 0.05, 0.08,
+    0.13, 0.18, 0.23, 0.26, 0.28, 0.31, 0.34, 0.35, 0.33, 0.32, 0.3, 0.28, 0.24, 0.2, 0.17, 0.13, 0.08, 0.04, 0.01,
+    -0.01, -0.04, -0.06, -0.07, -0.08, -0.08, -0.1, -0.11, -0.11, -0.1, -0.11, -0.13, -0.14, -0.15, -0.17, -0.21, -0.25,
+    -0.28, -0.31, -0.34, -0.37, -0.4, -0.42, -0.44, -0.44, -0.43, -0.41, -0.39, -0.36, -0.3, -0.23, -0.17, -0.11, -0.04,
+    0.03, 0.11, 0.17, 0.22, 0.27, 0.31, 0.34, 0.36, 0.37, 0.36, 0.35, 0.33, 0.31, 0.28, 0.24, 0.2, 0.17, 0.14, 0.1,
+    0.07, 0.04, 0.02, 0.01, 0.0, 0.0, -0.02, -0.02, -0.01, -0.01, -0.02, -0.03, -0.03, -0.03, -0.04, -0.06, -0.07,
+    -0.09, -0.1, -0.12
+])
+
+_wav_bytes = (
+    b'RIFFd\x06\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80>\x00\x00'
+    b'\x00}\x00\x00\x02\x00\x10\x00data@\x06\x00\x00\x8b\x0e\xa2\x0b\xba\x08\xd1\x05\xe8\x02\x00\x00'
+    b'\x00\x00\x00\x00\x00\x00\x18\xfd\x18\xfd\x18\xfd\x18\xfd\x00\x00\x00\x00\x00\x00\xe8\x02\xd1\x05\xba\x08\xa2\x0b'
+    b't\x11\\\x14E\x17.\x1a\x17\x1d\xff\x1f\xff\x1f\xff\x1f\xff\x1f\x17\x1d.\x1aE\x17t\x11\xa2\x0b'
+    b'\xd1\x05\x00\x00F\xf7\x8c\xee\xd2\xe5\x01\xe0G\xd7\xa4\xcb\xd2\xc5\x18\xbd^\xb4\x8d\xae\xbb\xa8\xea\xa2'
+    b'\x01\xa0\x19\x9d\x19\x9d\x01\xa0\xea\xa2\xbb\xa8\x8d\xae0\xba\xea\xc2\xa4\xcbG\xd7\xd2\xe5^\xf4\x00\x00'
+    b'\xba\x08\\\x14\x17\x1d\xd1%\xa2+s1E7.:\x16=\x16=.:E7\\4s1'
+    b'\x8b.\xa2+\xb9(\xd1%\xe8"\xff\x1f\x17\x1d.\x1aE\x17E\x17\\\x14\\\x14t\x11\x8b\x0e'
+    b'\x8b\x0e\xa2\x0b\xba\x08\xba\x08\xd1\x05\xe8\x02\xe8\x02\x00\x00\x00\x00\x00\x00\x00\x00\x18\xfd\x18\xfd\x00\x00'
+    b'\x00\x00\x00\x00\x00\x00\xe8\x02\xd1\x05\xa2\x0b\x8b\x0et\x11\\\x14.\x1a\x17\x1d\xff\x1f\xe8"\xe8"'
+    b'\xe8"\xe8"\xff\x1f\xff\x1f\x17\x1d.\x1a\\\x14\x8b\x0e\xba\x08\xe8\x02\x18\xfdF\xf7u\xf1\xbb\xe8'
+    b'\xe9\xe2\x18\xdd^\xd4\x8d\xce\xd2\xc5\x01\xc00\xbau\xb1\xa4\xab\xd3\xa5\xea\xa2\x01\xa0\x19\x9d\x19\x9d'
+    b'\x01\xa0\xea\xa2\xbb\xa8u\xb10\xba\xd2\xc5u\xd1\x18\xdd\xa4\xeb/\xfa\xe8\x02\x8b\x0e.\x1a\xd1%'
+    b'\xa2+\\4.:\x16=\xff?\xff?\xff?\xff?\x16=E7\\4s1\x8b.\xa2+'
+    b'\xd1%\xe8"\xff\x1f\x17\x1d\x17\x1d.\x1aE\x17E\x17\\\x14t\x11\x8b\x0e\xa2\x0b\xba\x08\xd1\x05'
+    b'\xe8\x02\x00\x00\x18\xfd/\xfa/\xfaF\xf7^\xf4u\xf1u\xf1^\xf4^\xf4F\xf7/\xfa\x18\xfd'
+    b'\x00\x00\xd1\x05\xba\x08\x8b\x0eE\x17.\x1a\xff\x1f\xd1%\xb9(\xa2+\x8b.\x8b.\x8b.\xa2+'
+    b'\xb9(\xd1%\xff\x1f.\x1a\\\x14\xa2\x0b\xd1\x05\x00\x00/\xfa^\xf4\xa4\xeb\xd2\xe5\x01\xe0/\xda'
+    b'^\xd4\xa4\xcb\xd2\xc5\x01\xc0G\xb7\x8d\xae\xbb\xa8\xea\xa2\x19\x9d0\x9aG\x97G\x970\x9a\x19\x9d'
+    b'\x01\xa0\xd3\xa5u\xb10\xba\xbb\xc8^\xd4\xe9\xe2^\xf4\x00\x00\xa2\x0b.\x1a\xd1%\x8b.E7'
+    b'\x16=\xe8B\xb9H\xb9H\xb9H\xb9H\xb9H\xe8B\xff?.:E7s1\xa2+\xd1%'
+    b'\xe8"\xff\x1f\x17\x1dE\x17\\\x14t\x11\x8b\x0e\xa2\x0b\xba\x08\xd1\x05\xe8\x02\x00\x00\x18\xfdF\xf7'
+    b'^\xf4u\xf1\x8c\xee\xa4\xeb\xbb\xe8\xbb\xe8\xa4\xeb\xa4\xeb\x8c\xeeu\xf1F\xf7\x18\xfd\x00\x00\xd1\x05'
+    b'\x8b\x0eE\x17\x17\x1d\xe8"\xa2+s1\\4E7.:.:E7\\4s1\x8b.'
+    b'\xb9(\xff\x1f.\x1at\x11\xa2\x0b\xd1\x05\x00\x00/\xfa^\xf4\x8c\xee\xbb\xe8\xe9\xe2\x18\xdd/\xda'
+    b'^\xd4\xa4\xcb\xea\xc2\x01\xc00\xbau\xb1\xd3\xa5\x01\xa0\x19\x9dG\x97v\x91\x8d\x8e\x8d\x8ev\x91'
+    b'^\x940\x9a\xd3\xa5u\xb1\x01\xc0\x8d\xce\x18\xdd\x8c\xee\x00\x00\xa2\x0b.\x1a\xb9(E7\xe8B'
+    b'\xb9H\x8bN\\TEW\\TsQsQ\xa2K\xd0E\xff?.:\\4\xa2+\xd1%'
+    b'\xff\x1f\x17\x1dE\x17\\\x14\x8b\x0e\xa2\x0b\xa2\x0b\xba\x08\xd1\x05\x00\x00\x00\x00\x00\x00/\xfaF\xf7'
+    b'^\xf4u\xf1\xa4\xeb\xbb\xe8\xbb\xe8\xd2\xe5\xd2\xe5\xbb\xe8\xa4\xeb\x8c\xeeu\xf1F\xf7\x00\x00\xd1\x05'
+    b'\x8b\x0e\\\x14\x17\x1d\xd1%\x8b.\\4.:\xff?\xe8B\xe8B\xff?\x16=\x16=\\4'
+    b'\x8b.\xb9(\xff\x1fE\x17t\x11\xba\x08\xe8\x02\x18\xfdF\xf7u\xf1\x8c\xee\xbb\xe8\xd2\xe5\x01\xe0'
+    b'/\xdaG\xd7u\xd1\x8d\xce\xea\xc2\x01\xc0G\xb7\x8d\xae\xd3\xa5\x19\x9dG\x97v\x91\xa4\x8b\xd3\x85'
+    b'\xd3\x85\xa4\x8bv\x91G\x97\x19\x9d\xa4\xab\x18\xbd\xa4\xcb\x18\xdd\x8c\xee\x00\x00t\x11\xff\x1f\x8b.'
+    b'\x16=\xb9HsQEW\x16]\xff_\xff_\x16]EWsQ\xa2K\xe8B.:\\4'
+    b'\xa2+\xd1%\x17\x1dE\x17\\\x14t\x11\xa2\x0b\xba\x08\xba\x08\xd1\x05\xe8\x02\x00\x00\x00\x00\x18\xfd'
+    b'/\xfaF\xf7u\xf1\x8c\xee\xa4\xeb\xd2\xe5\xe9\xe2\x01\xe0\x01\xe0\x18\xdd\x01\xe0\xe9\xe2\xbb\xe8\x8c\xee'
+    b'u\xf1/\xfa\x00\x00\x8b\x0e\\\x14\x17\x1d\xb9(\\4\x16=\xe8B\xd0E\xa2K\x8bN\xa2K'
+    b'\xd0E\xe8B\x16=E7\x8b.\xd1%\x17\x1d\\\x14\xa2\x0b\xe8\x02\x00\x00/\xfa^\xf4\x8c\xee'
+    b'\xbb\xe8\xbb\xe8\xd2\xe5\x01\xe0\x18\xdd/\xdaG\xd7\x8d\xce\xbb\xc8\xea\xc2\x18\xbdG\xb7\xbb\xa8\x19\x9d'
+    b'0\x9a^\x94\x8d\x8e\xbc\x88\xd3\x85\xbc\x88\x8d\x8e\x8d\x8eG\x97\xd3\xa5G\xb7\xbb\xc8G\xd7\xbb\xe8'
+    b'\x00\x00t\x11\xff\x1f\x8b.\xff?\x8bNEW\x16]\xe7b\xd0e\xe7b\xff_-ZEW'
+    b'\x8bN\xd0E.:\\4\x8b.\xd1%.\x1a\\\x14t\x11\x8b\x0e\xba\x08\xe8\x02\xe8\x02\xe8\x02'
+    b'\x00\x00\x18\xfd/\xfa/\xfaF\xf7^\xf4\x8c\xee\xa4\xeb\xbb\xe8\xe9\xe2\x18\xdd/\xda/\xdaG\xd7'
+    b'G\xd7G\xd7\x18\xdd\xe9\xe2\xbb\xe8u\xf1/\xfa\xe8\x02\x8b\x0e.\x1a\xd1%\\4\x16=\xd0E'
+    b'\x8bN\\TEW-Z-Z\\T\x8bN\xd0E\x16=s1\xb9(\x17\x1d\\\x14\xa2\x0b'
+    b'\x00\x00\x18\xfdF\xf7u\xf1\xa4\xeb\xbb\xe8\xd2\xe5\xe9\xe2\x01\xe0\x18\xdd\x18\xdd/\xda^\xd4\x8d\xce'
+    b'\xbb\xc8\xea\xc20\xbau\xb1\xd3\xa5\x19\x9dG\x97\x8d\x8e\xd3\x85\x01\x80\x01\x80\xea\x82\xbc\x88\x8d\x8e'
+    b'G\x97\xbb\xa80\xba\xa4\xcb/\xda\x8c\xee\xd1\x05.\x1a\xb9(E7\xb9HEW\xff_\xd0e'
+    b'\xb9h\xb9h\xd0e\xff_\x16]\\T\xb9H\x16=E7\x8b.\xd1%.\x1at\x11\xa2\x0b'
+    b'\xba\x08\xe8\x02\x00\x00\x18\xfd\x18\xfd\x18\xfd\x18\xfd/\xfaF\xf7F\xf7F\xf7^\xf4\x8c\xee\xa4\xeb'
+    b'\xbb\xe8\xd2\xe5\x01\xe0\x18\xdd/\xdaG\xd7G\xd7G\xd7\x18\xdd\xe9\xe2\xd2\xe5\xa4\xeb/\xfa\xd1\x05'
+    b'\x8b\x0eE\x17\xd1%\\4\xe8B\xa2KsQ-Z\xe7b\xd0e\xff_\x16]EWsQ'
+    b'\xd0E.:s1\xd1%E\x17\xa2\x0b\xe8\x02\x18\xfd^\xf4\x8c\xee\xa4\xeb\xbb\xe8\xbb\xe8\xe9\xe2'
+    b'\x01\xe0\x01\xe0\xe9\xe2\x01\xe0/\xdaG\xd7^\xd4\x8d\xce\xea\xc2G\xb7\x8d\xae\xd3\xa5\x19\x9d^\x94'
+    b'\xa4\x8b\xd3\x85\x01\x80\x01\x80\xea\x82\xbc\x88\x8d\x8eG\x97\xbb\xa8\x18\xbd\x8d\xce\x01\xe0^\xf4\xba\x08'
+    b'\xff\x1fs1\xff?\x8bN-Z\xe7b\xb9h\xa2k\xb9h\xd0e\xff_-ZsQ\xd0E'
+    b'.:s1\xb9(\x17\x1d\\\x14\xa2\x0b\xd1\x05\xe8\x02\x00\x00\x00\x00/\xfa/\xfa\x18\xfd\x18\xfd'
+    b'/\xfaF\xf7F\xf7F\xf7^\xf4\x8c\xee\xa4\xeb\xd2\xe5\xe9\xe2\x18\xdd'
+)
+
+assert len(_wav) == 800
+assert len(_wav_bytes) == 1644
+assert _sr == 16000
+
+
+def test_aukit():
+    t0 = time.time()
+    import aukit
+    t = time.time() - t0
+    assert t < 5
+
+
+def test_audio_io():
+    from aukit.audio_io import load_wav, save_wav, anything2bytesio, anything2wav, anything2bytes, Dict2Obj, _sr
+
+    out = anything2bytes(_wav, sr=_sr)
+    assert len(out) == len(_wav_bytes)
+
+    out = anything2wav(_wav_bytes, sr=_sr)
+    assert len(out) == len(_wav)
+
+    my_obj = Dict2Obj({"my_key": "my_value"})
+    assert my_obj.my_key == "my_value"
+
+
+def test_audio_spectrogram():
+    from aukit.audio_spectrogram import linear_spectrogram, mel_spectrogram
+    from aukit.audio_spectrogram import default_hparams as hparams_spectrogram
+    from aukit.audio_spectrogram import linear2mel_spectrogram, mel2linear_spectrogram
+
+    out_linear = linear_spectrogram(_wav, hparams=hparams_spectrogram)
+    assert out_linear.shape == (401, 5)
+
+    out_mel = mel_spectrogram(_wav, hparams=hparams_spectrogram)
+    assert out_mel.shape == (80, 5)
+
+    out_mel_from_linear = linear2mel_spectrogram(out_linear, hparams=hparams_spectrogram)
+    assert out_mel_from_linear.shape == (80, 5)
+
+    out_linear_from_mel = mel2linear_spectrogram(out_mel, hparams=hparams_spectrogram)
+    assert out_linear_from_mel.shape == (401, 5)
+
+
+def test_audio_griffinlim():
+    from aukit.audio_griffinlim import inv_linear_spectrogram, inv_linear_spectrogram_tf, inv_mel_spectrogram
+    from aukit.audio_griffinlim import default_hparams as hparams_griffinlim
+    from aukit.audio_spectrogram import linear_spectrogram, mel_spectrogram
+
+    out_linear = linear_spectrogram(_wav, hparams=hparams_griffinlim)
+    out_wav = inv_linear_spectrogram(out_linear, hparams=hparams_griffinlim)
+    assert out_wav.shape == (800,)
+
+    out_mel = mel_spectrogram(_wav, hparams=hparams_griffinlim)
+    out_wav = inv_mel_spectrogram(out_mel, hparams=hparams_griffinlim)
+    assert out_wav.shape == (800,)
+
+
+def test_audio_player():
+    from aukit.audio_player import play_audio
+
+    play_audio(_wav, sr=_sr)
+    play_audio(_wav_bytes, sr=_sr)
+
+
+def test_audio_editor():
+    from aukit.audio_editor import split_silence_wave, strip_silence_wave, remove_silence_wave
+
+    out_wav = strip_silence_wave(_wav, sr=_sr, keep_silence_len=2)
+    assert len(out_wav) == 800
+    out_wav = remove_silence_wave(_wav, sr=_sr, keep_silence_len=2)
+    assert len(out_wav) == 800
+    out_wavs = split_silence_wave(_wav, sr=_sr, keep_silence_len=2)
+    assert len(out_wavs) == 1
+
+
+def test_normalizer():
+    from aukit.audio_normalizer import tune_volume, remove_silence
+
+    out_wav = remove_silence(_wav, max_silence_ms=20)
+    out_wav = tune_volume(_wav, target_dBFS=-10)
+
+
+if __name__ == "__main__":
+    print(__file__)

hhh/Browser_ads_kw.py

@@ -0,0 +1,52 @@
+from typing import NoReturn
+from selenium import webdriver
+import time
+import networkx as nx
+import dataset
+import pickle
+import codecs
+from selenium.webdriver.common import keys
+from selenium.webdriver.common.keys import Keys
+import sys
+import os
+import time
+import re
+import pandas as pd
+import df2sheet
+from browser_common import JBrowser
+import datetime
+import dataset
+
+#db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+
+
+def get_designer_statistics():
+    global db
+    jb=JBrowser()
+    data=""
+    jb.set_profile_path("Profile 7")
+#    googleurl='https://ads.google.com/aw/keywordplanner/'
+#    googleurl='https://ads.google.com/aw/overview?ocid=600024232&euid=459838964&__u=6055300436&uscid=600024232&__c=2195332968&authuser=0&subid=ALL-zh-TW-et-g-aw-c-home-awhp_xin1_signin!o2'
+    googleurl='https://ads.google.com/aw/keywordplanner/home?ocid=600024232&euid=459838964&__u=6055300436&uscid=600024232&__c=2195332968&authuser=0&subid=ALL-zh-TW-et-g-aw-c-home-awhp_xin1_signin%21o2'
+    jb.get(googleurl)
+    driver=jb.get_driver()
+    time.sleep(40)
+    print('after sleep')
+#    elmts=driver.find_elements_by_xpath("//div[@class='keyword-text _ngcontent-owh-97']")
+    elmts=driver.find_elements_by_xpath("//zippy-icon/..//keyword-text")
+    for elmt in elmts:
+        print(elmt.text)
+        data+=elmt.text+"\n"
+
+    fw=codecs.open('c:/tmp/out.txt','w','utf-8')
+    fw.write(data)
+    fw.close()
+
+#        print(elmt)
+
+    time.sleep(9999)
+
+    return 'ok'
+
+
+get_designer_statistics()

hhh/DB_Designer_Social.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python3
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import dataset
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+
+def save_sheet(df,filename,tabname,startpos='A1'):
+
+    scope = ['https://spreadsheets.google.com/feeds',
+            'https://www.googleapis.com/auth/drive']
+
+    credentials = ServiceAccountCredentials.from_json_keyfile_name('c:\\keys\\spread2.json', scope)
+#    credentials = ServiceAccountCredentials.from_json_keyfile_name('/var/keys/spread2.json', scope)
+
+    gc = gspread.authorize(credentials)
+    spread = Spread(filename,creds=credentials)
+
+    spread.df_to_sheet(df, index=False, sheet=tabname, start=startpos, replace=False)
+
+
+def do_jobs():
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+    cursor=db.query("SELECT designer,m01,ptt,pix FROM hhh.designer_social")
+    df = pd.DataFrame(columns=('designer','mobile01','ptt','blogs'))
+
+    idx=0
+
+    for c in cursor:
+        df.loc[idx]=[c['designer'],c['m01'],c['ptt'],c['pix']]
+    #    df.loc[idx]=['okok',333]
+        idx+=1
+    save_sheet(df,'Designer_SocialMedia','statistics')
+
+t = threading.Thread(target = do_jobs)
+t.start()
+t.join()
+

hhh/GATest.py

@@ -106,7 +106,7 @@ def main():
 #(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)
 
   body=[{ 'viewId': VIEW_ID,
-  'dateRanges': [{'startDate': '2021-05-24', 'endDate': '2021-06-10'}],
+  'dateRanges': [{'startDate': '2021-05-31', 'endDate': '2021-06-18'}],
 #  'filtersExpression': 'ga:sourceMedium=~(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)',
   'filtersExpression': 'ga:sourceMedium=~(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)',
   'metrics': [{'expression': 'ga:users'},{'expression': 'ga:newusers'},{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'},{'expression': 'ga:bounceRate'},{'expression': 'ga:pageviewsPerSession'}],
@@ -120,7 +120,9 @@ def main():
       table.insert(elmt)
 
   body=[{ 'viewId': VIEW_ID,
-  'dateRanges': [{'startDate': '2021-05-24', 'endDate': '2021-06-10'}],
+#  'dateRanges': [{'startDate': '2021-05-24', 'endDate': '2021-06-10'}],
+  'dateRanges': [{'startDate': '2021-05-31', 'endDate': '2021-06-18'}],
+
 #  'filtersExpression': 'ga:sourceMedium=~(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)',
   'filtersExpression': 'ga:sourceMedium!~(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)',
   'metrics': [{'expression': 'ga:users'},{'expression': 'ga:newusers'},{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'},{'expression': 'ga:bounceRate'},{'expression': 'ga:pageviewsPerSession'}],

hhh/GA_DB_KW_to_Sheep.py

@@ -32,7 +32,7 @@ def save_sheet(df,filename,tabname,startpos='A1'):
 
 def do_jobs():
     db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
-    cursor=db.query("SELECT k1.query,k1.clicks,k1.impressions,k1.ctr,(k2.clicks-k1.clicks)/k1.clicks*100 as growth FROM hhh.hhh_weekly_keywords k1, hhh_weekly_keywords k2 where k1.weeknum='22' and k2.weeknum='23' and k1.query=k2.query order by (k2.clicks-k1.clicks)/k1.clicks*100 desc;")
+    cursor=db.query("SELECT k1.query,k1.clicks,k1.impressions,k1.ctr,(k2.clicks-k1.clicks)/k1.clicks*100 as growth FROM hhh.hhh_weekly_keywords k1, hhh_weekly_keywords k2 where k1.weeknum='23' and k2.weeknum='24' and k1.query=k2.query order by (k2.clicks-k1.clicks)/k1.clicks*100 desc;")
     df = pd.DataFrame(columns=('query','clicks','impressions','ctr','growth'))
 
     idx=0

hhh/GA_Keywords.py

@@ -42,9 +42,10 @@ def gen_report(weeknum,begindate,enddate):
         records.append({'sessionid':sessionid,'query':query,'clicks':clicks,'impressions':int(impressions),'ctr':float(ctr),'position':float(position),'dt':datetime.datetime.now(),'weeknum':weeknum})
 
 
-gen_report('21','2021-05-24','2020-05-30')
+#gen_report('21','2021-05-24','2020-05-30')
 gen_report('22','2021-05-31','2020-06-06')
 gen_report('23','2021-06-07','2020-06-13')
+gen_report('24','2021-06-14','2020-06-18')
 
 print('complete')
 

hhh/SEO/gsc_exp.py

@@ -0,0 +1,61 @@
+import traceback
+import dataset
+import codecs
+import sys
+import pickle
+import os
+import searchconsole
+
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@127.0.0.1:3306/hhh?charset=utf8mb4')
+#db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+
+db.begin()
+#db = dataset.connect('sqlite:///:memory:')
+table=db['gsc_page_query']
+#pname='korea'
+rid=0
+
+def checkig():
+    global instl
+    global table
+    global pname
+    global rid
+    lst=[]
+    cntdict={}
+    codelist={}
+    idx=0
+    flag_break=False
+
+    fname=os.path.abspath(__file__)
+    elmts=fname.split(os.path.sep)
+    path2=os.path.sep.join(elmts[0:-1])
+    keysdir=path2+os.path.sep+'../keys'+os.path.sep
+
+#    account = searchconsole.authenticate(client_config='c:/keys/client_secret.json',credentials='c:/keys/credentials.json')
+    account = searchconsole.authenticate(client_config=keysdir+'client_secret.json',credentials=keysdir+'credentials.json')
+
+#    webproperty = account['https://ipromise.com.tw/']
+#    webproperty = account['https://'+pname+'.face8ook.org/']
+#    webproperty = account['https://www.damanwoo.com/']
+    webproperty = account['https://hhh.com.tw/']
+
+#    report=webproperty.query.range('2021-03-01', '2021-06-17').dimension('page','query').get()
+#    report=webproperty.query.range('2021-06-01', '2021-06-17').dimension('page','query').get()
+#    report=webproperty.query.range('2020-06-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/(491|31|293|278|31|24|594|356|307|491|33|385)', 'equals').get()
+#    report=webproperty.query.range('2020-03-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/'+pgnum, 'contains').get()
+#    report=webproperty.query.range('2020-03-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/'+pgnum, 'contains').get()
+    report=webproperty.query.range('2020-03-01', '2021-06-24').dimension('page','query').get()
+
+    result=[]
+    for r in report.rows:
+        entry={'page':r[0],'query':r[1]}
+        result.append(entry)
+
+    for r in result:
+        table.insert(r)
+    db.commit()
+
+
+r=checkig()
+

hhh/ads/gauthenticate.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""This example creates an OAuth 2.0 refresh token for the Google Ads API.
+
+This illustrates how to step through the OAuth 2.0 native / installed
+application flow.
+
+It is intended to be run from the command line and requires user input.
+"""
+
+
+import argparse
+
+from google_auth_oauthlib.flow import InstalledAppFlow
+
+
+SCOPE = "https://www.googleapis.com/auth/adwords"
+
+
+def main(client_secrets_path, scopes):
+    flow = InstalledAppFlow.from_client_secrets_file(
+        client_secrets_path, scopes=scopes
+    )
+
+    flow.run_console()
+
+    print("Access token: %s" % flow.credentials.token)
+    print("Refresh token: %s" % flow.credentials.refresh_token)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generates OAuth 2.0 credentials with the specified "
+        "client secrets file."
+    )
+    # The following argument(s) should be provided to run the example.
+    parser.add_argument(
+        "--client_secrets_path",
+        required=True,
+        help=(
+            "Path to the client secrets JSON file from the "
+            "Google Developers Console that contains your "
+            "client ID and client secret."
+        ),
+    )
+    parser.add_argument(
+        "--additional_scopes",
+        default=None,
+        help=(
+            "Additional scopes to apply when generating the "
+            "refresh token. Each scope should be separated "
+            "by a comma."
+        ),
+    )
+    args = parser.parse_args()
+
+    configured_scopes = [SCOPE]
+
+    if args.additional_scopes:
+        configured_scopes.extend(
+            args.additional_scopes.replace(" ", "").split(",")
+        )
+
+    main(args.client_secrets_path, configured_scopes)

hhh/ads/get_camp.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""This example illustrates how to get all campaigns.
+
+To add campaigns, run add_campaigns.py.
+"""
+
+
+import argparse
+import sys
+
+from google.ads.googleads.client import GoogleAdsClient
+from google.ads.googleads.errors import GoogleAdsException
+
+
+def main(client, customer_id):
+    ga_service = client.get_service("GoogleAdsService")
+
+    query = """
+        SELECT
+          campaign.id,
+          campaign.name
+        FROM campaign
+        ORDER BY campaign.id"""
+
+    # Issues a search request using streaming.
+    response = ga_service.search_stream(customer_id=customer_id, query=query)
+
+    for batch in response:
+        for row in batch.results:
+            print(
+                f"Campaign with ID {row.campaign.id} and name "
+                f'"{row.campaign.name}" was found.'
+            )
+
+
+if __name__ == "__main__":
+    # GoogleAdsClient will read the google-ads.yaml configuration file in the
+    # home directory if none is specified.
+    googleads_client = GoogleAdsClient.load_from_storage(version="v7")
+
+    parser = argparse.ArgumentParser(
+        description="Lists all campaigns for specified customer."
+    )
+    # The following argument(s) should be provided to run the example.
+    parser.add_argument(
+        "-c",
+        "--customer_id",
+        type=str,
+        required=True,
+        help="The Google Ads customer ID.",
+    )
+    args = parser.parse_args()
+
+    try:
+        main(googleads_client, args.customer_id)
+    except GoogleAdsException as ex:
+        print(
+            f'Request with ID "{ex.request_id}" failed with status '
+            f'"{ex.error.code().name}" and includes the following errors:'
+        )
+        for error in ex.failure.errors:
+            print(f'	Error with message "{error.message}".')
+            if error.location:
+                for field_path_element in error.location.field_path_elements:
+                    print(f"\t\tOn field: {field_path_element.field_name}")
+        sys.exit(1)

hhh/ads/kw_planner.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""This example generates keyword ideas from a list of seed keywords."""
+
+
+import argparse
+import sys
+from google.ads.googleads.client import GoogleAdsClient
+from google.ads.googleads.errors import GoogleAdsException
+
+# Location IDs are listed here:
+# https://developers.google.com/google-ads/api/reference/data/geotargets
+# and they can also be retrieved using the GeoTargetConstantService as shown
+# here: https://developers.google.com/google-ads/api/docs/targeting/location-targeting
+_DEFAULT_LOCATION_IDS = ["1023191"]  # location ID for New York, NY
+# A language criterion ID. For example, specify 1000 for English. For more
+# information on determining this value, see the below link:
+# https://developers.google.com/google-ads/api/reference/data/codes-formats#expandable-7
+_DEFAULT_LANGUAGE_ID = "1000"  # language ID for English
+
+
+# [START generate_keyword_ideas]
+def main(
+    client, customer_id, location_ids, language_id, keyword_texts, page_url
+):
+    keyword_plan_idea_service = client.get_service("KeywordPlanIdeaService")
+    keyword_competition_level_enum = client.get_type(
+        "KeywordPlanCompetitionLevelEnum"
+    ).KeywordPlanCompetitionLevel
+    keyword_plan_network = client.get_type(
+        "KeywordPlanNetworkEnum"
+    ).KeywordPlanNetwork.GOOGLE_SEARCH_AND_PARTNERS
+    location_rns = _map_locations_ids_to_resource_names(client, location_ids)
+    language_rn = client.get_service(
+        "LanguageConstantService"
+    ).language_constant_path(language_id)
+
+    # Either keywords or a page_url are required to generate keyword ideas
+    # so this raises an error if neither are provided.
+    if not (keyword_texts or page_url):
+        raise ValueError(
+            "At least one of keywords or page URL is required, "
+            "but neither was specified."
+        )
+
+    # Only one of the fields "url_seed", "keyword_seed", or
+    # "keyword_and_url_seed" can be set on the request, depending on whether
+    # keywords, a page_url or both were passed to this function.
+    request = client.get_type("GenerateKeywordIdeasRequest")
+    request.customer_id = customer_id
+    request.language = language_rn
+    request.geo_target_constants = location_rns
+    request.include_adult_keywords = False
+    request.keyword_plan_network = keyword_plan_network
+
+    # To generate keyword ideas with only a page_url and no keywords we need
+    # to initialize a UrlSeed object with the page_url as the "url" field.
+    if not keyword_texts and page_url:
+        request.url_seed.url = page_url
+
+    # To generate keyword ideas with only a list of keywords and no page_url
+    # we need to initialize a KeywordSeed object and set the "keywords" field
+    # to be a list of StringValue objects.
+    if keyword_texts and not page_url:
+        request.keyword_seed.keywords.extend(keyword_texts)
+
+    # To generate keyword ideas using both a list of keywords and a page_url we
+    # need to initialize a KeywordAndUrlSeed object, setting both the "url" and
+    # "keywords" fields.
+    if keyword_texts and page_url:
+        request.keyword_and_url_seed.url = page_url
+        request.keyword_and_url_seed.keywords.extend(keyword_texts)
+
+    keyword_ideas = keyword_plan_idea_service.generate_keyword_ideas(
+        request=request
+    )
+
+    for idea in keyword_ideas:
+        competition_value = idea.keyword_idea_metrics.competition.name
+        print(
+            f'Keyword idea text "{idea.text}" has '
+            f'"{idea.keyword_idea_metrics.avg_monthly_searches}" '
+            f'average monthly searches and "{competition_value}" '
+            "competition.\n"
+        )
+    # [END generate_keyword_ideas]
+
+
+def map_keywords_to_string_values(client, keyword_texts):
+    keyword_protos = []
+    for keyword in keyword_texts:
+        string_val = client.get_type("StringValue")
+        string_val.value = keyword
+        keyword_protos.append(string_val)
+    return keyword_protos
+
+
+def _map_locations_ids_to_resource_names(client, location_ids):
+    """Converts a list of location IDs to resource names.
+
+    Args:
+        client: an initialized GoogleAdsClient instance.
+        location_ids: a list of location ID strings.
+
+    Returns:
+        a list of resource name strings using the given location IDs.
+    """
+    build_resource_name = client.get_service(
+        "GeoTargetConstantService"
+    ).geo_target_constant_path
+    return [build_resource_name(location_id) for location_id in location_ids]
+
+
+if __name__ == "__main__":
+    # GoogleAdsClient will read the google-ads.yaml configuration file in the
+    # home directory if none is specified.
+    googleads_client = GoogleAdsClient.load_from_storage(version="v7")
+
+    parser = argparse.ArgumentParser(
+        description="Generates keyword ideas from a list of seed keywords."
+    )
+
+    # The following argument(s) should be provided to run the example.
+    parser.add_argument(
+        "-c",
+        "--customer_id",
+        type=str,
+        required=True,
+        help="The Google Ads customer ID.",
+    )
+    parser.add_argument(
+        "-k",
+        "--keyword_texts",
+        nargs="+",
+        type=str,
+        required=False,
+        default=[],
+        help="Space-delimited list of starter keywords",
+    )
+    # To determine the appropriate location IDs, see:
+    # https://developers.google.com/google-ads/api/reference/data/geotargets
+    parser.add_argument(
+        "-l",
+        "--location_ids",
+        nargs="+",
+        type=str,
+        required=False,
+        default=_DEFAULT_LOCATION_IDS,
+        help="Space-delimited list of location criteria IDs",
+    )
+    # To determine the appropriate language ID, see:
+    # https://developers.google.com/google-ads/api/reference/data/codes-formats#expandable-7
+    parser.add_argument(
+        "-i",
+        "--language_id",
+        type=str,
+        required=False,
+        default=_DEFAULT_LANGUAGE_ID,
+        help="The language criterion ID.",
+    )
+    # Optional: Specify a URL string related to your business to generate ideas.
+    parser.add_argument(
+        "-p",
+        "--page_url",
+        type=str,
+        required=False,
+        help="A URL string related to your business",
+    )
+
+    args = parser.parse_args()
+
+    try:
+        main(
+            googleads_client,
+            args.customer_id,
+            args.location_ids,
+            args.language_id,
+            args.keyword_texts,
+            args.page_url,
+        )
+    except GoogleAdsException as ex:
+        print(
+            f'Request with ID "{ex.request_id}" failed with status '
+            f'"{ex.error.code().name}" and includes the following errors:'
+        )
+        for error in ex.failure.errors:
+            print(f'\tError with message "{error.message}".')
+            if error.location:
+                for field_path_element in error.location.field_path_elements:
+                    print(f"\t\tOn field: {field_path_element.field_name}")
+        sys.exit(1)

hhh/farm/gen_architect.py

@@ -0,0 +1,78 @@
+import codecs
+import sys
+import os
+fr=codecs.open('c:/data/CSVservlet.csv','r','utf-8')
+lines=fr.readlines()
+print(lines[0:5])
+fr.close()
+
+addr_dict={}
+all_records=[]
+
+def gen_card(r):
+    cont=''
+    cont+='### '+r['name']+'\n\n'
+    cont+=' * 統一編號: '+r['id']+'\n'
+    cont+=' * 公司名稱: '+r['name']+'\n'
+    cont+=' * 公司地址: '+r['addr']+'\n'
+    cont+=' * 設立狀態: '+r['status']+'\n\n'
+    return cont
+
+def gen_record(r,rlist):
+    cont='+++\n'
+    cont+='title = " '+r['name']+' "\n'
+    cont+='date = "2021-06-13T09:07:20+08:00"\n'
+    cont+='tags = ["'+r['addr'][0:3]+'室內設計" ]\n'
+    cont+='categories = ["'+r['addr'][0:3]+'" ]\n'
+    cont+='author = "幸福經紀人"\n'
+    cont+='draft = false\n'
+    cont+='id="'+r['id']+'"\n'
+    cont+='+++\n\n'
+    cont+='## 室內設計基本資料\n\n'
+    cont+=gen_card(r)
+    cont+='## 附近室內設計推薦\n\n'
+    for rr in rlist:
+        if rr['id']!=r['id']:
+            cont+=gen_card(rr)
+    return cont
+
+#    print(cont)
+#    sys.exit()
+def ins_dict(r):
+    global addr_dict
+    key=r['addr'][0:9]
+    if addr_dict.get(key) is None:
+        addr_dict[key]=[r]
+    else:
+        addr_dict[key].append(r)
+
+#for l in lines[0:300]:
+for l in lines[1:]:
+
+    l=l.replace("'",'').replace('\n','')
+    elmts=l.split(',')
+    record={}
+    record['id']=elmts[0]
+    record['name']=elmts[1]
+    record['addr']=elmts[2]
+    record['status']=elmts[3]
+    ins_dict(record)
+    if '核准' in record['status']:
+        all_records.append(record)
+
+
+
+
+maxlen=0
+for k,v in addr_dict.items():
+    for vv in v:
+        cont=gen_record(vv,v)
+        fw=codecs.open('C:/gitlab/hhh_post/webSite/content/designer/'+vv['id']+'.md','w','utf-8')
+        fw.write(cont)
+        fw.close()
+#    l=len(v)
+#    if l>maxlen:
+#       maxlen=l
+#print(maxlen)
+
+#print(all_records)

hhh/farm/gendata.py

@@ -0,0 +1,78 @@
+import codecs
+import sys
+import os
+fr=codecs.open('c:/data/dec_db.csv','r','utf-8')
+lines=fr.readlines()
+print(lines[0:5])
+fr.close()
+
+addr_dict={}
+all_records=[]
+
+def gen_card(r):
+    cont=''
+    cont+='### '+r['name']+'\n\n'
+    cont+=' * 統一編號: '+r['id']+'\n'
+    cont+=' * 公司名稱: '+r['name']+'\n'
+    cont+=' * 公司地址: '+r['addr']+'\n'
+    cont+=' * 設立狀態: '+r['status']+'\n\n'
+    return cont
+
+def gen_record(r,rlist):
+    cont='+++\n'
+    cont+='title = " '+r['name']+' "\n'
+    cont+='date = "2021-06-13T09:07:20+08:00"\n'
+    cont+='tags = ["'+r['addr'][0:3]+'室內設計" ]\n'
+    cont+='categories = ["'+r['addr'][0:3]+'" ]\n'
+    cont+='author = "幸福經紀人"\n'
+    cont+='draft = false\n'
+    cont+='id="'+r['id']+'"\n'
+    cont+='+++\n\n'
+    cont+='## 室內設計基本資料\n\n'
+    cont+=gen_card(r)
+    cont+='## 附近室內設計推薦\n\n'
+    for rr in rlist:
+        if rr['id']!=r['id']:
+            cont+=gen_card(rr)
+    return cont
+
+#    print(cont)
+#    sys.exit()
+def ins_dict(r):
+    global addr_dict
+    key=r['addr'][0:9]
+    if addr_dict.get(key) is None:
+        addr_dict[key]=[r]
+    else:
+        addr_dict[key].append(r)
+
+#for l in lines[0:300]:
+for l in lines[1:]:
+
+    l=l.replace("'",'').replace('\n','')
+    elmts=l.split(',')
+    record={}
+    record['id']=elmts[0]
+    record['name']=elmts[1]
+    record['addr']=elmts[2]
+    record['status']=elmts[3]
+    ins_dict(record)
+    if '核准' in record['status']:
+        all_records.append(record)
+
+
+
+
+maxlen=0
+for k,v in addr_dict.items():
+    for vv in v:
+        cont=gen_record(vv,v)
+        fw=codecs.open('C:/gitlab/hhh_post/webSite/content/designer/'+vv['id']+'.md','w','utf-8')
+        fw.write(cont)
+        fw.close()
+#    l=len(v)
+#    if l>maxlen:
+#       maxlen=l
+#print(maxlen)
+
+#print(all_records)

hhh/farm/gendata_all.py

@@ -0,0 +1,78 @@
+import codecs
+import sys
+import os
+fr=codecs.open('c:/data/allcomp.csv','r','utf-8')
+lines=fr.readlines()
+print(lines[0:5])
+fr.close()
+
+addr_dict={}
+all_records=[]
+
+def gen_card(r):
+    cont=''
+    cont+='### '+r['name']+'\n\n'
+    cont+=' * 統一編號: '+r['id']+'\n'
+    cont+=' * 公司名稱: '+r['name']+'\n'
+    cont+=' * 公司地址: '+r['addr']+'\n'
+    cont+=' * 設立狀態: '+r['status']+'\n\n'
+    return cont
+
+def gen_record(r,rlist):
+    cont='+++\n'
+    cont+='title = " '+r['name']+' "\n'
+    cont+='date = "2021-06-13T09:07:20+08:00"\n'
+    cont+='tags = ["'+r['addr'][0:3]+'室內設計" ]\n'
+    cont+='categories = ["'+r['addr'][0:3]+'" ]\n'
+    cont+='author = "幸福經紀人"\n'
+    cont+='draft = false\n'
+    cont+='id="'+r['id']+'"\n'
+    cont+='+++\n\n'
+    cont+='## 室內設計基本資料\n\n'
+    cont+=gen_card(r)
+    cont+='## 附近室內設計推薦\n\n'
+    for rr in rlist:
+        if rr['id']!=r['id']:
+            cont+=gen_card(rr)
+    return cont
+
+#    print(cont)
+#    sys.exit()
+def ins_dict(r):
+    global addr_dict
+    key=r['addr'][0:9]
+    if addr_dict.get(key) is None:
+        addr_dict[key]=[r]
+    else:
+        addr_dict[key].append(r)
+
+#for l in lines[0:300]:
+for l in lines[1:]:
+
+    l=l.replace('"','').replace('\n','')
+    elmts=l.split(',')
+    record={}
+    record['id']=elmts[0]
+    record['name']=elmts[1]
+    record['addr']=elmts[2]
+    record['status']=elmts[3]
+    if '核准' in record['status'] and ('新北市' in record['addr'] or '臺北市' in record['addr']):
+        ins_dict(record)
+        all_records.append(record)
+
+
+
+
+maxlen=0
+for k,v in addr_dict.items():
+    for vv in v:
+        cont=gen_record(vv,v)
+        fw=codecs.open('C:/gitlab/hhh_post/webSite/content/company/'+vv['id']+'.md','w','utf-8')
+        fw.write(cont)
+        fw.close()
+#    l=len(v)
+#    if l>maxlen:
+#       maxlen=l
+#print(maxlen)
+
+#print(all_records)

hhh/gspace_fetch_ranks.py

@@ -170,7 +170,7 @@ def get_designer_statistics(designer_list):
 
 
 qlist=[]
-cursor=db.query('select name,vip from customer_list order by updated asc limit 25')
+cursor=db.query('select name,vip from customer_list order by updated asc limit 50')
 #cursor=db.query('select name,vip from customer_list where vip =  0 order by updated asc limit 20')
 #cursor=db.query('select name,vip from customer_list where vip =  1 order by updated asc limit 20')
 

hhh/hhh_tree.py

@@ -0,0 +1,243 @@
+from instaloader import Instaloader, Profile
+import traceback
+import copy
+import operator
+import dataset
+import pandas as pd
+import networkx as nx
+#import pysftp
+import codecs
+import pyvis
+import sys
+import pickle
+import os
+import searchconsole
+from pyvis.network import Network
+import jieba
+db = dataset.connect('sqlite:///:memory:')
+table=db['tmp']
+#pname='cont'
+#pname='damanwoo'
+#pname='drama'
+#pname='news'
+
+#pname='www'
+
+#pname='ipromise'
+#pname='sports'
+pname='rumor'
+#pname='korea'
+rid=0
+
+def get_css():
+    fr=codecs.open('jared/data/css.txt','r','utf-8')
+    lines=fr.readlines()
+    content=' '.join(lines)
+    fr.close()
+    return content
+
+def modify_file(fname):
+    fr=codecs.open(fname,'r','utf-8')
+    lines=fr.readlines()
+    fr.close()
+#    css=get_css()
+    css=''
+    content_output=''
+    for l in lines:
+        if '<body>' in l[0:10]:
+            content_output+=l
+            content_output+='\n<div id="google">\n'
+            continue
+        if '<style type="text' in l[0:22]:
+            content_output+=l
+            content_output+="\n"+css+"\n"
+            continue
+        if '<div id = "mynetwork"' in l[0:30]:
+            content_output+=l
+            content_output+='\n</div>\n'
+            continue
+
+        content_output+=l
+
+    fw=codecs.open("mod_"+fname,'w','utf-8')
+    fw.write(content_output)
+    fw.close()
+
+def checkig(pgnum):
+    global instl
+    global table
+    global pname
+    global rid
+    lst=[]
+    cntdict={}
+    codelist={}
+    idx=0
+    flag_break=False
+
+    fname=os.path.abspath(__file__)
+    elmts=fname.split(os.path.sep)
+    path2=os.path.sep.join(elmts[0:-1])
+    keysdir=path2+os.path.sep+'keys'+os.path.sep
+
+#    account = searchconsole.authenticate(client_config='c:/keys/client_secret.json',credentials='c:/keys/credentials.json')
+    account = searchconsole.authenticate(client_config=keysdir+'client_secret.json',credentials=keysdir+'credentials.json')
+
+#    webproperty = account['https://ipromise.com.tw/']
+#    webproperty = account['https://'+pname+'.face8ook.org/']
+#    webproperty = account['https://www.damanwoo.com/']
+    webproperty = account['https://hhh.com.tw/']
+
+#    report=webproperty.query.range('2021-03-01', '2021-06-17').dimension('page','query').get()
+#    report=webproperty.query.range('2021-06-01', '2021-06-17').dimension('page','query').get()
+#    report=webproperty.query.range('2020-06-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/(491|31|293|278|31|24|594|356|307|491|33|385)', 'equals').get()
+    report=webproperty.query.range('2020-03-01', '2021-06-22').dimension('page','query').filter('page', '/designers/cases/'+pgnum, 'contains').get()
+
+
+    urlq={}
+    for r in report.rows:
+        if urlq.get(r[0]) is None:
+            urlq[r[0]]=[r[1]]
+        else:
+            urlq[r[0]].append(r[1])
+
+
+    allrows=[]
+    for k,v in urlq.items():
+#        if len(v)<40:
+        if len(v)<0:
+            continue
+#        print(k)
+        for q in v:
+#            elmts=jieba.cut(q)
+            elmts=q.split(' ')
+            for elmt in elmts:
+#                print(elmt)
+                table.insert({'q':elmt,'rid':rid,'url':k})
+        rid+=1
+        allrows.append([r[0],r[1] ])
+
+    db.commit()
+
+
+#    cursor=db.query('(select q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 )')
+
+
+def gen_pic():
+    G=None
+#    if os.path.exists(pname):
+#        G = pickle.load( open( pname, "rb" ) )
+#    else:
+#        G = nx.Graph()
+    G = nx.Graph()
+
+    finallist=[]
+
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having count(url) <= 3) as tbl1 ) order by q')
+#    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 2 group by q having ) as tbl1 ) order by q')
+    cursor=db.query('select q,rid,url from tmp where q in (select distinct q from  (select q,count(url) from tmp where length(q)> 1 group by q  ) as tbl1 ) order by q')
+
+    riddict={}
+    prev=''
+    curnode=''
+    cururl=''
+
+    total_idx=0
+    for c in cursor:
+        if c['q']!=prev:
+            cururl=c['url']
+            prev=c['q']
+            total_idx+=1
+#            if total_idx >= 200:
+#                break
+        else:
+#            G.add_edge(cururl,c['url'],weight=3,width=3,borderwidth=3)
+#            G.add_edge(cururl[40:51],c['url'][40:51],weight=3,width=3,borderwidth=3)
+#            G.add_edge(c['q'],cururl[40:51],weight=3,width=3,borderwidth=3)
+#            G.add_edge(c['q'],c['url'][40:51],weight=3,width=3,borderwidth=3)
+            query=c['q']
+            rid=c['rid']
+            if riddict.get(rid) is None:
+                riddict[rid]={}
+                riddict[rid][query]=1
+            else:
+                if riddict[rid].get(query) is not None:
+                    riddict[rid][query]=1
+                else:
+                    riddict[rid][query]={}
+
+            G.add_edge(c['q'],c['rid'],weight=3,width=3,borderwidth=3)
+#            G.add_edge(c['q'],c['rid'],weight=3,width=3,borderwidth=3)
+
+    pickle.dump( G, open( pname, "wb" ) )
+
+#    G2 = [G.subgraph(c).copy() for c in nx.connected_components(G)]
+#    remove = [node for node,degree in dict(G.degree()).items() if degree <2]
+#    G.remove_nodes_from(remove)
+
+    remove=[]
+#    for n in G.nodes:
+#        if '承諾' in n:
+#            remove.append(n)
+#        if 'promise' in n:
+#            remove.append(n)
+#    G.remove_nodes_from(remove)
+
+
+    G.remove_edges_from(nx.selfloop_edges(G))
+    G.remove_nodes_from(list(nx.isolates(G)))
+#    lst= [G.subgraph(c).copy() for c in nx.connected_components(G)]
+#    lst=[]
+#    for c in nx.connected_components(G):
+#        cc=G.subgraph(c).copy()
+#        if cc.number_of_nodes()>7:
+#            lst.append(cc)
+
+#        if nx.diameter(cc, e=None, usebounds=False)>1:
+#            lst.append(cc)
+
+#    G2=nx.compose_all(lst)
+    G2=G
+#    pyG = Network(height="750px", width="100%",bgcolor="#333333",font_color="white")
+    pyG = Network(height="600px", width="100%",bgcolor="#444444",font_color="white")
+
+
+
+
+    pyG.from_nx(G2)
+    pyG.show(pname+'.html')
+    modify_file(pname+'.html')
+
+#    cnopts = pysftp.CnOpts()
+#    cnopts.hostkeys = None
+#    s = pysftp.Connection(host='www.choozmo.com', username='jared', password='sstc5202',cnopts=cnopts)
+#    local_path = "mod_"+pname+".html"
+#    remote_path = "/home/nginx/farmoutput/tags/"+"mod_"+pname+".html"
+#    s.put(local_path, remote_path)
+
+
+    return finallist
+
+#r=checkig('投資')
+
+#r=checkig('保險')
+#r=checkig('嘖嘖')
+#r=checkig('募資')
+#r=checkig('遠赤外線')
+#lst=['491','31','293','278','24','594','356','307','491','33','385']
+lst=['491','31','293','278','24','594','356','307','491','33','385']
+
+for l in lst:
+    r=checkig(l)
+gen_pic()
+#r=checkig('信用卡')
+#print(r)
+
+#        network.on( 'click', function(properties) {
+#    var ids = properties.nodes;
+#    var clickedNodes = nodes.get(ids);
+# var copyText = clickedNodes[0].label;
+# var promise = navigator.clipboard.writeText(copyText);
+#//    console.log('clicked nodes:', clickedNodes);
+#});
+

hhh/old_tree.py

@@ -0,0 +1,258 @@
+from instaloader import Instaloader, Profile
+import traceback
+import copy
+import operator
+import dataset
+import pandas as pd
+import networkx as nx
+#import pysftp
+import codecs
+import pyvis
+import sys
+import pickle
+import os
+import searchconsole
+from pyvis.network import Network
+import jieba
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import dataset
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+
+def save_sheet(df,filename,tabname,startpos='A1'):
+
+    scope = ['https://spreadsheets.google.com/feeds',
+            'https://www.googleapis.com/auth/drive']
+
+    credentials = ServiceAccountCredentials.from_json_keyfile_name('c:\\keys\\spread2.json', scope)
+#    credentials = ServiceAccountCredentials.from_json_keyfile_name('/var/keys/spread2.json', scope)
+
+    gc = gspread.authorize(credentials)
+    spread = Spread(filename,creds=credentials)
+
+    spread.df_to_sheet(df, index=False, sheet=tabname, start=startpos, replace=False)
+
+
+
+
+
+db = dataset.connect('sqlite:///:memory:')
+table=db['tmp']
+#pname='cont'
+#pname='damanwoo'
+#pname='drama'
+pname='news'
+
+#pname='www'
+
+#pname='ipromise'
+#pname='sports'
+#pname='rumor'
+#pname='korea'
+
+def get_css():
+    return ''
+#    fr=codecs.open('jared/data/css.txt','r','utf-8')
+#   lines=fr.readlines()
+#   content=' '.join(lines)
+#   fr.close()
+#   return content
+
+def modify_file(fname):
+    fr=codecs.open(fname,'r','utf-8')
+    lines=fr.readlines()
+    fr.close()
+    css=get_css()
+    content_output=''
+    for l in lines:
+        if '<body>' in l[0:10]:
+            content_output+=l
+            content_output+='\n<div id="google">\n'
+            continue
+        if '<style type="text' in l[0:22]:
+            content_output+=l
+            content_output+="\n"+css+"\n"
+            continue
+        if '<div id = "mynetwork"' in l[0:30]:
+            content_output+=l
+            content_output+='\n</div>\n'
+            continue
+
+        content_output+=l
+
+    fw=codecs.open("mod_"+fname,'w','utf-8')
+    fw.write(content_output)
+    fw.close()
+
+def checkig(kw):
+    global instl
+    global table
+    global pname
+    lst=[]
+    idx=0
+    cntdict={}
+    codelist={}
+    G=None
+#    if os.path.exists(pname):
+#        G = pickle.load( open( pname, "rb" ) )
+#    else:
+#        G = nx.Graph()
+    G = nx.Graph()
+
+    finallist=[]
+    idx=0
+    flag_break=False
+#    account = searchconsole.authenticate(client_config='c:/keys/client_secret.json',credentials='c:/keys/credentials.json')
+#    webproperty = account['https://ipromise.com.tw/']
+#    webproperty = account['https://'+pname+'.face8ook.org/']
+#    webproperty = account['https://www.damanwoo.com/']
+
+    fname=os.path.abspath(__file__)
+    elmts=fname.split(os.path.sep)
+    path2=os.path.sep.join(elmts[0:-1])
+    keysdir=path2+os.path.sep+'keys'+os.path.sep
+
+
+
+    account = searchconsole.authenticate(client_config=keysdir+'client_secret.json',credentials=keysdir+'credentials.json')
+
+#    webproperty = account['https://ipromise.com.tw/']
+#    webproperty = account['https://'+pname+'.face8ook.org/']
+#    webproperty = account['https://www.damanwoo.com/']
+    webproperty = account['https://hhh.com.tw/']
+
+
+
+#    report=webproperty.query.range('2021-06-11', '2021-06-18').dimension('page','query').get()
+    report=webproperty.query.range('2020-06-22', '2021-06-21').dimension('page','query').limit(10000).get()
+
+
+    urlq={}
+    for r in report.rows:
+        if urlq.get(r[0]) is None:
+            urlq[r[0]]=[r[1]]
+        else:
+            urlq[r[0]].append(r[1])
+
+
+    allrows=[]
+    rid=0
+    for k,v in urlq.items():
+#        if len(v)<35:
+        if len(v)<2:
+            continue
+        print(len(v))
+        for q in v:
+#            elmts=jieba.cut(q)
+            elmts=q.split(' ')
+            for elmt in elmts:
+#                print(elmt)
+                table.insert({'q':elmt,'rid':rid})
+        rid+=1
+        allrows.append([r[0],r[1] ])
+
+    db.commit()
+
+    cursor=db.query('select q,rid from tmp order by q')
+    prev=''
+    curnode=''
+
+
+
+    df = pd.DataFrame(columns=('rid','query'))
+
+
+    repdict={}
+    idx=0
+    for c in cursor:
+        if c['rid']!=prev:
+            curnode=c['q']
+            prev=c['rid']
+        else:
+            if repdict.get((curnode,c['q'])) is None:
+                repdict[(curnode,c['q'])]=1
+#                repdict[(c['q'],curnode)]=1
+                df.loc[idx]=[curnode,c['q']]
+                idx+=1
+                G.add_edge(curnode,c['q'],weight=3,width=3,borderwidth=3)
+
+    pickle.dump( G, open( pname, "wb" ) )
+#    save_sheet(df,'BigGraph','nodes')
+
+#    G2 = [G.subgraph(c).copy() for c in nx.connected_components(G)]
+#    remove = [node for node,degree in dict(G.degree()).items() if degree <2]
+#    G.remove_nodes_from(remove)
+
+    remove=[]
+    for n in G.nodes:
+        if '承諾' in n:
+            remove.append(n)
+        if 'promise' in n:
+            remove.append(n)
+    G.remove_nodes_from(remove)
+
+
+    G.remove_edges_from(nx.selfloop_edges(G))
+    G.remove_nodes_from(list(nx.isolates(G)))
+#    lst= [G.subgraph(c).copy() for c in nx.connected_components(G)]
+#    lst=[]
+#    for c in nx.connected_components(G):
+#        cc=G.subgraph(c).copy()
+#        if cc.number_of_nodes()>7:
+#            lst.append(cc)
+
+#        if nx.diameter(cc, e=None, usebounds=False)>1:
+#            lst.append(cc)
+
+#    G2=nx.compose_all(lst)
+    G2=G
+#    pyG = Network(height="750px", width="100%",bgcolor="#333333",font_color="white")
+    pyG = Network(height="600px", width="100%",bgcolor="#444444",font_color="white")
+
+
+
+
+    pyG.from_nx(G2)
+    pyG.show(pname+'.html')
+    modify_file(pname+'.html')
+
+#    cnopts = pysftp.CnOpts()
+#    cnopts.hostkeys = None
+#    s = pysftp.Connection(host='www.choozmo.com', username='jared', password='sstc5202',cnopts=cnopts)
+#    local_path = "mod_"+pname+".html"
+#    remote_path = "/home/nginx/farmoutput/tags/"+"mod_"+pname+".html"
+#    s.put(local_path, remote_path)
+
+
+    return finallist
+
+#r=checkig('投資')
+
+#r=checkig('保險')
+#r=checkig('嘖嘖')
+#r=checkig('募資')
+#r=checkig('遠赤外線')
+r=checkig('インソール')
+#r=checkig('信用卡')
+#print(r)
+
+#        network.on( 'click', function(properties) {
+#    var ids = properties.nodes;
+#    var clickedNodes = nodes.get(ids);
+# var copyText = clickedNodes[0].label;
+# var promise = navigator.clipboard.writeText(copyText);
+#//    console.log('clicked nodes:', clickedNodes);
+#});
+

hhh/parse_result.py

@@ -0,0 +1,82 @@
+import codecs
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+
+def save_sheet(df,filename,tabname,startpos='A1'):
+
+    scope = ['https://spreadsheets.google.com/feeds',
+            'https://www.googleapis.com/auth/drive']
+
+#    credentials = ServiceAccountCredentials.from_json_keyfile_name('c:\\keys\\service\\gspread.json', scope)
+    credentials = ServiceAccountCredentials.from_json_keyfile_name('c:\\keys\\spread2.json', scope)
+
+    gc = gspread.authorize(credentials)
+    spread = Spread(filename,creds=credentials)
+
+    spread.df_to_sheet(df, index=False, sheet=tabname, start=startpos, replace=False)
+
+
+
+fr=codecs.open('c:/tmp/out.txt','r','utf-8')
+lines=fr.readlines()
+fr.close()
+cur_kw=None
+fulldict={}
+for l in lines:
+    l=l.replace('\n','').replace('\u202a','').replace('\u202c','')
+    if '(' in l:
+        cur_kw=l
+    else:
+        elmts=l.split(',')
+        lst=[]
+        for elmt in elmts:
+            lst.append(elmt)
+        fulldict[cur_kw]=lst
+#        fulldict
+
+#    print(l)
+print(fulldict)
+
+
+def get_num_n(fulldict,n):
+    result=[]
+    for k,v in fulldict.items():
+        if len(fulldict[k])<n+1:
+            result.append(' ')
+        else:
+            result.append(fulldict[k][n])
+    return result
+
+cols=[]
+maxlen=0
+for k,v in fulldict.items():
+    elmts=k.split('(')
+    k=elmts[0]
+    cols.append(k)
+    if len(v)>maxlen:
+        maxlen=len(v)
+
+df = pd.DataFrame(columns=tuple(cols))
+llen=len(fulldict.items())
+
+
+for i in range(maxlen):
+    df.loc[i]=get_num_n(fulldict,i)
+print(df)
+save_sheet(df,'keyword-details','風水')
+
+
+
+

hhh/remote_gspace_fetch_ranks.py

@@ -0,0 +1,182 @@
+from typing import NoReturn
+from selenium import webdriver
+import time
+import networkx as nx
+import dataset
+import pickle
+import codecs
+from selenium.webdriver.common import keys
+from selenium.webdriver.common.keys import Keys
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import sys
+import os
+import time
+import re
+import pandas as pd
+import df2sheet
+from browser_common import JBrowser
+import datetime
+import dataset
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+
+def find_master_by_designer(masters,designer):
+    for m in masters:
+        if m['designer']==designer:
+            return m
+
+
+def get_designer_statistics(designer_list):
+    global db
+
+    details=[]
+    masters=[]
+
+
+    table=db['customer_list']
+    table_details=db['rank_details']
+    table_summary=db['rank_summary']
+    driver = webdriver.Remote(
+        command_executor='http://dev2.choozmo.com:34444/wd/hub',
+    #command_executor='http://192.53.174.202:4444/wd/hub',
+    #command_executor='http://dev2.choozmo.com:14444/wd/hub',
+    desired_capabilities=DesiredCapabilities.CHROME)
+    driver.set_window_size(1400,1000)
+
+
+
+    for tup in designer_list:
+        q=tup[0]
+        vip=tup[1]
+        googleurl='https://www.google.com/search?q='+q
+
+
+
+        driver.get(googleurl)
+
+        time.sleep(3)
+
+
+#        elmts=driver.find_elements_by_xpath("//div[@class='g']//div[@class='yuRUbf']//a")
+        elmts=driver.find_elements_by_xpath("//div[@class='TbwUpd NJjxre']/cite")
+#        print(elmts)
+#        time.sleep(9999)
+
+
+        idx=1
+        ranking=-1
+        searchhome=-1
+        hhh=-1
+        com100=-1
+        txt=None
+        href=None
+        for elmt in elmts:
+
+
+#            href=elmt.get_attribute('href')
+            elmt_titl3=elmt.find_element_by_xpath("../..//h3")
+            elmt_href=elmt.find_element_by_xpath("../..")
+
+            print(elmt_titl3.text)
+            print(elmt_href.text)
+            txt=elmt_titl3.text
+            href=elmt_href.text
+
+
+            print(idx)
+            print(len(elmts))
+            print(txt)
+            print(href)
+#            time.sleep(9999)
+#            if len(txt)>2:
+            if not ('google.com' in href):
+
+                if '100.com' in href:
+                    com100=idx
+
+                if 'searchome' in href:
+                    searchhome=idx
+
+                if 'hhh.com.tw' in href:
+                    hhh=idx
+                    ranking=idx
+                    print("updsert")
+                    print({'name':q,'updated':datetime.datetime.now()})
+                    table.upsert({'name':q,'updated':datetime.datetime.now()},keys=['name'])
+                    m={'designer':q,'title':txt,'url':href,'hhh':hhh,'dt':datetime.datetime.now(),'searchome':searchhome,'c_100':com100,'vip':vip}
+                    masters.append(m)
+                    table_summary.insert(m)
+
+                print(href)
+                print(txt)
+    #            table.insert({'designer':q,'title':txt,'url':href,'ranking':idx})
+                dtstr=datetime.datetime.now().strftime("%Y-%m/%d %H:%M:%S")
+                d={'designer':q,'title':txt,'url':href,'ranking':idx}
+                details.append(d)
+                table_details.insert({'q':q,'dt':datetime.datetime.now(),'designer':d['designer'],'title':d['title'],'url':d['url'],'ranking':idx,'hhh':hhh,'searchome':searchhome,'c_100':com100})
+
+                idx+=1
+
+#        time.sleep(9999)
+
+        if ranking==-1:
+            ranking=idx
+            print("updsert")
+            print({'name':q,'updated':datetime.datetime.now()})
+            table.upsert({'name':q,'updated':datetime.datetime.now()},keys=['name'])
+            m={'designer':q,'title':txt,'url':href,'hhh':hhh,'dt':datetime.datetime.now(),'searchome':searchhome,'c_100':com100}
+            masters.append(m)
+            table_summary.insert(m)
+
+        db.commit()
+
+        time.sleep(3)
+
+#    print(masters)
+#    print(details)
+    return {'masters':masters,'details':details}
+
+
+# 寓子設計
+#qlist=['元均制作']
+#qlist=['三宅一秀']
+#qlist=['采品室內設計']
+#qlist=['寓子設計']
+#qlist=['綵韻室內設計','春雨時尚空間','阿曼空間設計','雅典設計','境庭國際設計']
+#qlist=['豐聚室內裝修','張馨室內設計','尚藝室內裝修','富億空間設計','比沙列室內裝修']
+#qlist=['森境王俊宏設計','格綸設計','齊舍設計','采舍空間設計','大琚空間設計']
+#qlist=['將作空間','昱承室內裝修','YHS DESIGN','德本迪室內設計','東風室內設計']
+#qlist=['陶璽空間設計','惹雅國際設計','浩室設計','藝谷空間設計','IS國際設計']
+#qlist=['摩登雅舍室內','星葉室內裝修','浩室設計','演拓空間','千綵胤空間']
+#qlist=['京璽國際','元典設計','朱英凱室內設計','亞維空間設計','馥築時尚設計']
+#qlist=['文儀室內裝修','寓子設計','恆岳空間設計','卓林室內設計','歐德傢俱']
+#qlist=['大久空間設計','成綺空間設計','知域設計','尚展空間設計','演繹動線空間']
+#qlist=['苡希創意設計','玖柞設計','維耕設計','昱森室內設計','上築空間設計']
+#qlist=['HATCH合砌設計','至文室內裝修','上陽設計','禾禾設計','聯寬室內裝修']
+
+#設計
+#qlist=['三宅一秀','萬寶隆空間設計','含仰空間設計','元均制作','承炫裝修']
+
+
+qlist=[]
+cursor=db.query('select name,vip from customer_list order by updated asc limit 50')
+#cursor=db.query('select name,vip from customer_list where vip =  0 order by updated asc limit 20')
+#cursor=db.query('select name,vip from customer_list where vip =  1 order by updated asc limit 20')
+
+for c in cursor:
+    qlist.append((c['name'],c['vip']))
+
+#get_designer_statistics([qlist[0]])
+get_designer_statistics(qlist)
+#        if d['designer']==q:
+#            df.loc[idx]=[d['designer'],d['title'],d['url'],d['ranking'],r['ranking'],dtstr]
+#            idx+=1
+#    df2sheet.save_sheet(df,'designer_ranking',q,startpos='A1')
+

hhh/tests/achanger.py

@@ -0,0 +1,261 @@
+#!usr/bin/env python
+# -*- coding: utf-8 -*-
+# author: kuangdd
+# date: 2019/12/22
+"""
+### audio_changer
+变声器，变高低音，变语速，变萝莉音，回声。
+"""
+from pathlib import Path
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(Path(__name__).stem)
+
+import numpy as np
+import librosa
+
+from .audio_io import _sr
+
+
+def change_pitch(wav, sr=_sr, rate=0.):
+    """
+    调音高。
+    :param rate:-20~20,float，0:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    return librosa.effects.pitch_shift(wav, sr=sr, n_steps=rate)
+
+
+def change_speed(wav, sr=_sr, rate=0.):
+    """
+    调语速。
+    :param rate:0~5,float，0:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    return librosa.effects.time_stretch(wav, rate)
+
+
+def change_sample(wav, sr=_sr, rate=1):
+    """
+    调采样率，语速和音高同时改变。
+    :param rate:0~5,float，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    return librosa.resample(wav, orig_sr=sr, target_sr=int(sr * rate))
+
+
+def change_reback(wav, sr=_sr, rate=1):
+    """
+    回声。
+    :param rate:1~10,int，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = pool(D, size=(1, rate))
+    D = repeat(D, rate)
+    return librosa.istft(D)
+
+
+def change_pitchspeed(wav, sr=_sr, rate=1):
+    """
+    音高和语速同时变化。
+    :param rate:0~10,float，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    n = int(D.shape[0] * rate)
+    if n <= D.shape[0]:
+        D = drop(D, D.shape[0] - n, mode="r")
+    else:
+        D = rewardshape(D, (n, D.shape[1]))
+    return librosa.istft(D)
+
+
+def change_attention(wav, sr=_sr, rate=0):
+    """
+    突出高音或低音段。
+    :param rate:-100~100,int，0:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = roll(D, rate)
+    return librosa.istft(D)
+
+
+def change_male(wav, sr=_sr, rate=0):
+    """
+    变男声。
+    :param rate:0~1025,int，0,1,1025:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = pool_step(D, rate)
+    return librosa.istft(D)
+
+
+def change_stretch(wav, sr=_sr, rate=1):
+    """
+    成倍拉伸延长。
+    :param rate:1~10,int，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = spread(D, rate)
+    return librosa.istft(D)
+
+
+def change_vague(wav, sr=_sr, rate=1):
+    """
+    模糊。
+    :param rate:1~10,int，1:原声
+    :param wav:
+    :param sr:
+    :return:
+    """
+    frequencies, D = librosa.ifgram(wav, sr=sr)
+    D = pool(D, (1, rate))
+    D = spread(D, (1, rate))
+    return librosa.istft(D)
+
+
+class CheckStep(object):
+    def __init__(self, step):
+        self.step = step
+        self.index = 0
+
+    def __call__(self, *args):
+        self.index += 1
+        return self.index % self.step != 0
+
+
+def spread(D, size=(3, 3)):
+    """传播，重复每个数据点。"""
+    if isinstance(size, tuple):
+        if size[0] > 1:
+            D = np.repeat(D, size[0], axis=0)
+        if size[1] > 1:
+            D = np.repeat(D, size[1], axis=1)
+        if size[0] * size[1] > 1:
+            D = D / (size[0] * size[1])
+    elif isinstance(size, int):
+        D = np.repeat(D, size, axis=1)
+    return D
+
+
+def drop(D, n, mode="l"):
+    """丢弃,mode:left,right,side,center"""
+    if n == 0:
+        return D
+    if mode == "l":
+        return D[n:]
+    elif mode == "r":
+        return D[:-n]
+    elif mode == "s":
+        return D[n // 2:-(n // 2)]
+    elif mode == "c":
+        if n < len(D):
+            return np.vstack((D[:n // 2], D[-(n // 2):]))
+        else:
+            return ()
+    else:
+        raise AssertionError
+
+
+def repeat(D, n, axis=0):
+    """重复"""
+    return np.repeat(D, n, axis=axis)
+
+
+def roll(D, n):
+    """循环移动"""
+    return np.roll(D, n, axis=0)
+
+
+def rewardshape(D, shape):
+    """填充"""
+    x = shape[0] - D.shape[0]
+    y = shape[1] - D.shape[1]
+    if x > 0:
+        bottomlist = np.zeros([x, D.shape[1]])
+        D = np.r_[D, bottomlist]
+    if y > 0:
+        rightlist = np.zeros([D.shape[0], y])
+        D = np.c_[D, rightlist]
+    return D
+
+
+def pool_step(D, step):
+    """步长池化"""
+    _shape = D.shape
+    if step < 2:
+        return D
+    cs = CheckStep(step)
+    return rewardshape(np.array(list(filter(cs, D))), _shape)
+
+
+def pool(D, size=(3, 3), shapeed=False):
+    """池化"""
+    _shape = D.shape
+    if isinstance(size, tuple):
+        if size[1] > 1:
+            D = _pool(D, size[1])
+        if size[0] > 1:
+            D = _pool(D.T, size[0]).T
+    elif isinstance(size, int):
+        D = _pool(D.T, size).T
+    if shapeed:
+        D = rewardshape(D, _shape)
+    return D
+
+
+def _pool(D, poolsize):
+    """池化方法"""
+    x = D.shape[1] // poolsize
+    restsize = D.shape[1] % poolsize
+    if restsize > 0:
+        x += 1
+        rightlist = np.zeros([D.shape[0], poolsize - restsize])
+        D = np.c_[D, rightlist]
+    D = D.reshape((-1, poolsize))
+    D = D.sum(axis=1).reshape(-1, x)
+    return D
+
+
+if __name__ == '__main__':
+    from aukit.audio_player import play_audio
+
+    path = r"C:/Users/jared/Downloads/hi.mp3"
+    y, sr = librosa.load(path)
+
+    # y = change_vague(3, y, sr)
+    # y = change_pitch(-6, y, sr)
+    # y = change_speed(0.5, y, sr)
+    # y = change_sample(0.8, y, sr)
+    # y = change_reback(3, y, sr)
+    # y = change_pitchspeed(2, y, sr)
+    # y = change_attention(50, y, sr)
+    # y = change_male(5, y, sr)
+    # y = change_vague(6, y, sr)
+
+    """童声"""
+    y = change_pitch(6, y, sr)
+    y = change_male(20, y, sr)
+
+    play_audio(y, sr=sr)

hhh/tests/get_data.py

@@ -0,0 +1,64 @@
+import dataset
+import codecs
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+
+def save_sheet(df,filename,tabname,startpos='A1'):
+
+    scope = ['https://spreadsheets.google.com/feeds',
+            'https://www.googleapis.com/auth/drive']
+
+#    credentials = ServiceAccountCredentials.from_json_keyfile_name('c:\\keys\\service\\gspread.json', scope)
+    credentials = ServiceAccountCredentials.from_json_keyfile_name('c:\\keys\\spread2.json', scope)
+
+    gc = gspread.authorize(credentials)
+    spread = Spread(filename,creds=credentials)
+
+    spread.df_to_sheet(df, index=False, sheet=tabname, start=startpos, replace=False)
+
+
+
+db = dataset.connect('mysql://hhh7796hhh:lYmWsu^ujcA1@127.0.0.1:13306/crm?charset=utf8mb4')
+
+contracts=[]
+
+def any_contract(comp):
+    global contracts
+    for c in contracts:
+        if comp in c:
+            return True
+    return False
+
+#cursor=db.query("SELECT company,contract_time FROM xoops.execute_form where contract_time>'2020-01-01' and is_delete <> 'Y'")
+cursor=db.query("SELECT distinct f.company,f.contract_time FROM xoops.execute_form f, xoops.execute_detail i  where f.exf_id=i.exf_id and f.contract_time>'2020-01-01' and f.is_delete <> 'Y'")
+
+for c in cursor:
+    contracts.append(c['company'])
+
+
+df = pd.DataFrame(columns=('company','contact','title','telephone','worktype','sales','dt'))
+idx=0
+#cursor=db.query("SELECT company_s_temp, company_f_temp,sales,create_time FROM crm.customer_contact where create_time >='2020-01-01' and create_time <='2020-21-31'")
+
+cursor=db.query("SELECT c.company_s_temp,c.company_f_temp,c.name,c.title,c.telete,m.work_type, m.sales, m.set_date FROM crm.customer_maintenance m, customer_contact c where ( (m.work_type='M') or (m.work_type='MTWR') ) and  c.cus_id=m.cus_id and m.set_date >='2020-01-01' and m.set_date <='2020-21-31'")
+
+for c in cursor:
+    if c['company_s_temp'] is None:
+        continue
+    if not any_contract(c['company_s_temp']):
+        df.loc[idx]=[c['company_f_temp'],c['name'],c['title'],c['telete'],c['work_type'],c['sales'],c['set_date']]
+        idx+=1
+
+save_sheet(df,'sales_report','report')

hhh/tests/gsearch.py

@@ -0,0 +1,107 @@
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+import time
+import os
+import urllib.parse
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+import codecs
+import random
+from bs4 import BeautifulSoup
+import requests
+import time
+import rpyc
+import sys
+import docker
+import dataset
+import re
+
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+
+
+headers = {
+        "Authorization": "Bearer " + "t35vhZtWNgvDNWHc3DJh0OKll3mcB9GvC8K2EAkBug2",
+        "Content-Type": "application/x-www-form-urlencoded"
+}
+
+
+def process_query(q):
+    global driver
+    googleurl='https://www.google.com/search?q='+urllib.parse.quote(q)
+    driver.get(googleurl)
+    time.sleep(3)
+    try:
+        elmt=driver.find_element_by_xpath("//div[@class='HlosJb bOkdDe']")
+        print(elmt.text)
+        return 0
+    except:
+        print('not found')
+    try:
+        elmt=driver.find_element_by_xpath("//div[@id='result-stats']")
+        print(elmt.text)
+        m=re.search('找到约 ([\d,]+) 条结果',elmt.text)
+        if m:
+            txt=m.group(1).replace(',','')
+            return int(txt)
+    except:
+        print('not found')
+        return 0
+#    idx=1
+
+client = docker.from_env()
+ls=client.containers.list()
+print(ls)
+ls[0].restart()
+time.sleep(12)
+
+options = webdriver.ChromeOptions()
+options.add_argument("--proxy-server=socks5://172.104.67.159:8180")
+
+
+driver = webdriver.Remote(
+    command_executor='http://127.0.0.1:4444/wd/hub',
+#command_executor='http://192.53.174.202:4444/wd/hub',
+#command_executor='http://172.104.93.163:4444/wd/hub',
+ 
+#command_executor='http://dev2.choozmo.com:14444/wd/hub',
+desired_capabilities=options.to_capabilities())
+#desired_capabilities=DesiredCapabilities.CHROME)
+driver.set_window_size(1400,1000)
+
+name=None
+designers=[]
+#cursor=db.query('select name,vip from customer_list order by updated asc limit 3')
+cursor=db.query('select name,vip from customer_list where name not in (select designer from designer_social)')
+
+for c in cursor:
+    name=c['name']
+    designers.append(name)
+    print(name)
+#    break
+
+table=db['designer_social']
+
+for d in designers:
+    m01=process_query('"'+d+'" site:mobile01.com')
+    ptt=process_query('"'+d+'" site:ptt.cc')
+    pix=process_query('"'+d+'" site:pixnet.net')
+    table.insert({'designer':d,'m01':m01,'ptt':ptt,'pix':pix})
+
+db.commit()
+
+#num=process_query('"'+name+'"')
+#process_query('彙禾設計')
+
+time.sleep(9999)
+#https://whatismyipaddress.com/ip/61.230.75.30
+#driver.get('https://whatismyipaddress.com/')
+#fw=codecs.open('c:/tmp/gg.html','w','utf-8')
+#fw.write(driver.page_source)
+#fw.close()
+#import sys
+#sys.exit()
+
+
+#### qlist=get_list()

hhh/tests/phantomtest.py

@@ -63,34 +63,29 @@ def process_query(q):
 #KPI: IS國際設計(293)陳嘉鴻 
 # 浩室設計(278)  富億設計(31)  豐聚設計(24)  上築設計(594)   京璽國際(356)  摩登雅舍(307)   優尼客(491)  比沙列(33)   亞維(385)
 # /designers/cases/(491|31|293|278|31|24|594|356|307|491|33|385)
-#qlist=['上築空間設計','豐聚室內裝修','富億空間設計','浩室設計','京璽國際','IS國際設計','優尼客','摩登雅舍','亞維空間設計','比沙列','富億設計','豐聚設計']
-#qlist=['IS國際設計']
-#qlist=['YHS DESIGN','齊舍設計','奧立佛竺居','阿曼空間設計','界陽大司室內設計','尚藝室內裝修','禾禾設計','上陽設計','苡希創意設計','羽筑空間設計','將作空間']
-#qlist=['富億空間','幸福空間節目 富億','幸福空間設計師 富億','幸福空間 富億','幸福空間 fe設計','老屋 富億','老屋翻新 富億','天井設計案例 富億','IS國際設計','豐聚設計','浩室設計','優尼客','摩登雅舍','亞維空間設計','比沙列','京璽國際','上築空間設計']
-#qlist+=['合砌設計','羽筑空間設計','朱英凱室內設計','將作空間','藝谷空間','文儀室內裝修','維耕設計','亞維空間設計','比沙列','上築空間設計','京璽國際','IS國際設計','優尼客','摩登雅舍','玖柞設計','麻石設計','元典設計','大琚空間','惹雅國際','橙果創意','PLSB建境','聯寬室內裝修']
-#qlist+=['合砌設計','羽筑空間設計','朱英凱室內設計','將作空間','藝谷空間','文儀室內裝修','維耕設計','亞維空間設計','比沙列','上築空間設計','京璽國際','IS國際設計','優尼客','摩登雅舍','玖柞設計','麻石設計','元典設計','大琚空間','惹雅國際','橙果創意','PLSB建境','聯寬室內裝修']
-
-#qlist=['卓林室內設計','成綺空間設計','德本迪設計','昱森室內設計','千綵胤空間設計','上築空間設計']
-#qlist=['IS國際設計','IS國際設計','IS國際設計 陳嘉鴻','IS國際設計 費用','IS國際設計 陳嘉鴻','IS國際設計 風格','IS國際設計 接案風格','IS國際設計 現代風','IS國際設計 老屋翻新']
-
-#qlist=['IS國際設計','陳嘉鴻設計師設計費','陳嘉鴻設計師','陳嘉鴻 費用','IS國際設計 陳嘉鴻','幸福空間陳嘉鴻設計師','陳嘉鴻設計師評價','is國際設計ptt']
-#qlist=['黃翊峰','林峰安 幸福空间','幸福空間王思文作品','黃仲均','鄭抿丹','張德良收費','周彥如','演拓設計費','演拓設計收費','邱郁雯','簡瑋琪','張靜峰設計費','周彥如設計師','陳錦樹設計師']
-#qlist=['張靜峰設計師','伊太設計','is設計','比沙列室內裝修設計有限公司','陳嘉鴻設計師設計費','澄境室內設計','上築空間設計','比沙列設計','魏子涵']
-#qlist=['京璽 幸福空間','澄境 幸福空間','雅舍王 幸福空間','豐聚設計 幸福空間','演拓設計評價 幸福','豐聚室內裝修設計','浩室 幸福','張德良評價']
-#qlist=['幸福空間 魏子涵','豐聚室內裝修設計有限公司','水設設計ptt','室設計','設計鬼才','築空間設計']
-#qlist+=['奧立佛 設計師 鍾鼎','奧立佛 老屋翻新','奧立佛 住宅改造','奧立佛 監工費','奧立佛 設計 鍾黎']
-
-#qlist=['王思文','陳嘉鴻設計師','星葉設計','邱郁雯','演拓設計收費','簡瑋琪','張靜峰設計師','伊太空間設計','伊太']
-
-#qlist=['富億空間設計','富億空間設計 新成屋','富億空間設計 新成屋','富億空間設計 老屋翻新','富億空間設計 陳錦樹','富億空間設計 設計師','富億空間設計 室內設計','富億空間設計 室內設計師']
-#qlist+=['富億空間 新成屋','富億空間 新成屋','老屋翻新','富億空間 陳錦樹','富億空間 設計師','富億空間 室內設計']
-#qlist=['富億設計','富億設計 新成屋','富億設計 新成屋','富億設計 老屋翻新','富億設計 陳錦樹','富億設計 設計師','富億設計 室內設計','富億設計 室內設計師']
 
-
-#qlist=['豐聚設計','豐聚設計 黃翊峰','豐聚設計 李羽芝','豐聚設計 休閒風','豐聚設計 現代風','豐聚設計 工業風','豐聚設計 設計大獎']
-#qlist=['京璽國際','京璽國際 周彥如','京璽國際 設計師','京璽國際 現代風','京璽國際 美式風','京璽國際 奢華風','京璽國際 新成屋']
-#qlist=['合砌設計','合砌設計 臺北市','合砌設計 南港區','合砌設計 忠孝東路','合砌設計 簡約','合砌設計 現代風','合砌設計 徐俊福']
-#qlist=['朱英凱室內設計','朱英凱室內設計 室內設計','朱英凱室內設計 國際大賞','朱英凱室內設計 GLORY','朱英凱室內設計 動線','朱英凱室內設計 設計費','朱英凱室內設計 毛坯屋']
+#qlist=['上築空間設計','豐聚室內裝修','富億空間設計','浩室設計','京璽國際','IS國際設計','優尼客','摩登雅舍','亞維空間設計','比沙列','富億設計','豐聚設計']
+#qlist+=['YHS DESIGN','齊舍設計','奧立佛竺居','阿曼空間設計','界陽大司室內設計','尚藝室內裝修','禾禾設計','上陽設計','苡希創意設計','羽筑空間設計','將作空間']
+#qlist+=['富億空間','幸福空間節目 富億','幸福空間設計師 富億','幸福空間 富億','幸福空間 fe設計','老屋 富億','老屋翻新 富億','天井設計案例 富億','IS國際設計','豐聚設計','浩室設計','優尼客','摩登雅舍','亞維空間設計','比沙列','京璽國際','上築空間設計']
+#qlist+=['卓林室內設計','成綺空間設計','德本迪設計','昱森室內設計','千綵胤空間設計','上築空間設計']
+#qlist+=['IS國際設計','陳嘉鴻設計師設計費','陳嘉鴻設計師','陳嘉鴻 費用','IS國際設計 陳嘉鴻','幸福空間陳嘉鴻設計師','陳嘉鴻設計師評價','is國際設計ptt']
+#qlist+=['黃翊峰','林峰安 幸福空间','幸福空間王思文作品','黃仲均','鄭抿丹','張德良收費','周彥如','演拓設計費','演拓設計收費','邱郁雯','簡瑋琪','張靜峰設計費','周彥如設計師','陳錦樹設計師']
+#qlist+=['張靜峰設計師','伊太設計','is設計','比沙列室內裝修設計有限公司','陳嘉鴻設計師設計費','澄境室內設計','上築空間設計','比沙列設計','魏子涵']
+#qlist+=['幸福空間 魏子涵','豐聚室內裝修設計有限公司','水設設計ptt','室設計','設計鬼才','築空間設計']
+#qlist+=['富億設計','富億設計 新成屋','富億設計 新成屋','富億設計 老屋翻新','富億設計 陳錦樹','富億設計 設計師','富億設計 室內設計','富億設計 室內設計師']
+#qlist+=['京璽國際','京璽國際 周彥如','京璽國際 設計師','京璽國際 現代風','京璽國際 美式風','京璽國際 奢華風','京璽國際 新成屋']
+
+qlist=['IS國際設計']
+qlist+=['合砌設計','羽筑空間設計','朱英凱室內設計','將作空間','藝谷空間','文儀室內裝修','維耕設計','亞維空間設計','比沙列','上築空間設計','京璽國際','IS國際設計','優尼客','摩登雅舍','玖柞設計','麻石設計','元典設計','大琚空間','惹雅國際','橙果創意','PLSB建境','聯寬室內裝修']
+qlist+=['IS國際設計','IS國際設計','IS國際設計 陳嘉鴻','IS國際設計 費用','IS國際設計 陳嘉鴻','IS國際設計 風格','IS國際設計 接案風格','IS國際設計 現代風','IS國際設計 老屋翻新']
+qlist+=['京璽 幸福空間','澄境 幸福空間','雅舍王 幸福空間','豐聚設計 幸福空間','演拓設計評價 幸福','豐聚室內裝修設計','浩室 幸福','張德良評價']
+qlist+=['奧立佛 設計師 鍾鼎','奧立佛 老屋翻新','奧立佛 住宅改造','奧立佛 監工費','奧立佛 設計 鍾黎']
+qlist+=['王思文','陳嘉鴻設計師','星葉設計','邱郁雯','演拓設計收費','簡瑋琪','張靜峰設計師','伊太空間設計','伊太']
+qlist+=['富億空間設計','富億空間設計 新成屋','富億空間設計 新成屋','富億空間設計 老屋翻新','富億空間設計 陳錦樹','富億空間設計 設計師','富億空間設計 室內設計','富億空間設計 室內設計師']
+qlist+=['富億空間 新成屋','富億空間 新成屋','老屋翻新','富億空間 陳錦樹','富億空間 設計師','富億空間 室內設計']
+qlist+=['豐聚設計','豐聚設計 黃翊峰','豐聚設計 李羽芝','豐聚設計 休閒風','豐聚設計 現代風','豐聚設計 工業風','豐聚設計 設計大獎']
+qlist+=['合砌設計','合砌設計 臺北市','合砌設計 南港區','合砌設計 忠孝東路','合砌設計 簡約','合砌設計 現代風','合砌設計 徐俊福']
+qlist+=['朱英凱室內設計','朱英凱室內設計 室內設計','朱英凱室內設計 國際大賞','朱英凱室內設計 GLORY','朱英凱室內設計 動線','朱英凱室內設計 設計費','朱英凱室內設計 毛坯屋']
 #qlist=['演拓設計','演拓設計 殷崇淵','演拓設計 張德良','演拓設計 楊霈瀅','演拓設計 寬寬','演拓設計 廖文祥','演拓設計 設計裝潢']
 #qlist=['玖柞設計','玖柞設計 朱伯晟','玖柞設計 蔡雅怡','玖柞設計 新宅','玖柞設計 室內設計','玖柞設計 建築','玖柞設計 裝潢設計']
 #qlist=['摩登雅舍','摩登雅舍 汪忠錠','摩登雅舍 王思文','摩登雅舍 室內裝修','摩登雅舍 手作牆','摩登雅舍 浮雕','摩登雅舍 南歐']
@@ -168,39 +163,63 @@ result.append({'company':'IS國際設計','designers':['陳嘉鴻'],'fullname':'
 result.append({'company':'豐聚設計','designers':['黃翊峰','李羽芝'],'fullname':'豐聚室內裝修設計'})
 result.append({'company':'優尼客','designers':['黃仲均'],'fullname':'優尼客空間設計'})
 result.append({'company':'富億設計','designers':['陳錦樹'],'fullname':'富億空間設計'})
-qlist=[]
-for r in result:
-    ds=r['designers']
-    fullname=r['fullname']
-    abvname=r['company']
-    qlist.append(fullname)
-    qlist.append(fullname+"設計師")
-    qlist.append(fullname+" 設計師")
-    qlist.append(abvname+"設計師")
-    qlist.append(abvname+" 設計師")
-    qlist.append(abvname+"費用")
-    qlist.append(abvname+"評價")
-    qlist.append(abvname+" 費用")
-    qlist.append(abvname+" 評價")
-    qlist.append(abvname+"專線")
-    qlist.append(abvname+"免費專線")
-    qlist.append(abvname+"ptt")
-    qlist.append("幸福空間"+abvname)
-    for d in ds:
-#        qlist.append(d+"設計師")
-        qlist.append(abvname+d)
-        qlist.append(fullname+d)
-        qlist.append(d+"設計師設計費")
-        qlist.append(d+"設計師費用")
-        qlist.append(d+"設計師評價")
-        qlist.append("幸福空間"+d+"設計師")
-        qlist.append("幸福空間設計師"+d)
-        qlist.append("幸福 設計師 "+d)
-        qlist.append(d+"免費專線")
+
+result.append({'company':'三宅一秀','designers':['郁琇琇','天使總監郁琇琇'],'fullname':'三宅一秀空間創藝有限公司'})
+result.append({'company':'元均','designers':['馬愷君','設計總監馬愷君'],'fullname':'元均制作空間設計'})
+result.append({'company':'格綸設計','designers':['馬愷君','設計總監馬愷君'],'fullname':'格綸設計工程'})
+result.append({'company':'雅典設計','designers':['雅典設計'],'fullname':'雅典設計工程'})
+result.append({'company':'雲方設計','designers':['潘仕敏'],'fullname':'雲方室內裝修設計'})
+result.append({'company':'采品設計','designers':['盧慧珊'],'fullname':'采品室內設計'})
+result.append({'company':'恆岳設計','designers':['蔡岳儒'],'fullname':'恆岳空間設計'})
+result.append({'company':'唐林設計','designers':['廖韋強'],'fullname':'唐林建築室內設計'})
+result.append({'company':'恆岳設計','designers':['蔡岳儒'],'fullname':'恆岳空間設計'})
+result.append({'company':'澤序設計','designers':['張于廷'],'fullname':'澤序空間設計'})
+result.append({'company':'允庭設計','designers':['張舜淵','林術榮','李晴沛'],'fullname':'允庭室內裝修設計'})
+result.append({'company':'禾亞國際','designers':['李柔瑩'],'fullname':'禾亞國際室內裝修'})
+result.append({'company':'禾築設計','designers':['譚淑靜'],'fullname':'禾築國際設計'})
+result.append({'company':'我思空間','designers':['陳佳佑'],'fullname':'我思空間設計'})
+result.append({'company':'沛沛設計','designers':['郭沛沛'],'fullname':'沛沛國際室內設計'})
+
+
+
+def gen_long():
+    global qlist
+    for r in result:
+        ds=r['designers']
+        fullname=r['fullname']
+        abvname=r['company']
+        qlist.append(fullname)
+        qlist.append(fullname+"設計師")
+        qlist.append(fullname+" 設計師")
+        qlist.append(abvname+"設計師")
+        qlist.append(abvname+" 設計師")
+#        qlist.append(abvname+"費用")
+#        qlist.append(abvname+"評價")
+#        qlist.append(abvname+" 費用")
+#        qlist.append(abvname+" 評價")
+#        qlist.append(abvname+"專線")
+#        qlist.append(abvname+"免費專線")
+#        qlist.append(abvname+"ptt")
+#        qlist.append("幸福空間"+abvname)
+        for d in ds:
+            qlist.append(d+" 設計師")
+            qlist.append(d+"設計師")
+
+#            qlist.append(abvname+d)
+#            qlist.append(fullname+d)
+#            qlist.append(d+"設計師設計費")
+#            qlist.append(d+"設計師費用")
+#            qlist.append(d+"設計師評價")
+#            qlist.append("幸福空間"+d+"設計師")
+#            qlist.append("幸福空間設計師"+d)
+#            qlist.append("幸福 設計師 "+d)
+#            qlist.append(d+"免費專線")
 
 print(qlist)
 #sys.exit()
 
+#qlist=['IS國際設計費用?','IS國際設計電話?','上築設計的室內設計師是誰?','上築設計獎項?','上築設計免費專線?']
+
 client = docker.from_env()
 ls=client.containers.list()
 print(ls)
@@ -251,8 +270,8 @@ driver.set_window_size(1400,1000)
 
 while True:
     q=random.choice(qlist)
-    prob=random.randint(0,4)
-#    prob=random.randint(0,2)
+#    prob=random.randint(0,4)
+    prob=random.randint(0,2)
 
     if prob <=1:
     #    q=qlist[0]
@@ -262,7 +281,9 @@ while True:
     else:
         empty_query(q)
     #    intsleep=random.randint(20,40)
-    intsleep=random.randint(7,20)
+#    intsleep=random.randint(7,20)
+    intsleep=random.randint(9,20)
+
 #    intsleep=random.randint(5,12)
 #    intsleep=random.randint(3,6)
 

hhh/tests/seo_designer_worker.py

@@ -90,11 +90,17 @@ def process_query(q):
 
 #qlist=['德本迪設計','德本迪設計 宜荷','德本迪設計 室內設計','德本迪設計 裝修','德本迪設計 宋雯鈴','德本迪設計 宋志鍾','德本迪設計 室內設計諮詢','德本迪設計 系統家具']
 
+options = webdriver.ChromeOptions()
+options.add_argument("--proxy-server=socks5://172.104.67.159:8180")
+
 driver = webdriver.Remote(
-#    command_executor='http://127.0.0.1:4444/wd/hub',
+    command_executor='http://127.0.0.1:4444/wd/hub',
 #command_executor='http://192.53.174.202:4444/wd/hub',
-command_executor='http://dev2.choozmo.com:14444/wd/hub',
-desired_capabilities=DesiredCapabilities.CHROME)
+#command_executor='http://172.104.93.163:4444/wd/hub',
+ 
+#command_executor='http://dev2.choozmo.com:14444/wd/hub',
+desired_capabilities=options.to_capabilities())
+#desired_capabilities=DesiredCapabilities.CHROME)
 driver.set_window_size(1400,1000)
 
 

hhh/tests/seo_gen_designer.py

@@ -0,0 +1,27 @@
+import traceback
+import copy
+import operator
+import codecs
+import sys
+import os
+import searchconsole
+import dataset
+import datetime
+import time
+
+#db = dataset.connect('mysql://choozmo:pAssw0rd@127.0.0.1:3306/hhh?charset=utf8mb4')
+
+result=[]
+
+result.append({'company':'上築','designers':['魏子涵'],'fullname':'上築空間設計'})
+result.append({'company':'摩登雅舍','designers':['王思文','汪忠錠'],'fullname':'摩登雅舍室內裝修設計'})
+result.append({'company':'亞維','designers':['簡瑋琪'],'fullname':'亞維空間設計'})
+result.append({'company':'比沙列','designers':['張靜峰'],'fullname':'比沙列室內裝修設計'})
+result.append({'company':'京璽國際','designers':['周彥如'],'fullname':'京璽國際'})
+result.append({'company':'京築聯合','designers':['周彥如'],'fullname':'京築聯合室內裝修'})
+result.append({'company':'浩室設計','designers':['浩室設計'],'fullname':'浩室設計'})
+result.append({'company':'IS國際設計','designers':['陳嘉鴻'],'fullname':'IS國際設計'})
+result.append({'company':'豐聚設計','designers':['黃翊峰','李羽芝'],'fullname':'豐聚室內裝修設計'})
+result.append({'company':'優尼客','designers':['黃仲均'],'fullname':'優尼客空間設計'})
+result.append({'company':'富億設計','designers':['陳錦樹'],'fullname':'富億空間設計'})
+

hhh/tests/seo_gsc_dump.py

@@ -0,0 +1,68 @@
+import traceback
+import copy
+import operator
+import codecs
+import sys
+import os
+import searchconsole
+import dataset
+import datetime
+import time
+
+#SELECT * FROM hhh.gsc_designer where
+# (`page` = 'https://hhh.com.tw/designers/cases/491/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/31/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/293/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/278/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/24/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/594/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/356/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/307/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/491/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/33/1-page/new-sort/') or
+# (`page` = 'https://hhh.com.tw/designers/cases/385/1-page/new-sort/') 
+# order by impressions/position desc;
+
+
+
+fname=os.path.abspath(__file__)
+elmts=fname.split(os.path.sep)
+path2=os.path.sep.join(elmts[0:-1])
+keysdir=path2+os.path.sep+'..\\keys'+os.path.sep
+
+account = searchconsole.authenticate(client_config=keysdir+'client_secret.json',credentials=keysdir+'credentials.json')
+webproperty = account['https://hhh.com.tw/']
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+db.begin()
+
+table=db['gsc_designer']
+
+report=webproperty.query.range('2021-06-19', '2020-06-19').dimension('page','query').filter('page', 'https://hhh.com.tw/designers/cases/', 'contains').get()
+total_pos=0
+total_count=0
+total_clicks=0
+records=[]
+
+ts = time.time()
+sessionid=int(ts)
+
+cnt=0
+full_list=[]
+for r in report:
+#    print(r)
+    page=r[0]
+    query=r[1]
+    clicks=r[2]
+    impressions=r[3]
+    ctr=r[4]
+    position=r[5]
+    data={'sessionid':sessionid,'page':page,'query':query,'clicks':int(clicks),'impressions':int(impressions),'ctr':float(ctr),'position':float(position),'dt':datetime.datetime.now()}
+    print(data)
+    cnt+=1
+    full_list.append(data)
+
+for l in full_list:
+    table.insert(l)
+db.commit()
+print(cnt)

hhh/tests/seo_rds_sendjob.py

@@ -1,14 +1,88 @@
 import redis
 import time
-
+import dataset
+import random
+import codecs
 #qlist=['富億空間設計','富億空間設計 新成屋','富億空間設計 新成屋','富億空間設計 老屋翻新','富億空間設計 陳錦樹','富億空間設計 設計師','富億空間設計 室內設計','富億空間設計 室內設計師']
-qlist=['上築空間設計','豐聚室內裝修','富億空間設計','浩室設計','京璽國際','IS國際設計','優尼客','摩登雅舍','亞維空間設計','比沙列','富億設計','豐聚設計']
-#qlist=['奧立佛竺居 設計師 鍾鼎','奧立佛竺居 老屋翻新','奧立佛竺居 住宅改造','奧立佛竺居 監工費','奧立佛竺居 設計 鍾黎']
-#qlist=['IS國際設計','IS國際設計','IS國際設計 陳嘉鴻','IS國際設計 費用','IS國際設計 陳嘉鴻 mobile01','IS國際設計 風格','IS國際設計 接案風格','IS國際設計 現代風','IS國際設計 老屋翻新']
+#qlist+=['上築空間設計','豐聚室內裝修','富億空間設計','浩室設計','京璽國際','IS國際設計','優尼客','摩登雅舍','亞維空間設計','比沙列','富億設計','豐聚設計']
+
+#qlist=['豐聚設計','豐聚設計 黃翊峰','豐聚設計 李羽芝','豐聚設計 休閒風','豐聚設計 現代風','豐聚設計 工業風','豐聚設計 設計大獎']
+#qlist+=['京璽國際','京璽國際 周彥如','京璽國際 設計師','京璽國際 現代風','京璽國際 美式風','京璽國際 奢華風','京璽國際 新成屋']
+#qlist+=['合砌設計','合砌設計 臺北市','合砌設計 南港區','合砌設計 忠孝東路','合砌設計 簡約','合砌設計 現代風','合砌設計 徐俊福']
+#qlist+=['朱英凱室內設計','朱英凱室內設計 室內設計','朱英凱室內設計 國際大賞','朱英凱室內設計 GLORY','朱英凱室內設計 動線','朱英凱室內設計 設計費','朱英凱室內設計 毛坯屋']
+#qlist+=['演拓設計','演拓設計 殷崇淵','演拓設計 張德良','演拓設計 楊霈瀅','演拓設計 寬寬','演拓設計 廖文祥','演拓設計 設計裝潢']
+#qlist+=['玖柞設計','玖柞設計 朱伯晟','玖柞設計 蔡雅怡','玖柞設計 新宅','玖柞設計 室內設計','玖柞設計 建築','玖柞設計 裝潢設計']
+#qlist+=['卓林室內設計','卓林室內設計 林繹寬','卓林室內設計 設計師','卓林室內設計 機能宅','卓林室內設計 室內設計','卓林室內設計 室內設計','卓林室內設計 坪效','卓林室內設計 空間設計']
+#qlist+=['合砌設計','羽筑空間設計','朱英凱室內設計','將作空間','藝谷空間','文儀室內裝修','維耕設計','亞維空間設計','比沙列','上築空間設計','京璽國際','IS國際設計','優尼客','摩登雅舍','玖柞設計','麻石設計','元典設計','大琚空間','惹雅國際','橙果創意','PLSB建境','聯寬室內裝修']
+#qlist+=['奧立佛竺居 設計師 鍾鼎','奧立佛竺居 老屋翻新','奧立佛竺居 住宅改造','奧立佛竺居 監工費','奧立佛竺居 設計 鍾黎','奧立佛竺居 謝雨竹']
+#qlist+=['IS國際設計','IS國際設計','IS國際設計 陳嘉鴻','IS國際設計 費用','IS國際設計 陳嘉鴻 mobile01','IS國際設計 風格','IS國際設計 接案風格','IS國際設計 現代風','IS國際設計 老屋翻新']
+#qlist+=['富億設計','富億設計 新成屋','富億設計 新成屋','富億設計 老屋翻新','富億設計 陳錦樹','富億設計 設計師','富億設計 室內設計','富億設計 室內設計師']
+
+
+
 #qlist=['界陽大司','界陽大司 空間設計','界陽大司 設計','界陽大司 馬健凱','界陽大司 室內設計師','界陽大司 案例','界陽大司 室內設計獎']
 
-r = redis.Redis(host='db.ptt.cx', port=6379, db=1)
-#p = r.pubsub(ignore_subscribe_messages=True)
-for q in qlist:
-    r.publish('q_hhh_seo',q)
+def get_list():
+    result=[]
+    fr=codecs.open('C:/data/Queries.csv','r','utf-8')
+    lines=fr.readlines()
+    fr.close()
+    for l in lines[1:]:
+        elmts=l.split(',')
+        pos=float(elmts[4])
+        if pos<=11:
+            result.append(elmts[0])
+    return result
+
+qlist=[]
+
+def get_list_from_db():
+    global qlist
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+    cursor=db.query("SELECT kw FROM hhh.seo_designer where vipgroup='V';")
+    for c in cursor:
+        qlist.append(c['kw'])
+
+def get_list_from_db2():
+    global qlist
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+    query="""SELECT query as kw FROM hhh.gsc_designer where
+ (`page` = 'https://hhh.com.tw/designers/cases/491/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/31/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/293/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/278/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/24/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/594/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/356/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/307/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/491/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/33/1-page/new-sort/') or
+ (`page` = 'https://hhh.com.tw/designers/cases/385/1-page/new-sort/') and position <=10 
+ order by impressions/position desc;
+    """
+    cursor=db.query(query)
+    for c in cursor:
+        qlist.append(c['kw'])
+
+
+qlist=get_list()
+#get_list_from_db()
+#get_list_from_db2()
+
+def process_one():
+    global qlist
+    r = redis.Redis(host='db.ptt.cx', port=6379, db=1)
+    #p = r.pubsub(ignore_subscribe_messages=True)
+    for i in range(12):
+        q=random.choice(qlist)
+        r.publish('q_hhh_seo',q)
+
+
+
+
+#    for q in qlist:
+#        r.publish('q_hhh_seo',q)
 
+#process_one()
+for i in range(3):
+    process_one()

hhh/tree_designers.py

@@ -0,0 +1,258 @@
+from instaloader import Instaloader, Profile
+import traceback
+import copy
+import operator
+import dataset
+import pandas as pd
+import networkx as nx
+#import pysftp
+import codecs
+import pyvis
+import sys
+import pickle
+import os
+import searchconsole
+from pyvis.network import Network
+import jieba
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import dataset
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+
+def save_sheet(df,filename,tabname,startpos='A1'):
+
+    scope = ['https://spreadsheets.google.com/feeds',
+            'https://www.googleapis.com/auth/drive']
+
+    credentials = ServiceAccountCredentials.from_json_keyfile_name('c:\\keys\\spread2.json', scope)
+#    credentials = ServiceAccountCredentials.from_json_keyfile_name('/var/keys/spread2.json', scope)
+
+    gc = gspread.authorize(credentials)
+    spread = Spread(filename,creds=credentials)
+
+    spread.df_to_sheet(df, index=False, sheet=tabname, start=startpos, replace=False)
+
+
+
+
+
+db = dataset.connect('sqlite:///:memory:')
+table=db['tmp']
+#pname='cont'
+#pname='damanwoo'
+#pname='drama'
+pname='news'
+
+#pname='www'
+
+#pname='ipromise'
+#pname='sports'
+#pname='rumor'
+#pname='korea'
+
+def get_css():
+    return ''
+#    fr=codecs.open('jared/data/css.txt','r','utf-8')
+#   lines=fr.readlines()
+#   content=' '.join(lines)
+#   fr.close()
+#   return content
+
+def modify_file(fname):
+    fr=codecs.open(fname,'r','utf-8')
+    lines=fr.readlines()
+    fr.close()
+    css=get_css()
+    content_output=''
+    for l in lines:
+        if '<body>' in l[0:10]:
+            content_output+=l
+            content_output+='\n<div id="google">\n'
+            continue
+        if '<style type="text' in l[0:22]:
+            content_output+=l
+            content_output+="\n"+css+"\n"
+            continue
+        if '<div id = "mynetwork"' in l[0:30]:
+            content_output+=l
+            content_output+='\n</div>\n'
+            continue
+
+        content_output+=l
+
+    fw=codecs.open("mod_"+fname,'w','utf-8')
+    fw.write(content_output)
+    fw.close()
+
+def checkig(kw):
+    global instl
+    global table
+    global pname
+    lst=[]
+    idx=0
+    cntdict={}
+    codelist={}
+    G=None
+#    if os.path.exists(pname):
+#        G = pickle.load( open( pname, "rb" ) )
+#    else:
+#        G = nx.Graph()
+    G = nx.Graph()
+
+    finallist=[]
+    idx=0
+    flag_break=False
+#    account = searchconsole.authenticate(client_config='c:/keys/client_secret.json',credentials='c:/keys/credentials.json')
+#    webproperty = account['https://ipromise.com.tw/']
+#    webproperty = account['https://'+pname+'.face8ook.org/']
+#    webproperty = account['https://www.damanwoo.com/']
+
+    fname=os.path.abspath(__file__)
+    elmts=fname.split(os.path.sep)
+    path2=os.path.sep.join(elmts[0:-1])
+    keysdir=path2+os.path.sep+'keys'+os.path.sep
+
+
+
+    account = searchconsole.authenticate(client_config=keysdir+'client_secret.json',credentials=keysdir+'credentials.json')
+
+#    webproperty = account['https://ipromise.com.tw/']
+#    webproperty = account['https://'+pname+'.face8ook.org/']
+#    webproperty = account['https://www.damanwoo.com/']
+    webproperty = account['https://hhh.com.tw/']
+
+
+
+#    report=webproperty.query.range('2021-06-11', '2021-06-18').dimension('page','query').get()
+    report=webproperty.query.range('2021-06-01', '2021-06-22').dimension('page','query').filter('').limit(10000).get()
+
+
+    urlq={}
+    for r in report.rows:
+        if urlq.get(r[0]) is None:
+            urlq[r[0]]=[r[1]]
+        else:
+            urlq[r[0]].append(r[1])
+
+
+    allrows=[]
+    rid=0
+    for k,v in urlq.items():
+#        if len(v)<35:
+        if len(v)<2:
+            continue
+        print(len(v))
+        for q in v:
+#            elmts=jieba.cut(q)
+            elmts=q.split(' ')
+            for elmt in elmts:
+#                print(elmt)
+                table.insert({'q':elmt,'rid':rid})
+        rid+=1
+        allrows.append([r[0],r[1] ])
+
+    db.commit()
+
+    cursor=db.query('select q,rid from tmp order by q')
+    prev=''
+    curnode=''
+
+
+
+    df = pd.DataFrame(columns=('rid','query'))
+
+
+    repdict={}
+    idx=0
+    for c in cursor:
+        if c['rid']!=prev:
+            curnode=c['q']
+            prev=c['rid']
+        else:
+            if repdict.get((curnode,c['q'])) is None:
+                repdict[(curnode,c['q'])]=1
+#                repdict[(c['q'],curnode)]=1
+                df.loc[idx]=[curnode,c['q']]
+                idx+=1
+                G.add_edge(curnode,c['q'],weight=3,width=3,borderwidth=3)
+
+#    pickle.dump( G, open( pname, "wb" ) )
+#    save_sheet(df,'BigGraph','nodes')
+
+#    G2 = [G.subgraph(c).copy() for c in nx.connected_components(G)]
+#    remove = [node for node,degree in dict(G.degree()).items() if degree <2]
+#    G.remove_nodes_from(remove)
+
+    remove=[]
+    for n in G.nodes:
+        if '承諾' in n:
+            remove.append(n)
+        if 'promise' in n:
+            remove.append(n)
+    G.remove_nodes_from(remove)
+
+
+    G.remove_edges_from(nx.selfloop_edges(G))
+    G.remove_nodes_from(list(nx.isolates(G)))
+#    lst= [G.subgraph(c).copy() for c in nx.connected_components(G)]
+#    lst=[]
+#    for c in nx.connected_components(G):
+#        cc=G.subgraph(c).copy()
+#        if cc.number_of_nodes()>7:
+#            lst.append(cc)
+
+#        if nx.diameter(cc, e=None, usebounds=False)>1:
+#            lst.append(cc)
+
+#    G2=nx.compose_all(lst)
+    G2=G
+#    pyG = Network(height="750px", width="100%",bgcolor="#333333",font_color="white")
+    pyG = Network(height="600px", width="100%",bgcolor="#444444",font_color="white")
+
+
+
+
+    pyG.from_nx(G2)
+    pyG.show(pname+'.html')
+    modify_file(pname+'.html')
+
+#    cnopts = pysftp.CnOpts()
+#    cnopts.hostkeys = None
+#    s = pysftp.Connection(host='www.choozmo.com', username='jared', password='sstc5202',cnopts=cnopts)
+#    local_path = "mod_"+pname+".html"
+#    remote_path = "/home/nginx/farmoutput/tags/"+"mod_"+pname+".html"
+#    s.put(local_path, remote_path)
+
+
+    return finallist
+
+#r=checkig('投資')
+
+#r=checkig('保險')
+#r=checkig('嘖嘖')
+#r=checkig('募資')
+#r=checkig('遠赤外線')
+r=checkig('インソール')
+#r=checkig('信用卡')
+#print(r)
+
+#        network.on( 'click', function(properties) {
+#    var ids = properties.nodes;
+#    var clickedNodes = nodes.get(ids);
+# var copyText = clickedNodes[0].label;
+# var promise = navigator.clipboard.writeText(copyText);
+#//    console.log('clicked nodes:', clickedNodes);
+#});
+