import re import difflib import math from itertools import groupby from operator import itemgetter def syllable_count(word): word = word.lower() count = 0 vowels = "aeiouy" if word[0] in vowels: count += 1 for index in range(1, len(word)): if word[index] in vowels and word[index - 1] not in vowels: count += 1 if word.endswith("e"): count -= 1 if count == 0: count += 1 return count def split_sentence(in_str, maxLen): re.findall(r'[\u4e00-\u9fff]+', in_str) zh_idx = [] eng_idx= [] for i in range(len(in_str)): if in_str[i] > u'\u4e00' and in_str[i] < u'\u9fff': zh_idx.append(i) else: eng_idx.append(i) space_index = [m.start() for m in re.finditer(' ', in_str)] for idx in space_index: eng_idx.remove(idx) eng_range_list = [] for k, g in groupby(enumerate(eng_idx), lambda ix : ix[0] - ix[1]): eng_range = list(map(itemgetter(1), g)) eng_range_list.append(eng_range) total_syllable = 0 for i in range(len(eng_range_list)): total_syllable += (syllable_count(in_str[eng_range_list[i][0]:eng_range_list[i][-1]+1])+0.5) for i in range(len(zh_idx)): total_syllable+=1 #final chchchchchc[en][en][en] #[en] is a vocabulary dict with occurence of image zh_eng_idx_list = [] i = 0 while i < len(in_str): if in_str[i]==' ': i+=1 if i in zh_idx: zh_eng_idx_list.append(i) i+=1 if i in eng_idx: for ls in eng_range_list: if i in ls: zh_eng_idx_list.append(ls) i = ls[-1]+1 break zh_eng_dict_list = [{'content':'','time_ratio':0}] idx = 0 current_len = 0 sen_idx = 0 while idx < len(zh_eng_idx_list): str_from_idx = '' sylla_cnt = 1 if type(zh_eng_idx_list[idx])==type([]): str_from_idx = in_str[zh_eng_idx_list[idx][0]:zh_eng_idx_list[idx][-1]+1]+' ' sylla_cnt = syllable_count(str_from_idx) else: str_from_idx = in_str[zh_eng_idx_list[idx]] if len(zh_eng_dict_list[sen_idx]['content'])+sylla_cnt>=maxLen: zh_eng_dict_list[sen_idx]['time_ratio'] = current_len/total_syllable zh_eng_dict_list.append({'content':'','time_ratio':0}) sen_idx+=1 current_len = 0 else: current_len += sylla_cnt zh_eng_dict_list[sen_idx]['content'] += str_from_idx idx+=1 total_ratio = 0 for obj in zh_eng_dict_list: total_ratio+=obj['time_ratio'] zh_eng_dict_list[-1]['time_ratio'] = 1-total_ratio return zh_eng_dict_list def parse_script(file_path,gt_list): with open(file_path, 'r',encoding="utf-8") as f: raw_lines = [line.strip() for line in f] lines = adjustSub_by_text_similarity(gt_list,raw_lines) #make dict dict_list = [] for idx in range(len(lines)): script={} script['content'] = lines[idx] time_raw = raw_lines[idx * 4 +1 ].split(' --> ') start = time_raw[0].split(':') stop = time_raw[1].split(':') script['start'] = float(start[0])*3600 + float(start[1])*60 + float(start[2].replace(',','.')) script['stop'] = float(stop[0])*3600 + float(stop[1])*60 + float(stop[2].replace(',','.')) dict_list.append(script) #merge duplicated sentences script_not_dup_list = [] for idx in range(len(dict_list)): dup_list = [] for idx_inner in range(len(dict_list)): if dict_list[idx_inner]['content']==dict_list[idx]['content']: dup_list.append(idx_inner) for dup_idx in dup_list: if dup_idx == min(dup_list): dict_list[dup_idx]['type'] = 'lead_sentence' else: dict_list[dup_idx]['type'] = 'duplicated' dict_list[dup_list[0]]['stop'] = dict_list[dup_list[-1]]['stop'] if dict_list[idx]['type'] == 'lead_sentence': script_not_dup_list.append(dict_list[idx]) #avoid subtitle overlapping ? Timeline overlapping not found currently #cut by max length----> eng seperated problem {eng_idx} #ENG counts, zh counts, space counts new_idx = 0 splitted_dict = [] for dic in script_not_dup_list: dic_idx = 0 accumulated_duration = 0 duration = dic['stop']-dic['start'] print(duration) for sub_dic in split_sentence(dic['content'],13): new_dic = {} ind_duration = duration * sub_dic['time_ratio'] new_dic['start'] = dic['start'] + accumulated_duration accumulated_duration += ind_duration new_dic['content'] = sub_dic['content'] new_dic['duration'] = ind_duration splitted_dict.append(new_dic) for obj in splitted_dict: print(obj) return splitted_dict def adjustSub_by_text_similarity(gts,gens_raw): gens = [] for idx in range(int((len(gens_raw)+1)/4)): gens.append(gens_raw[idx*4+2]) combine2 = [''.join([i,j]) for i,j in zip(gts, gts[1:])] combine3 = [''.join([i,j,k]) for i,j,k in zip(gts, gts[1:], gts[2:])] alls = gts + combine2 + combine3 adjusted = [None]*len(gens) duplicated_list = [] for idx in range(len(gens)): match_text = difflib.get_close_matches(gens[idx], alls, cutoff=0.1) if match_text[0] in duplicated_list: for mt in match_text: if mt == adjusted[idx-1] or mt not in duplicated_list: adjusted[idx] = mt break else: adjusted[idx] = match_text[0] duplicated_list.append(match_text[0]) return adjusted def trim_punctuation(s): pat_block = u'[^\u4e00-\u9fff0-9a-zA-Z]+'; pattern = u'([0-9]+{0}[0-9]+)|{0}'.format(pat_block) res = re.sub(pattern, lambda x: x.group(1) if x.group(1) else u" " ,s) return res def splitter(s): for sent in re.findall(u'[^!?，。\!\?]+[!? 。\!\?]?', s, flags=re.U): yield sent def split_by_pun(s): res = list(splitter(s)) return res def split_by_word(s): slice_size = 3 paragraph_len = len(s) slice_num = int(math.ceil(paragraph_len/slice_size)) slice_list = [] for n in range(slice_num): slice_list.append(s[n*slice_size:n*slice_size+slice_size]) return slice_list raw_str = '更糟糕的是，與大量關注相伴的並非用戶讚賞，而是 Windows 10 on ARM 的不成熟暴露無遺，以及隨之而來的如潮差評──對用戶使用體驗影響最惡劣的，莫過於 Windows 10 on ARM 僅能透過模擬兼容老舊過時的 32 位元 x86 應用，而對效能與普及度俱佳的 64 位元 x86（即 x64）應用無能為力' sub_dict = parse_script("out.txt",split_by_pun(raw_str))