123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194 |
- import re
- import glob
- import time
- import tqdm
- from datetime import datetime
- from shutil import copyfile
- import os
- a1 = "머리를 (()) 그담에 뭐 하트도 날리고 뽀뽀도 해주고 윙크도 해주고 토닥토닥 안아주고 이 두 가지를 적절하게 잘 사용하면 굉장히 인제 조금 칭찬 방법이 되는 거죠."
- a2 = "니가 이반에서 제일 머리가 좋아 그런 거는 쪼끔 안 좋습니다 왜냐면 인제 자기보다 예쁜 아이가 있는 다른 반에 가지 않을려고 {laughing} 다른 친구보다 다른 형제자매들 보다 훨씬 낫다는 식의 비교 칭찬은 가급적 하지 않는 것이 좋습니다."
- a3 = "그니까 아까 왜 너 얼마나 잘할 거 같애 칠십 퍼센트요 팔십 퍼센트요 그니까 (아이 캔 두잇)(i can do it 나 할 수 있을 거 같애요 라고 하는 건데요."
- a4 = f" "
- a = [a1, a2, a3, a4]
- pattern_1 = r'\(\(\)\)'
- pattern_2 = r'\{(\w*)\}'
- pattern_3 = r'\((\w*)\)'
- pattern_4 = r'[\(\)\{\}]'
- pattern_4_1 = r'&([\w|-]*)&'
- pattern_5 = r"[^\uAC00-\uD7A30-9a-zA-Z\s]"
- pattern_6 = r" {2,}"
- def postprocess(origin):
- _a = re.sub(pattern=pattern_1, repl='', string = origin)
- _a = re.sub(pattern=pattern_2, repl='', string = _a)
- _a = re.sub(pattern=pattern_3, repl='', string = _a)
- tmp = re.search(pattern=pattern_4, string = _a)
- if tmp:
- return None
-
- tmp = re.search(pattern=pattern_4_1, string = _a)
- if tmp:
- return None
- _a = re.sub(pattern=pattern_5, repl = " ", string = _a)
- _a = re.sub(pattern=pattern_6, repl = " ", string = _a)
- _a = _a.strip()
- if len(_a) == 0:
- return None
- else:
- return _a
- root_dir = "/root/sata/broadcast_split/data/"
- total_txt = "/root/mnt/data_aug/audio_augmentation/script/total.txt"
-
- def broadcast_move_split2_split1(time_str):
- total_txt_file = '/root/mnt/data_aug/audio_augmentation/script/txt/broadcast_total_2.txt'
-
-
-
- ogg_path = '/root/sata/etc/broadcast_split/data'
-
-
- post_none_list = []
- not_exist_list = []
- ogg_list = []
- print('glob start !!')
- if os.path.isfile(total_txt_file):
- with open(total_txt_file, 'r') as f:
- for ogg in f:
- tt = ogg.strip()
- ogg_list.append(tt)
- else:
- ogg_list = glob.glob(os.path.join(ogg_path, '**/*.ogg'), recursive=True)
- with open(total_txt_file, 'w+') as f:
- for ogg in ogg_list:
- f.write(ogg + '\n')
-
- print('glob end !!', len(ogg_list))
-
- for source_ogg in tqdm.tqdm(ogg_list):
- source_ogg = source_ogg.strip()
-
- split1_txt_file = source_ogg.strip().replace("ogg", "txt")
- split2_txt_file = source_ogg.strip().replace("ogg", "txt").replace("broadcast_split", "broadcast/split")
- if not os.path.isfile(split2_txt_file):
- not_exist_list.append([source_ogg, split2_txt_file])
- else:
- with open(split2_txt_file, 'r') as f:
- content = f.read().strip()
- post = postprocess(content)
- with open(split1_txt_file, 'w') as f:
- if post is None:
- post_none_list.append([split2_txt_file, content])
- if os.path.isfile(split1_txt_file):
- os.remove(split1_txt_file)
- tmp_ogg = split1_txt_file.replace('txt', 'ogg')
- if os.path.isfile(tmp_ogg):
- os.remove(tmp_ogg)
- else:
- f.write(post + '\n')
- if post_none_list:
- print('postprocess none = ', len(post_none_list))
- txt = f'log/post_none_{time_str}.txt'
- with open(txt, 'w+') as f:
- for item in post_none_list:
- f.write("\t".join(item) + "\n")
-
- if not_exist_list:
- print('not exist = ', len(not_exist_list))
- txt = f'log/not_exist_{time_str}.txt'
- for item in post_none_list:
- with open(txt, 'w+') as f:
- f.write("\t".join(item) + "\n")
-
-
-
-
- if __name__ == '__main__':
- now = datetime.now()
- time_str = now.astimezone().strftime('%Y_%m_%d_%H_%M_%S')
-
-
- broadcast_move_split2_split1(time_str)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
|