failure_check.py 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170
  1. from tqdm import tqdm
  2. from postprocess import postprocess
  3. import glob
  4. import os
  5. import re
  6. import re
  7. from datetime import datetime
  8. now = datetime.now()
  9. time_str = now.strftime('%Y_%m_%d_%H_%M_%S')
  10. ## none 이면 저장 안됨.
  11. def broadcast_none_check():
  12. total_txt = "/root/mnt/data_aug/audio_augmentation/script/total.txt"
  13. failure_txt = f"broadcast_none_{time_str}.txt"
  14. ogg_path = '/root/sata/broadcast/split/data'
  15. total_ogg_txt = 'broadcast_total_ogg.txt'
  16. total_txt_set = set()
  17. ogg_set = set()
  18. failure = []
  19. cnt = 0
  20. def split1_split2_compare():
  21. failure_txt = f"broadcast_split1_split2_compare_{time_str}.txt"
  22. ogg_files = glob.glob(os.path.join('/root/sata/broadcast_split/data', '**/*.ogg'), recursive=True)
  23. for ogg in tqdm(ogg_files):
  24. txt_file = ogg.strip().replace("ogg", "txt").replace("broadcast_split", "broadcast/split")
  25. if not os.path.isfile(txt_file):
  26. failure.append([ogg, txt_file])
  27. with open(failure_txt, 'w+') as ft:
  28. for fail in failure:
  29. ft.write("\t".join(fail) + "\n")
  30. print("len failire = ", len(failure))
  31. def split2_split1_compare():
  32. failure_txt = f"broadcast_split2_split1_compare_{time_str}.txt"
  33. txt_files = glob.glob(os.path.join('/root/sata/broadcast/split/data', '**/*.txt'), recursive=True)
  34. for txt in tqdm(txt_files):
  35. ogg_file = txt.strip().replace("txt", "ogg").replace("broadcast/split", "broadcast_split" )
  36. if not os.path.isfile(ogg_file):
  37. failure.append([txt, ogg_file])
  38. with open(failure_txt, 'w+') as ft:
  39. for fail in failure:
  40. ft.write("\t".join(fail) + "\n")
  41. print("len failire = ", len(failure))
  42. split2_split1_compare()
  43. return
  44. def broadcast_special_check():
  45. failure = []
  46. failure_txt = f"broadcast_special_check{time_str}.txt"
  47. pattern_1 = r'[%$#]'
  48. txt_files = glob.glob(os.path.join('/root/sata/broadcast/split/data', '**/*.txt'), recursive=True)
  49. for txt in tqdm(txt_files):
  50. with open(txt, 'r') as t:
  51. tt = t.read().strip()
  52. tt = tt.strip()
  53. tmp = re.search(pattern=pattern_1, string = tt)
  54. if tmp:
  55. failure.append([txt, tt])
  56. with open(failure_txt, 'w+') as ft:
  57. for fail in failure:
  58. ft.write("\t".join(fail) + "\n")
  59. print("len failire = ", len(failure))
  60. def broadcast_post_check():
  61. failure_list = []
  62. for source_txt in tqdm.tqdm(glob.glob(os.path.join('/root/sata/broadcast/split/data', '**/*.txt'), recursive=True)):
  63. with open(source_txt, 'r') as f:
  64. content = f.read().strip()
  65. post = postprocess(content)
  66. if post is None:
  67. failure_list.append([source_txt, content])
  68. failure_txt = f"broadcast_post.txt"
  69. with open(failure_txt, 'w+') as ft:
  70. for fail in failure_list:
  71. ft.write("\t".join(fail) + "\n")
  72. print("len = ", len(failure_list))
  73. def broadcast_post_check():
  74. total_txt = "/root/mnt/data_aug/audio_augmentation/script/total.txt"
  75. failure_txt = f"broadcast_none_{time_str}.txt"
  76. ogg_path = '/root/sata/broadcast_split/data'
  77. total_ogg_txt = 'broadcast_total_ogg.txt'
  78. total_txt_set = set()
  79. ogg_set = set()
  80. failure = []
  81. cnt = 0
  82. try:
  83. with open(total_ogg_txt, 'r') as tf:
  84. for ogg in tqdm(tf):
  85. ogg = ogg.strip()
  86. txt_file = ogg.replace('ogg', 'txt')
  87. if not os.path.isfile(txt_file):
  88. failure.append(ogg)
  89. else:
  90. cnt += 1
  91. except:
  92. tf.close()
  93. finally:
  94. with open(failure_txt, 'w+') as ft:
  95. for fail in failure:
  96. ft.write(fail + "\n")
  97. print('cnt', cnt, len(failure))
  98. return
  99. def freetalk_special_check():
  100. ogg_path = '/root/nas/data/freetalk/ogg'
  101. total_txt = 'freetalk_total.txt'
  102. failure_txt = f"freetalk_failure_{time_str}.txt"
  103. failure = []
  104. pattern = r"[^\uAC00-\uD7A30-9a-zA-Z\s]" ## 특문
  105. with open(total_txt, 'w+') as tf:
  106. for txt_file in tqdm(glob.glob(os.path.join(ogg_path, '**/*.txt'), recursive=True), leave=True):
  107. tf.write(f"{txt_file}\n")
  108. with open(txt_file, 'r') as f:
  109. content = f.read().strip()
  110. exist = re.search(pattern=pattern, string=content)
  111. if exist:
  112. failure.append((file, content))
  113. if idx == 100:
  114. break
  115. if len(failure) >= 1:
  116. with open(failure_txt, "w+") as f:
  117. for _failure in failure:
  118. f.write(f"{' '.join(_failure)}\n")
  119. return
  120. # print('test')
  121. # freetalk_special_check()
  122. if __name__ == '__main__':
  123. # broadcast_none_check()
  124. broadcast_special_check()
  125. # freetalk_special_check()