This commit is contained in:
sml2h3
2022-02-20 19:45:19 +08:00
parent d5789d3174
commit f6913ebd96
5 changed files with 18 additions and 14 deletions
+3 -2
View File
@@ -63,8 +63,7 @@ class CacheData:
def __collect_data(self, lines, base_path, is_file=False):
labels = []
caches = []
if not self.conf['Model']['Word']:
labels.append(" ")
for file in tqdm.tqdm(lines):
if is_file:
line_list = file.split('\t')
@@ -88,6 +87,8 @@ class CacheData:
else:
logger.warning("\nFile({}) has a suffix that is not allowed! We will remove it!".format(file))
labels = list(set(labels))
if not self.conf['Model']['Word']:
labels.insert(0, " ")
logger.info("\nCoolect labels is {}".format(json.dumps(labels, ensure_ascii=False)))
self.conf['System']['Path'] = base_path
self.conf['Model']['CharSet'] = labels