mirror of
https://github.com/netfun2000/dddd_trainer.git
synced 2026-08-12 23:59:25 +08:00
beta0.1
This commit is contained in:
+3
-2
@@ -63,8 +63,7 @@ class CacheData:
|
||||
def __collect_data(self, lines, base_path, is_file=False):
|
||||
labels = []
|
||||
caches = []
|
||||
if not self.conf['Model']['Word']:
|
||||
labels.append(" ")
|
||||
|
||||
for file in tqdm.tqdm(lines):
|
||||
if is_file:
|
||||
line_list = file.split('\t')
|
||||
@@ -88,6 +87,8 @@ class CacheData:
|
||||
else:
|
||||
logger.warning("\nFile({}) has a suffix that is not allowed! We will remove it!".format(file))
|
||||
labels = list(set(labels))
|
||||
if not self.conf['Model']['Word']:
|
||||
labels.insert(0, " ")
|
||||
logger.info("\nCoolect labels is {}".format(json.dumps(labels, ensure_ascii=False)))
|
||||
self.conf['System']['Path'] = base_path
|
||||
self.conf['Model']['CharSet'] = labels
|
||||
|
||||
Reference in New Issue
Block a user