我想要训练自己的 pos tag, 将句子中的产品找出. 看了前面的讨论, 不需要使用 ner, 直接训练自己的词性标注就可以了.
这个是我的训练代码:
tagger = TransformerTagger()
tagger.fit(‘data/train.csv’,
‘data/test.csv’,
‘saved_model’,
transformer=‘albert_base_zh’,
max_seq_length=20,
warmup_steps_ratio=0.1,
epochs=20,
learning_rate=5e-5)
但是运行报错:
20-10-28 09:16:46 INFO Hyperparameter:
{
“batch_size”: 32,
“epochs”: 20,
“run_eagerly”: false,
“transformer”: “albert_base_zh”,
“optimizer”: “adamw”,
“learning_rate”: 5e-05,
“weight_decay_rate”: 0,
“epsilon”: 1e-08,
“clipnorm”: 1.0,
“warmup_steps_ratio”: 0.1,
“use_amp”: false,
“max_seq_length”: 20,
“metrics”: “accuracy”
}
IndexError Traceback (most recent call last)
in
----> 1 tagger.fit(‘data/train.csv’,
2 ‘data/test.csv’,
3 ‘saved_model’,
4 transformer=‘albert_base_zh’,
5 max_seq_length=20,
~/opt/miniconda3/envs/nlp/lib/python3.8/site-packages/hanlp/components/taggers/transformers/transformer_tagger.py in fit(self, trn_data, dev_data, save_dir, transformer, optimizer, learning_rate, weight_decay_rate, epsilon, clipnorm, warmup_steps_ratio, use_amp, max_seq_length, batch_size, epochs, metrics, run_eagerly, logger, verbose, **kwargs)
55 verbose=True,
56 **kwargs):
—> 57 return super().fit(**merge_locals_kwargs(locals(), kwargs))
58
59 # noinspection PyMethodOverriding
~/opt/miniconda3/envs/nlp/lib/python3.8/site-packages/hanlp/common/component.py in fit(self, trn_data, dev_data, save_dir, batch_size, epochs, run_eagerly, logger, verbose, **kwargs)
327 logger = init_logger(name=‘train’, root_dir=save_dir, level=logging.INFO if verbose else logging.WARN)
328 logger.info(‘Hyperparameter:\n’ + self.config.to_json())
–> 329 num_examples = self.build_vocab(trn_data, logger)
330 # assert num_examples, ‘You forgot to return the number of training examples in your build_vocab’
331 logger.info(‘Building…’)
~/opt/miniconda3/envs/nlp/lib/python3.8/site-packages/hanlp/components/taggers/transformers/transformer_tagger.py in build_vocab(self, trn_data, logger)
81
82 def build_vocab(self, trn_data, logger):
—> 83 train_examples = super().build_vocab(trn_data, logger)
84 warmup_steps_per_epoch = math.ceil(train_examples * self.config.warmup_steps_ratio / self.config.batch_size)
85 self.config.warmup_steps = warmup_steps_per_epoch * self.config.epochs
~/opt/miniconda3/envs/nlp/lib/python3.8/site-packages/hanlp/common/component.py in build_vocab(self, trn_data, logger)
306
307 def build_vocab(self, trn_data, logger):
–> 308 train_examples = self.transform.fit(trn_data, **self.config)
309 self.transform.summarize_vocabs(logger)
310 return train_examples
~/opt/miniconda3/envs/nlp/lib/python3.8/site-packages/hanlp/components/taggers/transformers/transformer_transform.py in fit(self, trn_path, **kwargs)
48 self.tag_vocab = Vocab(unk_token=None)
49 num_samples = 0
—> 50 for words, tags in self.file_to_inputs(trn_path, gold=True):
51 num_samples += 1
52 self.tag_vocab.update(tags)
~/opt/miniconda3/envs/nlp/lib/python3.8/site-packages/hanlp/transform/tsv.py in file_to_inputs(self, filepath, gold)
68 def file_to_inputs(self, filepath: str, gold=True):
69 assert gold, ‘TsvTaggingFormat does not support reading non-gold files’
—> 70 yield from generator_words_tags(filepath, gold=gold, lower=self.config.get(‘lower’, False),
71 max_seq_length=self.max_seq_length)
72
~/opt/miniconda3/envs/nlp/lib/python3.8/site-packages/hanlp/utils/io_util.py in generator_words_tags(tsv_file_path, lower, gold, max_seq_length)
480 for shorter_words in split_long_sentence_into(words, max_seq_length):
481 if gold:
–> 482 shorter_tags = [cells[1] for cells in sent[offset:offset + len(shorter_words)]]
483 offset += len(shorter_words)
484 else:
~/opt/miniconda3/envs/nlp/lib/python3.8/site-packages/hanlp/utils/io_util.py in (.0)
480 for shorter_words in split_long_sentence_into(words, max_seq_length):
481 if gold:
–> 482 shorter_tags = [cells[1] for cells in sent[offset:offset + len(shorter_words)]]
483 offset += len(shorter_words)
484 else:
IndexError: list index out of range
不知道这个什么原因,是因为我的数据源的问题吗? 我的训练数据和测试数据都是这样的:
直接收费 金融服务/product
快递费/product
碳粉/product
原价 合计/product
折扣额 合计/product
92号 汽油/product
另外我需要先自己分词标组吗? 比如第一行"直接收费 金融服务/product", 我需要自己标注成 “直接/JJ 收费/VV 金融服务/product” 这样吗?
谢谢
