广告行业中那些趣事系列：详解BERT中分类器源码

class DataProcessor(object):
    """Base class for data converters for sequence classification data sets."""
    def get_train_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the train set."""
        raise NotImplementedError()
    def get_dev_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the dev set."""
        raise NotImplementedError()
    def get_test_examples(self, data_dir):
        """Gets a collection of `InputExample`s for prediction."""
        raise NotImplementedError()
    def get_labels(self):
        """Gets the list of labels for this data set."""
        raise NotImplementedError()
    @classmethod
    def _read_tsv(cls, input_file, quotechar=None):
        """Reads a tab separated value file."""
        with tf.gfile.Open(input_file, "r") as f:
            reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
            lines = []
            for line in reader:
                lines.append(line)
            return lines

def _create_examples(self, lines, set_type):
    """Creates examples for the training and dev sets."""
    examples = []
    print("length of lines:", len(lines))
    for (i, line) in enumerate(lines):
        # print('#i:',i,line)
        if i == 0:
            continue
        guid = "%s-%s" % (set_type, i)
        try:
            label = tokenization.convert_to_unicode(line[2])
            text_a = tokenization.convert_to_unicode(line[0])
            text_b = tokenization.convert_to_unicode(line[1])
            examples.append(
                InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        except Exception:
            print('###error.i:', i, line)
    return examples

"""
将数据处理模块得到的数据转化成TFRecord文件
input:
    examples:数据格式为[(guid,text_a,text_b,label),(guid,text_a,text_b,label),....]
    label_list：标签列表
    max_seq_length：允许的句子最大长度 
    tokenizer：分词器
    output_file：TFRecord文件存储路径
output:持久化到TFRecord格式文件
"""
def file_based_convert_examples_to_features(
        examples, 
        label_list, 
        max_seq_length, 
        tokenizer, output_file):

"""
将预处理数据加工成模型需要的特征
input:
    ex_index：数据条数索引
    example：数据格式为[(guid,text_a,text_b,label),(guid,text_a,text_b,label),....]
    label_list：标签列表
    max_seq_length：允许的句子最大长度，这里如果输入句子长度不足则补0
    tokenizer：分词器
output:  feature = InputFeatures(
  input_ids=input_ids：token embedding：表示词向量，第一个词是CLS，分隔词有SEP，是单词本身
  input_mask=input_mask：position embedding：为了令transformer感知词与词之间的位置关系
  segment_ids=segment_ids：segment embedding：text_a与text_b的句子关系
  label_id=label_id：标签
  is_real_example=True)
"""
def convert_single_example(ex_index, example, 
                label_list, max_seq_length,tokenizer):
    ....
    feature = InputFeatures(
        input_ids=input_ids,
        input_mask=input_mask,
        segment_ids=segment_ids,
        label_id=label_id,
        is_real_example=True)
    return feature

"""
自定义模型估计器(model_fn_builder)
input:bert_config：bert相关的配置
      num_labels：标签的数量
      init_checkpoint：预训练模型
      learning_rate：学习率
      num_train_steps：模型训练轮数 = (训练集总数/batch_size)*epochs
      num_warmup_steps：线性地增加学习率，num_warmup_steps = num_train_steps * warmup_proportion 
      use_tpu：是否使用TPU
output:构建好的模型
"""
def model_fn_builder(bert_config, num_labels, init_checkpoint, learning_rate,
                     num_train_steps, num_warmup_steps, use_tpu,
                     use_one_hot_embeddings):
    """Returns `model_fn` closure for TPUEstimator."""
    ......
    return model_fn

"""
创建模型，主要完成两件事：第一件事是调用modeling.py中国的BertModel类创建模型；
第二件事事计算交叉熵损失loss。交叉熵的值越小，两个概率分布就越接近。
"""
def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,
                 labels, num_labels, use_one_hot_embeddings):
    """Creates a classification model."""
    # 建立一个BERT分类模型(create_model)
    model = modeling.BertModel(
        config=bert_config,
        is_training=is_training,
        input_ids=input_ids,
        input_mask=input_mask,
        token_type_ids=segment_ids,
        use_one_hot_embeddings=use_one_hot_embeddings)
    ......
    return (loss, per_example_loss, logits, probabilities)

"""
模型标准输入
从TFRecord格式文件中读取特征并转化成TensorFlow标准的数据输入格式
input:input_file:
    input_file=train_file：输入文件，可以是训练集、验证集和预测集
    seq_length=FLAGS.max_seq_length：句子最大长度
    is_training=True：是否训练标志
    drop_remainder=True：表示在少于batch_size元素的情况下是否应删除最后一批 ; 默认是不删除。
output:TensorFlow标准的格式输入
"""
def file_based_input_fn_builder(input_file, seq_length, is_training,
                                drop_remainder):
  name_to_features = {
        "input_ids": tf.FixedLenFeature([seq_length], tf.int64),
        "input_mask": tf.FixedLenFeature([seq_length], tf.int64),
        "segment_ids": tf.FixedLenFeature([seq_length], tf.int64),
        "label_ids": tf.FixedLenFeature([], tf.int64),
        "is_real_example": tf.FixedLenFeature([], tf.int64),
    }
  ......
  return input_fn

"""
Estimator对象包装由model_fn指定的模型
input：给定输入和其他一些参数
    use_tpu：是否使用TPU
    model_fn：前面构建好的模型
    config：模型运行相关的配置
    train_batch_size：训练batch大小
    eval_batch_size：验证batch大小
    predict_batch_size：预测batch大小
output：需要进行训练、计算,或预测的操作
"""
estimator = tf.contrib.tpu.TPUEstimator(
    use_tpu=FLAGS.use_tpu,
    model_fn=model_fn,
    config=run_config,
    train_batch_size=FLAGS.train_batch_size,
    eval_batch_size=FLAGS.eval_batch_size,
    predict_batch_size=FLAGS.predict_batch_size)

if FLAGS.do_train:
    train_input_fn = file_based_input_fn_builder(
        input_file=train_file,
        seq_length=FLAGS.max_seq_length,
        is_training=True,
        drop_remainder=True)
    ....
    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)

if FLAGS.do_eval:
    eval_input_fn = file_based_input_fn_builder(
            input_file=eval_file,
            seq_length=FLAGS.max_seq_length,
            is_training=False,
            drop_remainder=eval_drop_remainder)
    ....
    result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=filename)

if FLAGS.do_predict:
    predict_input_fn = file_based_input_fn_builder(
            input_file=predict_file,
            seq_length=FLAGS.max_seq_length,
            is_training=False,
            drop_remainder=predict_drop_remainder)
    ....
    result = estimator.predict(input_fn=predict_input_fn)

import tensorflow as tf
# 日志的显示等级
tf.logging.set_verbosity(tf.logging.INFO) 
# 打印提示日志
tf.logging.info("***** Runningtraining *****")
# 打印传参日志
tf.logging.info("  Num examples = %d", len(train_examples))

import tensorflow as tf
flags = tf.flags
FLAGS = flags.FLAGS
flags.DEFINE_string(
   "data_dir", None,
   "The input data dir. Should contain the .tsv files (or other datafiles) "
"for thetask.")
# 设置哪些参数是必须要传入的
flags.mark_flag_as_required("data_dir")