tokens.append("[CLS]")
segment_ids.append(0)
for token in tokens_a:
  tokens.append(token)
  segment_ids.append(0)

tokens.append("[SEP]")
segment_ids.append(0)

for token in tokens_b:
  tokens.append(token)
  segment_ids.append(1)
tokens.append("[SEP]")
segment_ids.append(1)

import tensorflow as tf
token_id = 2
vocabulary_size = 5
position = 0
max_position_size = 6
segment_id = 0
segment_size = 2

embedding_size = 4
token_embedding = tf.constant([[-3.,-2,-1, 0],[1,2,3,4], [5,6,7,8], [9,10, 11,12], [13,14,15,16]]) #size: (vocabulary_size, embedding_size)
position_embedding = tf.constant([[17., 18, 19, 20], [21,22,23,24], [25,26,27,28], [29,30,31,32], [33,34,35,36], [37,38,39,40]]) #size:(max_position_size, embedding_size)
segment_embedding = tf.constant([[41.,42,43,44], [45,46,47,48]]) #size:(segment_size, embedding_size)

token_id_onehot = tf.one_hot(token_id, vocabulary_size)
position_id_onehot = tf.one_hot(position, max_position_size)
segment_id_onehot = tf.one_hot(segment_id, segment_size)

embd_embd_onehot_impl = tf.matmul([token_id_onehot], token_embedding)
embd_token = tf.nn.embedding_lookup(token_embedding, token_id)
embd_position = tf.nn.embedding_lookup(position_embedding, position)
embd_segment = tf.nn.embedding_lookup(segment_embedding, segment_id)
embd_sum = tf.reduce_sum([embd_token, embd_position, embd_segment], axis=0)

concat_id_onehot = tf.concat([token_id_onehot, position_id_onehot, segment_id_onehot], axis=0)
concat_embedding = tf.concat([token_embedding, position_embedding, segment_embedding], axis=0)
embd_cat = tf.matmul([concat_id_onehot], concat_embedding)

with tf.Session() as sess:
    print(sess.run(embd_embd_onehot_impl)) # [[5. 6. 7. 8.]]
    print(sess.run(embd_token)) # [5. 6. 7. 8.]
    print(sess.run(embd_position)) # [17. 18. 19. 20.]
    print(sess.run(embd_segment)) # [41. 42. 43. 44.]
    print(sess.run(embd_sum)) # [63. 66. 69. 72.]
    print(sess.run(concat_embedding))
    '''
    [[-3. -2. -1.  0.]
    [ 1.  2.  3.  4.]
    [ 5.  6.  7.  8.]
    [ 9. 10. 11. 12.]
    [13. 14. 15. 16.]
    [17. 18. 19. 20.]
    [21. 22. 23. 24.]
    [25. 26. 27. 28.]
    [29. 30. 31. 32.]
    [33. 34. 35. 36.]
    [37. 38. 39. 40.]
    [41. 42. 43. 44.]
    [45. 46. 47. 48.]]
    [[63. 66. 69. 72.]]
    '''
    print(sess.run(concat_id_onehot)) # [0. 0. 1. 0. 0. 1. 0. 0. 0. 0. 0. 1. 0.]
    print(sess.run(embd_cat)) # [[63. 66. 69. 72.]]

for index in cand_indexes:
if len(masked_lms) >= num_to_predict: # 15% of total tokens
    break
...
masked_token = None
# 80% of the time, replace with [MASK]
if rng.random() < 0.8:
    masked_token = "[MASK]"
else:
    # 10% of the time, keep original
    if rng.random() < 0.5:
    masked_token = tokens[index]
    # 10% of the time, replace with random word
    else:
    masked_token = vocab_words[rng.randint(0, len(vocab_words) - 1)]

output_tokens[index] = masked_token

# input = "unaffable"
# output = ["un", "##aff", "##able"]

def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng):
#Truncates a pair of sequences to a maximum sequence length.
    while True:
    total_length = len(tokens_a) + len(tokens_b)
    if total_length <= max_num_tokens:
        break
    trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b
    assert len(trunc_tokens) >= 1
    # We want to sometimes truncate from the front and sometimes from the
    # back to add more randomness and avoid biases.
    if rng.random() < 0.5:
        del trunc_tokens[0]
    else:
        trunc_tokens.pop()

```Python

  while len(input_ids) < max_seq_length:

    input_ids.append(0)

    input_mask.append(0)

    segment_ids.append(0)

```

features = collections.OrderedDict()
features["input_ids"] = create_int_feature(input_ids)
features["input_mask"] = create_int_feature(input_mask)
features["segment_ids"] = create_int_feature(segment_ids)
features["masked_lm_positions"] = create_int_feature(masked_lm_positions)
features["masked_lm_ids"] = create_int_feature(masked_lm_ids)
features["masked_lm_weights"] = create_float_feature(masked_lm_weights)
features["next_sentence_labels"] = create_int_feature([next_sentence_label])

features["input_ids"] = create_int_feature(feature.input_ids)
features["input_mask"] = create_int_feature(feature.input_mask)
features["segment_ids"] = create_int_feature(feature.segment_ids)
features["label_ids"] = create_int_feature([feature.label_id])

├── chinese_L-12_H-768_A-12 # 中文的预训练模型
│   ├── bert_config.json # 配置文件
│   ├── bert_model.ckpt.data-00000-of-00001 # 模型文件
│   ├── bert_model.ckpt.index # 模型文件
│   ├── bert_model.ckpt.meta # 模型文件1
│   └── vocab.txt # 模型字典
├── sample_text.txt # 预训练的语料
├── create_pretraining_data.py # 将预训练的语料转换为训练所需数据
├── run_pretraining.py # 进行预训练的脚本
├── modeling.py  # 模型结构脚本
├── run_classifier.py # 分类任务脚本
├── run_squad.py # SQuAD QA任务脚本
├── extract_features.py # 提取bert特征脚本，这里的特征指的是模型的所有层的输出特征，不止是最后一层

{
  "attention_probs_dropout_prob": 0.1, #  attention的分值的dropout
  "directionality": "bidi", # 未使用
  "hidden_act": "gelu",  # FFNN的激活函数，非线性的来源之一（另一个是self-attention）
  "hidden_dropout_prob": 0.1, # FFNN的droupout rate
  "hidden_size": 768, # embedding hidden的 size
  "initializer_range": 0.02, # 初始化的范围
  "intermediate_size": 3072, # FFNN的中间层size
  "max_position_embeddings": 512, # 最大的位置embedding size，所以我们使用这个预训练模型的话，不能超过512的长度
  "num_attention_heads": 12, # attention head 的个数
  "num_hidden_layers": 1, # hidden 层数
  "pooler_fc_size": 768, # 【CLS】token出来的维度
  "pooler_num_attention_heads": 12, #未使用
  "pooler_num_fc_layers": 3, #未使用
  "pooler_size_per_head": 128, #未使用
  "pooler_type": "first_token_transform", 
  "type_vocab_size": 2, # type_vocab_size 指的是 segment_embedding的 vocabulary size，所以说我们使用这个模型，只能使用两个 segment concat 最多
  "vocab_size": 21128 #词汇表的个数
}

This is the first sentence.
This is the second sentence and also the end of the paragraph.<eop>
Another paragraph.

Another document starts here.

python create_pretraining_data.py 
	--input_file=./sample_text.txt 
	--output_file=/tmp/tf_examples.tfrecord 
	--vocab_file=$BERT_BASE_DIR/vocab.txt 
	--do_lower_case=True 
	--max_seq_length=128 
	--max_predictions_per_seq=20 
	--masked_lm_prob=0.15 
	--random_seed=12345 
	--dupe_factor=5

def main(_):
	tokenizer = tokenization.FullTokenizer(
			vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)

	input_files = []
	# 省略了文件通配符的处理，我们假设输入的文件已经传入input_files

	rng = random.Random(FLAGS.random_seed)
	instances = create_training_instances(
		input_files, tokenizer, FLAGS.max_seq_length, FLAGS.dupe_factor,
		FLAGS.short_seq_prob, FLAGS.masked_lm_prob, FLAGS.max_predictions_per_seq,
		rng)

	output_files = ....

	write_instance_to_example_files(instances, tokenizer, FLAGS.max_seq_length,
		FLAGS.max_predictions_per_seq, output_files)

1. tokens = ["[CLS], "it", "is" "a", "[MASK]", "day", "[SEP]", "I", "apple", "to", "go", "out", "[SEP]", 0,0]
2. input_mask = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0] # 13 个1 ，2 个0 表示最后两个是padding的
3. segment_ids=[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0] # 前7个0表示句子A，后6个表示句子B，最后两个是padding
4. is_random_next=False
5. masked_lm_positions=[4, 8]
   #表示Mask后为["[CLS], "it", "is" "a", "[MASK]", "day", "[SEP]", "I", "apple", "to", "go", "out", "[SEP]"] 其中apple是随机替换成的
6.masked_lm_weights=[1, 1] # 表示前2个是真正mask的被mask的，长度=len(masked_lm_positions), 如果有的mask的token是padding，则weigjt = 0。
7. masked_lm_labels=["good", "want"]

python run_pretraining.py 
	--input_file=/tmp/tf_examples.tfrecord 
	--output_dir=/tmp/pretraining_output 
	--do_train=True 
	--do_eval=True 
	--bert_config_file=$BERT_BASE_DIR/bert_config.json 
	--init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt 
	--train_batch_size=32 
	--max_seq_length=128 
	--max_predictions_per_seq=20 
	--num_train_steps=20 
	--num_warmup_steps=10 
	--learning_rate=2e-5

def model_fn(features, labels, mode, params):  

  省略...

  model = modeling.BertModel(
		  config=bert_config,
		  is_training=is_training,
		  input_ids=input_ids,
		  input_mask=input_mask,
		  token_type_ids=segment_ids,
		  use_one_hot_embeddings=use_one_hot_embeddings)

  (masked_lm_loss,
  masked_lm_example_loss, masked_lm_log_probs) = get_masked_lm_output(
		  bert_config, model.get_sequence_output(), model.get_embedding_table(),
		  masked_lm_positions, masked_lm_ids, masked_lm_weights)

  (next_sentence_loss, next_sentence_example_loss,
  next_sentence_log_probs) = get_next_sentence_output(
		  bert_config, model.get_pooled_output(), next_sentence_labels)

  total_loss = masked_lm_loss + next_sentence_loss # 多任务学习

#其实就是
#        1。 将需要计算mask的词的hidden output 取出，再过一个dense layer（以及layer norm）
#        2。 再通过embedding table相乘，得到对应的字的index（参考XLNET的公式）（+ 每个的bias）
#        3。 计算这几个词的softmax with cross entropy（*weights 再norm）
#        output:
#        - loss: 归一化的总的cross entropy loss（去掉padding 的mask的loss）
#        - per_example_loss: 每个mask位置上的cross entropy loss
#        - log_probs: 输出每个位置的的log logits

def get_masked_lm_output(bert_config, input_tensor, output_weights, positions,
			label_ids, label_weights):
	#得到masked LM的loss和log概率
	# 只需要Mask位置的Token的输出。注意我们这边的input_tensor是模型的model.get_sequence_output()
	input_tensor = gather_indexes(input_tensor, positions)

	with tf.variable_scope("cls/predictions"):
		# 在输出之前再加一个非线性变换，这些参数只是用于训练，在Fine-Tuning的时候就不用了。
		with tf.variable_scope("transform"):
			input_tensor = tf.layers.dense(
					input_tensor,
					units=bert_config.hidden_size,
					activation=modeling.get_activation(bert_config.hidden_act),
					kernel_initializer=modeling.create_initializer(
						bert_config.initializer_range))
			input_tensor = modeling.layer_norm(input_tensor)

		# output_weights是复用输入的word Embedding，所以是传入的，
		# 这里再多加一个bias。
		output_bias = tf.get_variable(
				"output_bias",
				shape=[bert_config.vocab_size],
				initializer=tf.zeros_initializer())
		# input_tensor shape(batch size* masked length, hidden size)
		# output_wights = (vocabulary size, hidden size)
		logits = tf.matmul(input_tensor, output_weights, transpose_b=True)
		logits = tf.nn.bias_add(logits, output_bias)
		log_probs = tf.nn.log_softmax(logits, axis=-1)

		# label_ids的长度是20，表示最大的MASK的Token数
		# label_ids里存放的是MASK过的Token的id
		label_ids = tf.reshape(label_ids, [-1])
		label_weights = tf.reshape(label_weights, [-1])

		one_hot_labels = tf.one_hot(
			label_ids, depth=bert_config.vocab_size, dtype=tf.float32)

		# 但是由于实际MASK的可能不到20，比如只MASK18，那么label_ids有2个0(padding)
		# 而label_weights=[1, 1, ...., 0, 0]，说明后面两个label_id是padding的，计算loss要去掉。
		per_example_loss = -tf.reduce_sum(log_probs * one_hot_labels, axis=[-1])
		numerator = tf.reduce_sum(label_weights * per_example_loss)
		denominator = tf.reduce_sum(label_weights) + 1e-5
		loss = numerator / denominator

	return (loss, per_example_loss, log_probs)

#  input_tensor : 是模型的model.get_pooled_output()即CLS对应的hidden输出
#  经过一个dense层直接进行2分类。
#  Output：
#    - loss: batch的平均loss
#    - per_example_loss: 每个样本的输出loss
#   - log_probs: 每个样本的输出log 概率

def get_next_sentence_output(bert_config, input_tensor, labels):
	#预测下一个句子是否相关的loss和log概率

	# 简单的2分类，0表示真的下一个句子，1表示随机的。这个分类器的参数在实际的Fine-Tuning
	# 会丢弃掉。
	with tf.variable_scope("cls/seq_relationship"):
		output_weights = tf.get_variable(
				"output_weights",
				shape=[2, bert_config.hidden_size],
				initializer=modeling.create_initializer(bert_config.initializer_range))
		output_bias = tf.get_variable(
				"output_bias", shape=[2], initializer=tf.zeros_initializer())

		logits = tf.matmul(input_tensor, output_weights, transpose_b=True)
		logits = tf.nn.bias_add(logits, output_bias)
		log_probs = tf.nn.log_softmax(logits, axis=-1)
		labels = tf.reshape(labels, [-1])
		one_hot_labels = tf.one_hot(labels, depth=2, dtype=tf.float32)
		per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)
		loss = tf.reduce_mean(per_example_loss)
		return (loss, per_example_loss, log_probs)

class DomainClf_Processor(DataProcessor):
#Processor for the domain classification.
# 从文本中读取训练集数据
def get_train_examples(self, data_dir):
    #See base class.
    return self._create_examples(
        self._read_tsv(os.path.join(data_dir, FLAGS.train_file)), "train")
# 从文本中读取验证集数据
def get_dev_examples(self, data_dir):
    #See base class.
    return self._create_examples(
        self._read_tsv(os.path.join(data_dir, FLAGS.dev_file)), "dev")
# 从文本中读取测试集数据
def get_test_examples(self, data_dir):
    #See base class.
    return self._create_examples(
        self._read_tsv(os.path.join(data_dir, FLAGS.test_file)), "test")
# 定义分类的label
def get_labels(self):
    #See base class.
    return ["alerts", "baike", "calculator", "call", "car_limit", "chat", "cook_book", "fm", "general_command",
            "home_command", "master_command", "music", "news", "shopping", "stock", "time", "translator", "video",
            "weather"]
# 组装训练的example，我们这是单文本分类所以 使用的是 text-∅ pair
def _create_examples(self, lines, set_type):
    #Creates examples for the training and dev sets.
    examples = []
    for (i, line) in enumerate(lines):
        guid = "%s-%s" % (set_type, i)
        text_a = tokenization.convert_to_unicode(line[0])

        label_txt = line[1].replace("__label__", "")
        label = tokenization.convert_to_unicode(label_txt)
        examples.append(
            InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
    return examples

processors = {
        "domainclf": DomainClf_Processor,
        "cola": ColaProcessor,
        ...
    }

#!/bin/bash
#description: BERT fine-tuning

export BERT_BASE_DIR=AI_QA/models/RoBERTa-tiny-clue
export DATA_DIR=AI_QA/ALBERT/1-data/command
export TRAINED_CLASSIFIER=./output
export MODEL_NAME=roberta_tiny_clue_command_gpu

export CUDA_VISIBLE_DEVICES=2
python run_classifier_serving_gpu.py 
  --task_name=command 
  --do_train=false 
  --do_eval=false 
  --do_predict=true 
  --do_export=false 
  --do_frozen=true 
  --data_dir=$DATA_DIR 
  --vocab_file=$BERT_BASE_DIR/vocab.txt 
  --test_file=test  
  --bert_config_file=$BERT_BASE_DIR/bert_config.json 
  --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt 
  --max_seq_length=128 
  --train_batch_size=32 
  --learning_rate=1e-4 
  --num_train_epochs=6.0 
  --output_dir=$TRAINED_CLASSIFIER/$MODEL_NAME

#Creates a classification model.
    model = modeling.BertModel(
        config=bert_config,
        is_training=is_training,
        input_ids=input_ids,
        input_mask=input_mask,
        token_type_ids=segment_ids,
        use_one_hot_embeddings=use_one_hot_embeddings)

    # In the demo, we are doing a simple classification task on the entire
    # segment.
    #
    # If you want to use the token-level output, use model.get_sequence_output()
    # instead.
    output_layer = model.get_pooled_output()

    hidden_size = output_layer.shape[-1].value

    output_weights = tf.get_variable(
        "output_weights", [num_labels, hidden_size],
        initializer=tf.truncated_normal_initializer(stddev=0.02))

    output_bias = tf.get_variable(
        "output_bias", [num_labels], initializer=tf.zeros_initializer())

    with tf.variable_scope("loss"):
        if is_training:
            # I.e., 0.1 dropout
            output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)

        logits = tf.matmul(output_layer, output_weights, transpose_b=True)
        logits = tf.nn.bias_add(logits, output_bias)
        probabilities = tf.nn.softmax(logits, axis=-1)
        log_probs = tf.nn.log_softmax(logits, axis=-1)

        one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)

        per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)
        loss = tf.reduce_mean(per_example_loss)

        return (loss, per_example_loss, logits, probabilities)

self.sequence_output = self.all_encoder_layers[-1]
      # The "pooler" converts the encoded sequence tensor of shape
      # [batch_size, seq_length, hidden_size] to a tensor of shape
      # [batch_size, hidden_size]. This is necessary for segment-level
      # (or segment-pair-level) classification tasks where we need a fixed
      # dimensional representation of the segment.
with tf.variable_scope("pooler"):
  # We "pool" the model by simply taking the hidden state corresponding
  # to the first token. We assume that this has been pre-trained
  first_token_tensor = tf.squeeze(self.sequence_output[:, 0:1, :], axis=1)
  self.pooled_output = tf.layers.dense(
      first_token_tensor,
      config.hidden_size,
      activation=tf.tanh,
      kernel_initializer=create_initializer(config.initializer_range))

          bert_config, is_training, input_ids, input_mask, segment_ids, label_ids,

          num_labels, use_one_hot_embeddings)

tvars = tf.trainable_variables()
initialized_variable_names = {}
scaffold_fn = None
if init_checkpoint:
    (assignment_map, initialized_variable_names
    ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)
    tf.train.init_from_checkpoint(init_checkpoint, assignment_map)

def get_assignment_map_from_checkpoint(tvars, init_checkpoint):
#Compute the union of the current variables and checkpoint variables.
assignment_map = {}
initialized_variable_names = {}

name_to_variable = collections.OrderedDict()
for var in tvars:
    name = var.name
    m = re.match("^(.*):\d+$", name)
    if m is not None:
    name = m.group(1)
    name_to_variable[name] = var

init_vars = tf.train.list_variables(init_checkpoint)

assignment_map = collections.OrderedDict()
for x in init_vars:
    (name, var) = (x[0], x[1])
    if name not in name_to_variable:
    continue
    assignment_map[name] = name
    initialized_variable_names[name] = 1
    initialized_variable_names[name + ":0"] = 1

return (assignment_map, initialized_variable_names)