tune_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "usage: c:\\python36\\lib\\site-packages\\ipykernel_launcher.py -f C:\\Users\\daizhutao\\AppData\\Roaming\\jupyter\\runtime\\kernel-a40ecd64-c44f-4077-ab63-9ab2ce89543b.json\n",
      "                 ARG   VALUE\n",
      "__________________________________________________\n",
      "          batch_size = 64\n",
      "    bert_config_file = D:/project/python_project/bert-lstm-crf-ner/bert/chinese_L-12_H-768_A-12/bert_config.json\n",
      "                cell = lstm\n",
      "               clean = True\n",
      "                clip = 0.5\n",
      "            data_dir = D:/project/python_project/bert-lstm-crf-ner/data\n",
      "          device_map = 0\n",
      "             do_eval = True\n",
      "       do_lower_case = True\n",
      "          do_predict = True\n",
      "            do_train = True\n",
      "        dropout_rate = 0.5\n",
      "     filter_adam_var = True\n",
      "     init_checkpoint = D:/project/python_project/bert-lstm-crf-ner/bert/chinese_L-12_H-768_A-12/bert_model.ckpt\n",
      "          label_list = None\n",
      "       learning_rate = 2e-05\n",
      "           lstm_size = 128\n",
      "      max_seq_length = 128\n",
      "                 ner = ner\n",
      "          num_layers = 1\n",
      "    num_train_epochs = 3.0\n",
      "          output_dir = D:/project/python_project/bert-lstm-crf-ner/output\n",
      "save_checkpoints_steps = 500\n",
      "  save_summary_steps = 500\n",
      "           task_name = NER\n",
      "    train_batch_size = 32\n",
      "             verbose = False\n",
      "          vocab_file = D:/project/python_project/bert-lstm-crf-ner/bert\\chinese_L-12_H-768_A-12/vocab.txt\n",
      "   warmup_proportion = 0.1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#初始化配置\n",
    "import os\n",
    "import sys\n",
    "from train.train_helper import get_args_parser\n",
    "from train.bert_lstm_ner_cg_estimator import train\n",
    "\n",
    "args = get_args_parser()\n",
    "# print(args)\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = args.device_map\n",
    "args.task_name=\"NER\"\n",
    "args.do_train=True\n",
    "args.do_eval=True\n",
    "args.do_predict=True\n",
    "args.data_dir=\"D:/project/python_project/bert-lstm-crf-ner/data\"\n",
    "args.vocab_file=\"D:/project/python_project/bert-lstm-crf-ner/bert\\chinese_L-12_H-768_A-12/vocab.txt\"\n",
    "args.bert_config_file=\"D:/project/python_project/bert-lstm-crf-ner/bert/chinese_L-12_H-768_A-12/bert_config.json\"\n",
    "args.init_checkpoint=\"D:/project/python_project/bert-lstm-crf-ner/bert/chinese_L-12_H-768_A-12/bert_model.ckpt\"\n",
    "args.max_seq_length=128\n",
    "args.train_batch_size=32\n",
    "args.learning_rate=2e-5\n",
    "args.num_train_epochs=3.0\n",
    "args.output_dir=\"D:/project/python_project/bert-lstm-crf-ner/output\"\n",
    "#打印相关参数\n",
    "param_str = '\\n'.join(['%20s = %s' % (k, v) for k, v in sorted(vars(args).items())])\n",
    "print('usage: %s\\n%20s   %s\\n%s\\n%s\\n' % (' '.join(sys.argv), 'ARG', 'VALUE', '_' * 50, param_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import sys\n",
    "#sys.path.append('/home/idm/dzt/models/nlp_model/bert-lstm-crf-ner/')\n",
    "import collections\n",
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import codecs\n",
    "import pickle\n",
    "\n",
    "from train import tf_metrics\n",
    "from bert import modeling\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "\n",
    "# import\n",
    "\n",
    "from train.models import create_model, InputFeatures, InputExample\n",
    "\n",
    "__version__ = '0.1.0'\n",
    "\n",
    "__all__ = ['__version__', 'DataProcessor', 'NerProcessor', 'write_tokens', 'convert_single_example',\n",
    "           'filed_based_convert_examples_to_features', 'file_based_input_fn_builder',\n",
    "           'model_fn_builder', 'train']\n",
    "\n",
    "\n",
    "\n",
    "class DataProcessor(object):\n",
    "    \"\"\"Base class for data converters for sequence classification data sets.\"\"\"\n",
    "\n",
    "    def get_train_examples(self, data_dir):\n",
    "        \"\"\"Gets a collection of `InputExample`s for the train set.\"\"\"\n",
    "        raise NotImplementedError()\n",
    "\n",
    "    def get_dev_examples(self, data_dir):\n",
    "        \"\"\"Gets a collection of `InputExample`s for the dev set.\"\"\"\n",
    "        raise NotImplementedError()\n",
    "\n",
    "    def get_labels(self):\n",
    "        \"\"\"Gets the list of labels for this data set.\"\"\"\n",
    "        raise NotImplementedError()\n",
    "\n",
    "    @classmethod\n",
    "    def _read_data(cls, input_file):\n",
    "        \"\"\"Reads a BIO data.\"\"\"\n",
    "        with codecs.open(input_file, 'r', encoding='utf-8') as f:\n",
    "            lines = []\n",
    "            words = []\n",
    "            labels = []\n",
    "            for line in f:\n",
    "                contends = line.strip()\n",
    "                tokens = contends.split(' ')\n",
    "                if len(tokens) == 2:\n",
    "                    words.append(tokens[0])\n",
    "                    labels.append(tokens[1])\n",
    "                else:\n",
    "                    if len(contends) == 0:\n",
    "                        l = ' '.join([label for label in labels if len(label) > 0])\n",
    "                        w = ' '.join([word for word in words if len(word) > 0])\n",
    "                        lines.append([l, w])\n",
    "                        words = []\n",
    "                        labels = []\n",
    "                        continue\n",
    "                if contends.startswith(\"-DOCSTART-\"):\n",
    "                    words.append('')\n",
    "                    continue\n",
    "            return lines\n",
    "\n",
    "\n",
    "class NerProcessor(DataProcessor):\n",
    "    def __init__(self, output_dir):\n",
    "        self.labels = set()\n",
    "        self.output_dir = output_dir\n",
    "\n",
    "    def get_train_examples(self, data_dir):\n",
    "        return self._create_example(\n",
    "            self._read_data(os.path.join(data_dir, \"train.txt\")), \"train\"\n",
    "        )\n",
    "\n",
    "    def get_dev_examples(self, data_dir):\n",
    "        return self._create_example(\n",
    "            self._read_data(os.path.join(data_dir, \"dev.txt\")), \"dev\"\n",
    "        )\n",
    "\n",
    "    def get_test_examples(self, data_dir):\n",
    "        return self._create_example(\n",
    "            self._read_data(os.path.join(data_dir, \"test.txt\")), \"test\")\n",
    "\n",
    "    def get_labels(self, labels=None):\n",
    "        #传入参数可能是labels文件路径，也可能是逗号分隔的labels文本\n",
    "        if labels is not None:\n",
    "            try:\n",
    "                # 支持从文件中读取标签类型\n",
    "                if os.path.exists(labels) and os.path.isfile(labels):\n",
    "                    with codecs.open(labels, 'r', encoding='utf-8') as fd:\n",
    "                        for line in fd:\n",
    "                            self.labels.append(line.strip())\n",
    "                else:\n",
    "                    # 否则通过传入的参数，按照逗号分割\n",
    "                    self.labels = labels.split(',')\n",
    "                self.labels = set(self.labels) # to set\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "        # 通过读取train文件获取标签的方法会出现一定的风险。\n",
    "        if os.path.exists(os.path.join(self.output_dir, 'label_list.pkl')):\n",
    "            with codecs.open(os.path.join(self.output_dir, 'label_list.pkl'), 'rb') as rf:\n",
    "                self.labels = pickle.load(rf)\n",
    "        else:\n",
    "            if len(self.labels) > 0:\n",
    "                #pkl文件不存在，就按照读取的标签集合加上一些其他标签，写入pkl\n",
    "                self.labels = self.labels.union(set([\"X\", \"[CLS]\", \"[SEP]\"]))\n",
    "                with codecs.open(os.path.join(self.output_dir, 'label_list.pkl'), 'wb') as rf:\n",
    "                    pickle.dump(self.labels, rf)\n",
    "            else:\n",
    "                #如果什么都没有，都按照代码写好的标签集合\n",
    "                self.labels = [\"O\", 'B-TIM', 'I-TIM', \"B-PER\", \"I-PER\", \"B-ORG\", \"I-ORG\", \"B-LOC\", \"I-LOC\", \"X\", \"[CLS]\", \"[SEP]\"]\n",
    "        return self.labels\n",
    "\n",
    "    def _create_example(self, lines, set_type):\n",
    "        #比如set_type是train，就表示训练数据\n",
    "        examples = []\n",
    "        for (i, line) in enumerate(lines):\n",
    "            guid = \"%s-%s\" % (set_type, i)\n",
    "            #使用bert内部的tokenization包，把字符串转化成unicode\n",
    "            text = tokenization.convert_to_unicode(line[1])\n",
    "            label = tokenization.convert_to_unicode(line[0])\n",
    "            # if i == 0:\n",
    "            #     print('label: ', label)\n",
    "            #模型训练的输入类，guid为唯一数据id\n",
    "            examples.append(InputExample(guid=guid, text=text, label=label))\n",
    "        return examples\n",
    "\n",
    "    def _read_data(self, input_file):\n",
    "        \"\"\"Reads a BIO data.\"\"\"\n",
    "        with codecs.open(input_file, 'r', encoding='utf-8') as f:\n",
    "            lines = []\n",
    "            words = []\n",
    "            labels = []\n",
    "            for line in f:\n",
    "                contends = line.strip()\n",
    "                tokens = contends.split(' ')\n",
    "                if len(tokens) == 2:\n",
    "                    words.append(tokens[0])\n",
    "                    labels.append(tokens[-1])\n",
    "                else:\n",
    "                    if len(contends) == 0 and len(words) > 0:\n",
    "                        label = []\n",
    "                        word = []\n",
    "                        for l, w in zip(labels, words):\n",
    "                            if len(l) > 0 and len(w) > 0:\n",
    "                                label.append(l)\n",
    "                                self.labels.add(l)\n",
    "                                word.append(w)\n",
    "                        lines.append([' '.join(label), ' '.join(word)])\n",
    "                        words = []\n",
    "                        labels = []\n",
    "                        continue\n",
    "                if contends.startswith(\"-DOCSTART-\"):\n",
    "                    continue\n",
    "            return lines\n",
    "\n",
    "\n",
    "def write_tokens(tokens, output_dir, mode):\n",
    "    \"\"\"\n",
    "    将序列解析结果写入到文件中\n",
    "    只在mode=test的时候启用\n",
    "    :param tokens:\n",
    "    :param mode:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    if mode == \"test\":\n",
    "        path = os.path.join(output_dir, \"token_\" + mode + \".txt\")\n",
    "        wf = codecs.open(path, 'a', encoding='utf-8')\n",
    "        for token in tokens:\n",
    "            if token != \"**NULL**\":\n",
    "                wf.write(token + '\\n')\n",
    "        wf.close()\n",
    "\n",
    "\n",
    "def convert_single_example(ex_index, example, label_list, max_seq_length, tokenizer, output_dir, mode):\n",
    "    \"\"\"\n",
    "    将一个样本进行分析，然后将字转化为id, 标签转化为id,然后结构化到InputFeatures对象中\n",
    "    :param ex_index: index\n",
    "    :param example: 一个样本\n",
    "    :param label_list: 标签列表\n",
    "    :param max_seq_length:\n",
    "    :param tokenizer:\n",
    "    :param output_dir\n",
    "    :param mode:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    label_map = {}\n",
    "    # 1表示从1开始对label进行index化\n",
    "    for (i, label) in enumerate(label_list, 1): #i从1开始增加\n",
    "        label_map[label] = i\n",
    "    # 保存label->index 的map\n",
    "    if not os.path.exists(os.path.join(output_dir, 'label2id.pkl')):\n",
    "        with codecs.open(os.path.join(output_dir, 'label2id.pkl'), 'wb') as w:\n",
    "            pickle.dump(label_map, w)\n",
    "\n",
    "    #因为从训练数据读取后，字，标签标记，都是用空格分隔的\n",
    "    textlist = example.text.split(' ')\n",
    "    labellist = example.label.split(' ')\n",
    "    tokens = []\n",
    "    labels = []\n",
    "    for i, word in enumerate(textlist):\n",
    "        # 分词，如果是中文，就是分字,但是对于一些不在BERT的vocab.txt中得字符会被进行WordPice处理（例如中文的引号），可以将所有的分字操作替换为list(input)\n",
    "        token = tokenizer.tokenize(word)\n",
    "        tokens.extend(token)\n",
    "        label_1 = labellist[i]\n",
    "        for m in range(len(token)):\n",
    "            if m == 0:\n",
    "                labels.append(label_1)\n",
    "            else:  # 一般不会出现else，因为只有一个词\n",
    "                labels.append(\"X\")\n",
    "    # tokens = tokenizer.tokenize(example.text)\n",
    "    # 序列截断\n",
    "    if len(tokens) >= max_seq_length - 1:\n",
    "        tokens = tokens[0:(max_seq_length - 2)]  # -2 的原因是因为序列需要加一个句首和句尾标志\n",
    "        labels = labels[0:(max_seq_length - 2)]\n",
    "    ntokens = []\n",
    "    segment_ids = []  #segment_ids的作用是？？？\n",
    "    label_ids = []\n",
    "    ntokens.append(\"[CLS]\")  # 句子开始设置CLS 标志\n",
    "    segment_ids.append(0)\n",
    "    # append(\"O\") or append(\"[CLS]\") not sure!\n",
    "    label_ids.append(label_map[\"[CLS]\"])  # O OR CLS 没有任何影响，不过我觉得O 会减少标签个数,不过句首和句尾使用不同的标志来标注，使用CLS 也没毛病\n",
    "    for i, token in enumerate(tokens):\n",
    "        ntokens.append(token)\n",
    "        segment_ids.append(0)\n",
    "        label_ids.append(label_map[labels[i]])\n",
    "    ntokens.append(\"[SEP]\")  # 句尾添加[SEP] 标志\n",
    "    segment_ids.append(0)\n",
    "    # append(\"O\") or append(\"[SEP]\") not sure!\n",
    "    label_ids.append(label_map[\"[SEP]\"])\n",
    "    #调用bert内部的token2id函数，把token转化成bert使用的tokenid\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(ntokens)  # 将序列中的字(ntokens)转化为ID形式\n",
    "    input_mask = [1] * len(input_ids)   #mask是隐藏的token，用于模型训练\n",
    "    # label_mask = [1] * len(input_ids)\n",
    "    # padding, 使用\n",
    "    #小于序列长度的，进行补全操作\n",
    "    while len(input_ids) < max_seq_length:\n",
    "        input_ids.append(0)\n",
    "        input_mask.append(0)\n",
    "        segment_ids.append(0)\n",
    "        # we don't concerned about it!\n",
    "        label_ids.append(0)\n",
    "        ntokens.append(\"**NULL**\")\n",
    "        # label_mask.append(0)\n",
    "    # print(len(input_ids))\n",
    "    assert len(input_ids) == max_seq_length\n",
    "    assert len(input_mask) == max_seq_length\n",
    "    assert len(segment_ids) == max_seq_length\n",
    "    assert len(label_ids) == max_seq_length\n",
    "    # assert len(label_mask) == max_seq_length\n",
    "\n",
    "    # 打印部分样本数据信息\n",
    "    if ex_index < 5:\n",
    "        tf.logging.info(\"*** Example ***\")\n",
    "        tf.logging.info(\"guid: %s\" % (example.guid))\n",
    "        tf.logging.info(\"tokens: %s\" % \" \".join(\n",
    "            [tokenization.printable_text(x) for x in tokens]))\n",
    "        tf.logging.info(\"input_ids: %s\" % \" \".join([str(x) for x in input_ids]))\n",
    "        tf.logging.info(\"input_mask: %s\" % \" \".join([str(x) for x in input_mask]))\n",
    "        tf.logging.info(\"segment_ids: %s\" % \" \".join([str(x) for x in segment_ids]))\n",
    "        tf.logging.info(\"label_ids: %s\" % \" \".join([str(x) for x in label_ids]))\n",
    "        # tf.logging.info(\"label_mask: %s\" % \" \".join([str(x) for x in label_mask]))\n",
    "\n",
    "    # 结构化为一个类，使用自定义类保存训练index化数据，\n",
    "    feature = InputFeatures(\n",
    "        input_ids=input_ids,\n",
    "        input_mask=input_mask,\n",
    "        segment_ids=segment_ids,\n",
    "        label_ids=label_ids,\n",
    "        # label_mask = label_mask\n",
    "    )\n",
    "    # mode='test'的时候才有效，把token写入文件\n",
    "    write_tokens(ntokens, output_dir, mode)\n",
    "    return feature\n",
    "\n",
    "\n",
    "def filed_based_convert_examples_to_features(\n",
    "        examples, label_list, max_seq_length, tokenizer, output_file, output_dir, mode=None):\n",
    "    \"\"\"\n",
    "    将数据转化为TF_Record 结构，作为模型数据输入\n",
    "    :param examples:  样本\n",
    "    :param label_list:标签list\n",
    "    :param max_seq_length: 预先设定的最大序列长度\n",
    "    :param tokenizer: tokenizer 对象\n",
    "    :param output_file: tf.record 输出路径\n",
    "    :param mode:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    writer = tf.python_io.TFRecordWriter(output_file)\n",
    "    # 遍历训练数据\n",
    "    for (ex_index, example) in enumerate(examples):\n",
    "        if ex_index % 5000 == 0:\n",
    "            tf.logging.info(\"Writing example %d of %d\" % (ex_index, len(examples)))\n",
    "        # 对于每一个训练样本,将一个样本进行分析，然后将字转化为id, 标签转化为id,然后结构化到InputFeatures对象中\n",
    "        # 返回的是bert模型训练需要的index化token，label，mask，segment等信息\n",
    "        feature = convert_single_example(ex_index, example, label_list, max_seq_length, tokenizer, output_dir, mode)\n",
    "\n",
    "        def create_int_feature(values):\n",
    "            f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))\n",
    "            return f\n",
    "\n",
    "        features = collections.OrderedDict()#除了记录k,v，还会记录k放入的顺序\n",
    "        features[\"input_ids\"] = create_int_feature(feature.input_ids)\n",
    "        features[\"input_mask\"] = create_int_feature(feature.input_mask)\n",
    "        features[\"segment_ids\"] = create_int_feature(feature.segment_ids)\n",
    "        features[\"label_ids\"] = create_int_feature(feature.label_ids)\n",
    "        # features[\"label_mask\"] = create_int_feature(feature.label_mask)\n",
    "        # tf.train.Example/Feature 是一种协议，方便序列化？？？\n",
    "        # Example中存放features特征，放入example是为了便于把特征序列化存储\n",
    "        tf_example = tf.train.Example(features=tf.train.Features(feature=features))\n",
    "        writer.write(tf_example.SerializeToString())\n",
    "\n",
    "\n",
    "def file_based_input_fn_builder(input_file, seq_length, is_training, drop_remainder,batch_size):\n",
    "    name_to_features = {\n",
    "        \"input_ids\": tf.FixedLenFeature([seq_length], tf.int64),\n",
    "        \"input_mask\": tf.FixedLenFeature([seq_length], tf.int64),\n",
    "        \"segment_ids\": tf.FixedLenFeature([seq_length], tf.int64),\n",
    "        \"label_ids\": tf.FixedLenFeature([seq_length], tf.int64),\n",
    "        # \"label_ids\":tf.VarLenFeature(tf.int64),\n",
    "        # \"label_mask\": tf.FixedLenFeature([seq_length], tf.int64),\n",
    "    }\n",
    "\n",
    "    def _decode_record(record, name_to_features):\n",
    "        #把recod变成字典example？,可能写入，读取都是按照tensorflow的某个标准\n",
    "        example = tf.parse_single_example(record, name_to_features)\n",
    "        for name in list(example.keys()):\n",
    "            t = example[name]\n",
    "            if t.dtype == tf.int64:\n",
    "                t = tf.to_int32(t)\n",
    "            example[name] = t\n",
    "        return example\n",
    "\n",
    "    d = tf.data.TFRecordDataset(input_file)\n",
    "    if is_training:\n",
    "        d = d.repeat()\n",
    "        d = d.shuffle(buffer_size=300)\n",
    "    #通过map函数，调用_decode_record，把int64的数据转化成int32的数据，通过apply，把数据转化成batch的形式\n",
    "        d = d.apply(tf.data.experimental.map_and_batch(lambda record: _decode_record(record, name_to_features),\n",
    "                                                       batch_size=batch_size,\n",
    "                                                       num_parallel_calls=4,  # 并行处理数据的CPU核心数量，不要大于你机器的核心数\n",
    "                                                       # num_parallel_calls=tf.data.experimental.AUTOTUNE, #根据机器动态调整并行数\n",
    "                                                       drop_remainder=drop_remainder))\n",
    "    else:\n",
    "        d = d.shuffle(buffer_size=300)\n",
    "        # 通过map函数，调用_decode_record，把int64的数据转化成int32的数据，通过apply，把数据转化成batch的形式\n",
    "        d = d.apply(tf.data.experimental.map_and_batch(lambda record: _decode_record(record, name_to_features),\n",
    "                                                       batch_size=1000,\n",
    "                                                       num_parallel_calls=4,  # 并行处理数据的CPU核心数量，不要大于你机器的核心数\n",
    "                                                       # num_parallel_calls=tf.data.experimental.AUTOTUNE, #根据机器动态调整并行数\n",
    "                                                       drop_remainder=drop_remainder))\n",
    "    d = d.prefetch(buffer_size=4)\n",
    "\n",
    "    return d\n",
    "\n",
    "\n",
    "def model_fn_builder(bert_config, num_labels, init_checkpoint, learning_rate,\n",
    "                     num_train_steps, num_warmup_steps, args):\n",
    "    \"\"\"\n",
    "    构建模型\n",
    "    :param bert_config:\n",
    "    :param num_labels:\n",
    "    :param init_checkpoint:\n",
    "    :param learning_rate:\n",
    "    :param num_train_steps:\n",
    "    :param num_warmup_steps:\n",
    "    :param use_tpu:\n",
    "    :param use_one_hot_embeddings:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "\n",
    "    def model_fn(features, labels, mode, params):\n",
    "        tf.logging.info(\"*** Features ***\")\n",
    "        for name in sorted(features.keys()):\n",
    "            tf.logging.info(\"  name = %s, shape = %s\" % (name, features[name].shape))\n",
    "        input_ids = features[\"input_ids\"]\n",
    "        input_mask = features[\"input_mask\"]\n",
    "        segment_ids = features[\"segment_ids\"]\n",
    "        label_ids = features[\"label_ids\"]\n",
    "\n",
    "        print('shape of input_ids', input_ids.shape)\n",
    "        # label_mask = features[\"label_mask\"]\n",
    "        is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
    "\n",
    "        # 使用参数构建模型,input_idx 就是输入的样本idx表示，label_ids 就是标签的idx表示\n",
    "        #全部损失，分数，，预测类别\n",
    "        total_loss, logits, trans, pred_ids = create_model(\n",
    "            bert_config, is_training, input_ids, input_mask, segment_ids, label_ids,\n",
    "            num_labels, False, args.dropout_rate, args.lstm_size, args.cell, args.num_layers)\n",
    "        # tf.summary.scalar('total_loss', total_loss)\n",
    "        # tf.summary.scalar('logits',logits)\n",
    "        # tf.summary.scalar('trans',trans)\n",
    "        # tf.summary.scalar('pred_ids',pred_ids)\n",
    "        #所有需要训练的变量\n",
    "        tvars = tf.trainable_variables()\n",
    "        # 加载BERT模型，assignmen_map，加载的预训练变量值\n",
    "        if init_checkpoint:\n",
    "            (assignment_map, initialized_variable_names) = \\\n",
    "                 modeling.get_assignment_map_from_checkpoint(tvars,\n",
    "                                                             init_checkpoint)\n",
    "            tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n",
    "\n",
    "        # 打印变量名\n",
    "        # logger.info(\"**** Trainable Variables ****\")\n",
    "        #\n",
    "        # # 打印加载模型的参数\n",
    "        # for var in tvars:\n",
    "        #     init_string = \"\"\n",
    "        #     if var.name in initialized_variable_names:\n",
    "        #         init_string = \", *INIT_FROM_CKPT*\"\n",
    "        #     logger.info(\"  name = %s, shape = %s%s\", var.name, var.shape,\n",
    "        #                     init_string)\n",
    "\n",
    "        output_spec = None\n",
    "        if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "            #train_op = optimizer.optimizer(total_loss, learning_rate, num_train_steps)\n",
    "            train_op = optimization.create_optimizer(\n",
    "                 total_loss, learning_rate, num_train_steps, num_warmup_steps, False)\n",
    "            hook_dict = {}\n",
    "            hook_dict['loss'] = total_loss\n",
    "            hook_dict['global_steps'] = tf.train.get_or_create_global_step()\n",
    "            logging_hook = tf.train.LoggingTensorHook(\n",
    "                hook_dict, every_n_iter=args.save_summary_steps)\n",
    "\n",
    "            output_spec = tf.estimator.EstimatorSpec(\n",
    "                mode=mode,\n",
    "                loss=total_loss,\n",
    "                train_op=train_op,\n",
    "                training_hooks=[logging_hook])\n",
    "\n",
    "        elif mode == tf.estimator.ModeKeys.EVAL:\n",
    "            # 针对NER ,进行了修改\n",
    "            def metric_fn(label_ids, pred_ids):\n",
    "                return {\n",
    "                    \"eval_loss\": tf.metrics.mean_squared_error(labels=label_ids, predictions=pred_ids),\n",
    "                }\n",
    "\n",
    "            eval_metrics = metric_fn(label_ids, pred_ids)\n",
    "            output_spec = tf.estimator.EstimatorSpec(\n",
    "                mode=mode,\n",
    "                loss=total_loss,\n",
    "                eval_metric_ops=eval_metrics\n",
    "            )\n",
    "        else:\n",
    "            output_spec = tf.estimator.EstimatorSpec(\n",
    "                mode=mode,\n",
    "                predictions=pred_ids\n",
    "            )\n",
    "        return output_spec\n",
    "\n",
    "    return model_fn\n",
    "\n",
    "\n",
    "# def load_data():\n",
    "#     processer = NerProcessor()\n",
    "#     processer.get_labels()\n",
    "#     example = processer.get_train_examples(FLAGS.data_dir)\n",
    "#     print()\n",
    "\n",
    "def get_last_checkpoint(model_path):\n",
    "    if not os.path.exists(os.path.join(model_path, 'checkpoint')):\n",
    "        tf.logging.info('checkpoint file not exits:'.format(os.path.join(model_path, 'checkpoint')))\n",
    "        return None\n",
    "    last = None\n",
    "    with codecs.open(os.path.join(model_path, 'checkpoint'), 'r', encoding='utf-8') as fd:\n",
    "        for line in fd:\n",
    "            line = line.strip().split(':')\n",
    "            if len(line) != 2:\n",
    "                continue\n",
    "            if line[0] == 'model_checkpoint_path':\n",
    "                last = line[1][2:-1]\n",
    "                break\n",
    "    return last\n",
    "\n",
    "\n",
    "def adam_filter(model_path):\n",
    "    \"\"\"\n",
    "    去掉模型中的Adam相关参数，这些参数在测试的时候是没有用的\n",
    "    :param model_path:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    last_name = get_last_checkpoint(model_path)\n",
    "    if last_name is None:\n",
    "        return\n",
    "    sess = tf.Session()\n",
    "    imported_meta = tf.train.import_meta_graph(os.path.join(model_path, last_name + '.meta'))\n",
    "    imported_meta.restore(sess, os.path.join(model_path, last_name))\n",
    "    need_vars = []\n",
    "    for var in tf.global_variables():\n",
    "        if 'adam_v' not in var.name and 'adam_m' not in var.name:\n",
    "            need_vars.append(var)\n",
    "    saver = tf.train.Saver(need_vars)\n",
    "    saver.save(sess, os.path.join(model_path, 'model.ckpt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:***** Running training *****\n",
      "INFO:tensorflow:  Num examples = 20864\n",
      "INFO:tensorflow:  Batch size = 64\n",
      "INFO:tensorflow:  Num steps = 978\n",
      "INFO:tensorflow:***** Running evaluation *****\n",
      "INFO:tensorflow:  Num examples = 2318\n",
      "INFO:tensorflow:  Batch size = 64\n"
     ]
    }
   ],
   "source": [
    "os.environ['CUDA_VISIBLE_DEVICES'] = args.device_map\n",
    "\n",
    "#一个处理的类，包括训练数据的输入等\n",
    "processors = {\n",
    "    \"ner\": NerProcessor\n",
    "}\n",
    "#载入bert配置文件\n",
    "bert_config = modeling.BertConfig.from_json_file(args.bert_config_file)\n",
    "\n",
    "#检查序列的最大长度是否超出范围\n",
    "if args.max_seq_length > bert_config.max_position_embeddings:\n",
    "    raise ValueError(\n",
    "        \"Cannot use sequence length %d because the BERT model \"\n",
    "        \"was only trained up to sequence length %d\" %\n",
    "        (args.max_seq_length, bert_config.max_position_embeddings))\n",
    "\n",
    "# 在re train 的时候，才删除上一轮产出的文件，在predicted 的时候不做clean\n",
    "if args.clean and args.do_train:\n",
    "    if os.path.exists(args.output_dir):\n",
    "        def del_file(path):\n",
    "            ls = os.listdir(path)\n",
    "            for i in ls:\n",
    "                c_path = os.path.join(path, i)\n",
    "                if os.path.isdir(c_path):\n",
    "                    del_file(c_path)\n",
    "                else:\n",
    "                    os.remove(c_path)\n",
    "        try:\n",
    "            del_file(args.output_dir)\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            print('pleace remove the files of output dir and data.conf')\n",
    "            exit(-1)\n",
    "\n",
    "#check output dir exists\n",
    "if not os.path.exists(args.output_dir):\n",
    "    os.mkdir(args.output_dir)\n",
    "\n",
    "#通过output_dir初始化数据处理类，processor\n",
    "processor = processors[args.ner](args.output_dir)\n",
    "#通过bert字典，初始化bert自带分词类\n",
    "tokenizer = tokenization.FullTokenizer(\n",
    "    vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)\n",
    "\n",
    "\n",
    "train_examples = None\n",
    "eval_examples = None\n",
    "num_train_steps = None\n",
    "num_warmup_steps = None\n",
    "\n",
    "#一般都是True\n",
    "if args.do_train and args.do_eval:\n",
    "    # 加载训练数据,train和dev，会自动拼接文件夹和train.txt\n",
    "    #返回的训练数据是一个list，每个元素是两个字符串，空格分隔字，空格分隔字标记，并写入训练examples类中\n",
    "    train_examples = processor.get_train_examples(args.data_dir)\n",
    "    #训练步数\n",
    "    num_train_steps = int(\n",
    "        len(train_examples) *1.0 / args.batch_size * args.num_train_epochs)\n",
    "    if num_train_steps < 1:\n",
    "        raise AttributeError('training data is so small...')\n",
    "    #\n",
    "    num_warmup_steps = int(num_train_steps * args.warmup_proportion)\n",
    "\n",
    "    tf.logging.info(\"***** Running training *****\")\n",
    "    tf.logging.info(\"  Num examples = %d\", len(train_examples))\n",
    "    tf.logging.info(\"  Batch size = %d\", args.batch_size)\n",
    "    tf.logging.info(\"  Num steps = %d\", num_train_steps)\n",
    "    #读取验证集\n",
    "    eval_examples = processor.get_dev_examples(args.data_dir)\n",
    "\n",
    "    # 打印验证集数据信息\n",
    "    tf.logging.info(\"***** Running evaluation *****\")\n",
    "    tf.logging.info(\"  Num examples = %d\", len(eval_examples))\n",
    "    tf.logging.info(\"  Batch size = %d\", args.batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_list = processor.get_labels()\n",
    "# label_list=[\"O\", 'B-TIM', 'I-TIM', \"B-PER\", \"I-PER\", \"B-ORG\", \"I-ORG\", \"B-LOC\", \"I-LOC\", \"X\", \"[CLS]\", \"[SEP]\"]\n",
    "# label_list=['B-PER', '[SEP]', 'I-ORG', 'O', 'I-LOC', 'I-PER', 'B-ORG', 'B-LOC', '[CLS]', 'X']\n",
    "num_labels = len(label_list) + 1\n",
    "init_checkpoint = args.init_checkpoint\n",
    "learning_rate = args.learning_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From D:\\project\\python_project\\bert-lstm-crf-ner\\bert\\modeling.py:673: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From c:\\python36\\lib\\site-packages\\tensorflow\\contrib\\crf\\python\\ops\\crf.py:213: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From c:\\python36\\lib\\site-packages\\tensorflow\\python\\ops\\rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From c:\\python36\\lib\\site-packages\\tensorflow\\python\\ops\\metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From c:\\python36\\lib\\site-packages\\tensorflow\\python\\training\\learning_rate_decay_v2.py:321: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python36\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_impl.py:110: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "with tf.name_scope('input'):\n",
    "    input_ids = tf.placeholder(tf.int32, [None, args.max_seq_length])\n",
    "    input_mask = tf.placeholder(tf.int32, [None, args.max_seq_length])\n",
    "    segment_ids  = tf.placeholder(tf.int32, [None, args.max_seq_length])\n",
    "    label_ids = tf.placeholder(tf.int32, [None, args.max_seq_length])\n",
    "    # is_training=tf.placeholder(tf.bool)\n",
    "    #对参数赋值，对于训练模型来说\n",
    "\n",
    "#is_training全部变成false，不使用dropout层\n",
    "total_loss, logits, trans, pred_ids = create_model(\n",
    "    bert_config, False, input_ids, input_mask, segment_ids, label_ids,\n",
    "    num_labels, False, args.dropout_rate, args.lstm_size, args.cell, args.num_layers)\n",
    "# total_loss_evl, logits_evl, trans_evl, pred_ids_evl = create_model(\n",
    "#     bert_config, False, input_ids, input_mask, segment_ids, label_ids,\n",
    "#     num_labels, False, args.dropout_rate, args.lstm_size, args.cell, args.num_layers)\n",
    "accuracy, acc_op = tf.metrics.accuracy(labels=label_ids,predictions=pred_ids)   # 计算准确率,pred_ids是预测序列，\n",
    "# accuracy_evl, acc_op_evl = tf.metrics.accuracy(labels=label_ids, predictions=pred_ids_evl)  # 计算准确率,pred_ids是预测序列\n",
    "#输出loss的smmary\n",
    "tf.summary.scalar('total_loss', total_loss)\n",
    "tf.summary.scalar('accuracy', acc_op)\n",
    "#---------------------输出验证集，测试集数据------------------------------\n",
    "# is_training_evl = False #bert模型不采用training模式\n",
    "# total_loss_evl, logits_evl, trans_evl, pred_ids_evl = create_model(\n",
    "#     bert_config, is_training_evl, input_ids, input_mask, segment_ids, label_ids,\n",
    "#     num_labels, False, args.dropout_rate, args.lstm_size, args.cell, args.num_layers)\n",
    "# accuracy_evl, acc_op_evl = tf.metrics.accuracy(labels=label_ids, predictions=pred_ids_evl)  # 计算准确率,pred_ids是预测序列\n",
    "# tf.summary.scalar('total_loss_evl', total_loss_evl)\n",
    "# tf.summary.scalar('accuracy_evl', acc_op_evl)\n",
    "#----------------------------------------------------------------------------\n",
    "#加载预训练隐变量\n",
    "tvars = tf.trainable_variables()\n",
    "# 加载BERT模型，assignmen_map，加载的预训练变量值\n",
    "if init_checkpoint:  #只会运行一次\n",
    "    (assignment_map, initialized_variable_names) = \\\n",
    "        modeling.get_assignment_map_from_checkpoint(tvars,\n",
    "                                                    init_checkpoint)\n",
    "    tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n",
    "#优化loss\n",
    "train_op = optimization.create_optimizer(\n",
    "    total_loss, learning_rate, num_train_steps, num_warmup_steps, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Writing example 0 of 20864\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-0\n",
      "INFO:tensorflow:tokens: 海 钓 比 赛 地 点 在 厦 门 与 金 门 之 间 的 海 域 。\n",
      "INFO:tensorflow:input_ids: 101 3862 7157 3683 6612 1765 4157 1762 1336 7305 680 7032 7305 722 7313 4638 3862 1818 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 10 3 5 10 3 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-1\n",
      "INFO:tensorflow:tokens: 这 座 依 山 傍 水 的 博 物 馆 由 国 内 一 流 的 设 计 师 主 持 设 计 ， 整 个 建 筑 群 精 美 而 恢 宏 。\n",
      "INFO:tensorflow:input_ids: 101 6821 2429 898 2255 988 3717 4638 1300 4289 7667 4507 1744 1079 671 3837 4638 6392 6369 2360 712 2898 6392 6369 8024 3146 702 2456 5029 5408 5125 5401 5445 2612 2131 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-2\n",
      "INFO:tensorflow:tokens: 但 作 为 一 个 共 产 党 员 、 人 民 公 仆 ， 应 当 胸 怀 宽 阔 ， 真 正 做 到 [UNK] 先 天 下 之 忧 而 忧 ， 后 天 下 之 乐 而 乐 [UNK] ， 淡 化 个 人 的 名 利 得 失 和 宠 辱 悲 喜 ， 把 改 革 大 业 摆 在 首 位 ， 这 样 才 能 超 越 自 我 ， 摆 脱 世 俗 ， 有 所 作 为 。\n",
      "INFO:tensorflow:input_ids: 101 852 868 711 671 702 1066 772 1054 1447 510 782 3696 1062 789 8024 2418 2496 5541 2577 2160 7333 8024 4696 3633 976 1168 100 1044 1921 678 722 2569 5445 2569 8024 1400 1921 678 722 727 5445 727 100 8024 3909 1265 702 782 4638 1399 1164 2533 1927 1469 2143 6802 2650 1599 8024 2828 3121 7484 1920 689 3030 1762 7674 855 8024 6821 3416 2798 5543 6631 6632 5632 2769 8024 3030 5564 686 921 8024 3300 2792 868 711 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-3\n",
      "INFO:tensorflow:tokens: 在 发 达 国 家 ， 急 救 保 险 十 分 普 及 ， 已 成 为 社 会 保 障 体 系 的 重 要 组 成 部 分 。\n",
      "INFO:tensorflow:input_ids: 101 1762 1355 6809 1744 2157 8024 2593 3131 924 7372 1282 1146 3249 1350 8024 2347 2768 711 4852 833 924 7397 860 5143 4638 7028 6206 5299 2768 6956 1146 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: train-4\n",
      "INFO:tensorflow:tokens: 日 俄 两 国 国 内 政 局 都 充 满 变 数 ， 尽 管 日 俄 关 系 目 前 是 历 史 最 佳 时 期 ， 但 其 脆 弱 性 不 言 自 明 。\n",
      "INFO:tensorflow:input_ids: 101 3189 915 697 1744 1744 1079 3124 2229 6963 1041 4007 1359 3144 8024 2226 5052 3189 915 1068 5143 4680 1184 3221 1325 1380 3297 881 3198 3309 8024 852 1071 5546 2483 2595 679 6241 5632 3209 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 10 10 5 5 5 5 5 5 5 5 5 5 5 5 5 5 10 10 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:Writing example 5000 of 20864\n",
      "INFO:tensorflow:Writing example 10000 of 20864\n",
      "INFO:tensorflow:Writing example 15000 of 20864\n",
      "INFO:tensorflow:Writing example 20000 of 20864\n"
     ]
    }
   ],
   "source": [
    "train_file = os.path.join(args.output_dir, \"train.tf_record\")\n",
    "    #ok\n",
    "if not os.path.exists(train_file):\n",
    "    filed_based_convert_examples_to_features(\n",
    "        train_examples, label_list, args.max_seq_length, tokenizer, train_file, args.output_dir)\n",
    "\n",
    "# 2.读取record 数据，组成batch，把上一部输出到文件的训练数据读取\n",
    "train_input_fn = file_based_input_fn_builder(\n",
    "    input_file=train_file,\n",
    "    seq_length=args.max_seq_length,\n",
    "    is_training=True,\n",
    "    drop_remainder=True,\n",
    "    batch_size=args.batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Writing example 0 of 2318\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-0\n",
      "INFO:tensorflow:tokens: 在 这 里 恕 弟 不 恭 之 罪 ， 敢 在 尊 前 一 诤 ： 前 人 论 书 ， 每 曰 [UNK] 字 字 有 来 历 ， 笔 笔 有 出 处 [UNK] ， 细 读 公 字 ， 何 尝 跳 出 前 人 藩 篱 ， 自 隶 变 而 后 ， 直 至 明 季 ， 兄 有 何 新 出 ？\n",
      "INFO:tensorflow:input_ids: 101 1762 6821 7027 2609 2475 679 2621 722 5389 8024 3140 1762 2203 1184 671 6420 8038 1184 782 6389 741 8024 3680 3288 100 2099 2099 3300 3341 1325 8024 5011 5011 3300 1139 1905 100 8024 5301 6438 1062 2099 8024 862 2214 6663 1139 1184 782 5974 5075 8024 5632 7405 1359 5445 1400 8024 4684 5635 3209 2108 8024 1040 3300 862 3173 1139 8043 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-1\n",
      "INFO:tensorflow:tokens: 相 比 之 下 ， 青 岛 海 牛 队 和 广 州 松 日 队 的 雨 中 之 战 虽 然 也 是 0 ∶ 0 ， 但 乏 善 可 陈 。\n",
      "INFO:tensorflow:input_ids: 101 4685 3683 722 678 8024 7471 2270 3862 4281 7339 1469 2408 2336 3351 3189 7339 4638 7433 704 722 2773 6006 4197 738 3221 121 388 121 8024 852 726 1587 1377 7357 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 8 6 6 6 6 5 8 6 6 6 6 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-2\n",
      "INFO:tensorflow:tokens: 理 由 多 多 ， 最 无 奈 的 却 是 ： 5 月 恰 逢 双 重 考 试 ， 她 攻 读 的 博 士 学 位 论 文 要 通 考 ； 她 任 教 的 两 所 学 校 ， 也 要 在 这 段 时 日 大 考 。\n",
      "INFO:tensorflow:input_ids: 101 4415 4507 1914 1914 8024 3297 3187 1937 4638 1316 3221 8038 126 3299 2623 6864 1352 7028 5440 6407 8024 1961 3122 6438 4638 1300 1894 2110 855 6389 3152 6206 6858 5440 8039 1961 818 3136 4638 697 2792 2110 3413 8024 738 6206 1762 6821 3667 3198 3189 1920 5440 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-3\n",
      "INFO:tensorflow:tokens: 分 工 ， 各 有 各 的 责 任 ； 合 作 ， 正 副 经 理 之 间 ， 全 厂 的 事 ， 不 管 由 谁 分 管 ， 也 不 管 你 有 什 么 事 找 到 谁 ， 绝 不 会 把 你 推 给 第 二 个 人 。\n",
      "INFO:tensorflow:input_ids: 101 1146 2339 8024 1392 3300 1392 4638 6569 818 8039 1394 868 8024 3633 1199 5307 4415 722 7313 8024 1059 1322 4638 752 8024 679 5052 4507 6443 1146 5052 8024 738 679 5052 872 3300 784 720 752 2823 1168 6443 8024 5318 679 833 2828 872 2972 5314 5018 753 702 782 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: dev-4\n",
      "INFO:tensorflow:tokens: 胡 老 说 ， 当 画 画 疲 倦 时 就 到 院 里 去 看 看 ， 给 这 盆 花 浇 点 水 ， 给 那 棵 花 剪 剪 枝 ， 回 来 再 接 着 画 ， 画 累 了 再 出 去 ， 如 此 循 环 往 复 ， 脑 体 结 合 ， 有 益 健 康 ， 胜 过 吃 药 。\n",
      "INFO:tensorflow:input_ids: 101 5529 5439 6432 8024 2496 4514 4514 4558 958 3198 2218 1168 7368 7027 1343 4692 4692 8024 5314 6821 4658 5709 3842 4157 3717 8024 5314 6929 3484 5709 1198 1198 3363 8024 1726 3341 1086 2970 4708 4514 8024 4514 5168 749 1086 1139 1343 8024 1963 3634 2542 4384 2518 1908 8024 5554 860 5310 1394 8024 3300 4660 978 2434 8024 5526 6814 1391 5790 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 7 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:Writing example 0 of 4636\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test-0\n",
      "INFO:tensorflow:tokens: 我 们 变 而 以 书 会 友 ， 以 书 结 缘 ， 把 欧 美 、 港 台 流 行 的 食 品 类 图 谱 、 画 册 、 工 具 书 汇 集 一 堂 。\n",
      "INFO:tensorflow:input_ids: 101 2769 812 1359 5445 809 741 833 1351 8024 809 741 5310 5357 8024 2828 3616 5401 510 3949 1378 3837 6121 4638 7608 1501 5102 1745 6480 510 4514 1085 510 2339 1072 741 3726 7415 671 1828 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 10 10 5 10 10 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test-1\n",
      "INFO:tensorflow:tokens: 为 了 跟 踪 国 际 最 新 食 品 工 艺 、 流 行 趋 势 ， 大 量 搜 集 海 外 专 业 书 刊 资 料 是 提 高 技 艺 的 捷 径 。\n",
      "INFO:tensorflow:input_ids: 101 711 749 6656 6679 1744 7354 3297 3173 7608 1501 2339 5686 510 3837 6121 6633 1232 8024 1920 7030 3017 7415 3862 1912 683 689 741 1149 6598 3160 3221 2990 7770 2825 5686 4638 2949 2520 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test-2\n",
      "INFO:tensorflow:tokens: 其 中 线 装 古 籍 逾 千 册 ； 民 国 出 版 物 几 百 种 ； 珍 本 四 册 、 稀 见 本 四 百 余 册 ， 出 版 时 间 跨 越 三 百 余 年 。\n",
      "INFO:tensorflow:input_ids: 101 1071 704 5296 6163 1367 5093 6874 1283 1085 8039 3696 1744 1139 4276 4289 1126 4636 4905 8039 4397 3315 1724 1085 510 4921 6224 3315 1724 4636 865 1085 8024 1139 4276 3198 7313 6659 6632 676 4636 865 2399 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test-3\n",
      "INFO:tensorflow:tokens: 有 的 古 木 交 柯 ， 春 机 荣 欣 ， 从 诗 人 句 中 得 之 ， 而 入 画 中 ， 观 之 令 人 心 驰 。\n",
      "INFO:tensorflow:input_ids: 101 3300 4638 1367 3312 769 3392 8024 3217 3322 5783 3615 8024 794 6408 782 1368 704 2533 722 8024 5445 1057 4514 704 8024 6225 722 808 782 2552 7720 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:*** Example ***\n",
      "INFO:tensorflow:guid: test-4\n",
      "INFO:tensorflow:tokens: 不 过 重 在 晋 趣 ， 略 增 明 人 气 息 ， 妙 在 集 古 有 道 、 不 露 痕 迹 罢 了 。\n",
      "INFO:tensorflow:input_ids: 101 679 6814 7028 1762 3232 6637 8024 4526 1872 3209 782 3698 2622 8024 1975 1762 7415 1367 3300 6887 510 679 7463 4575 6839 5387 749 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "INFO:tensorflow:label_ids: 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
     ]
    }
   ],
   "source": [
    "eval_file = os.path.join(args.output_dir, \"eval.tf_record\")\n",
    "if not os.path.exists(eval_file):\n",
    "    filed_based_convert_examples_to_features(\n",
    "        eval_examples, label_list, args.max_seq_length, tokenizer, eval_file, args.output_dir)\n",
    "#构建验证集数据\n",
    "eval_input_fn = file_based_input_fn_builder(\n",
    "    input_file=eval_file,\n",
    "    seq_length=args.max_seq_length,\n",
    "    is_training=False,\n",
    "    drop_remainder=False,\n",
    "    batch_size=args.batch_size)\n",
    "#构建预测集数据\n",
    "predict_examples = processor.get_test_examples(args.data_dir)\n",
    "predict_file = os.path.join(args.output_dir, \"predict.tf_record\")\n",
    "#保存训练数据到本地\n",
    "filed_based_convert_examples_to_features(predict_examples, label_list,\n",
    "                                         args.max_seq_length, tokenizer,\n",
    "                                         predict_file, args.output_dir, mode=\"test\")\n",
    "predict_drop_remainder = False\n",
    "predict_input_fn = file_based_input_fn_builder(\n",
    "    input_file=predict_file,\n",
    "    seq_length=args.max_seq_length,\n",
    "    is_training=False,\n",
    "    drop_remainder=predict_drop_remainder,\n",
    "    batch_size=args.batch_size)\n",
    "\n",
    "train_input=train_input_fn.make_one_shot_iterator()\n",
    "eval_input=eval_input_fn.make_one_shot_iterator()\n",
    "predict_input=predict_input_fn.make_one_shot_iterator()\n",
    "# sess = tf.InteractiveSession()\n",
    "max_step=1500\n",
    "merged = tf.summary.merge_all()\n",
    "\n",
    "meta_train_data = train_input.get_next()\n",
    "meta_eval_data = eval_input.get_next() #获取验证数据集\n",
    "meta_predict_data=predict_input.get_next()\n",
    "#参数batch_size是64，train_batch_size是32，不知道train_batch_size是什么用的\n",
    "#------------------解决FailedPreconditionError:问题，初始化所有变量，不知道这样会不会影响初始化的bert预训练变量------------------\n",
    "# init_op = tf.initialize_all_variables()\n",
    "# init_global= tf.global_variables_initializer()\n",
    "config = tf.ConfigProto(log_device_placement=False,allow_soft_placement=True)\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.5\n",
    "# config.gpu_options.allow_growth = True  # 动态申请显存\n",
    "sess = tf.Session(config=config)\n",
    "sess.run(tf.local_variables_initializer())\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20864\n",
      "2318\n",
      "4636\n"
     ]
    }
   ],
   "source": [
    "print(len(train_examples))\n",
    "print(len(eval_examples))\n",
    "print(len(predict_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': array([[ 101, 1961, 6432, ...,    0,    0,    0],\n",
      "       [ 101, 6435, 3800, ...,    0,    0,    0],\n",
      "       [ 101,  679,  719, ...,    0,    0,    0],\n",
      "       ...,\n",
      "       [ 101, 5445, 5307, ...,    0,    0,    0],\n",
      "       [ 101, 1453, 2617, ...,    0,    0,    0],\n",
      "       [ 101, 4225, 2634, ...,    0,    0,    0]]), 'input_mask': array([[1, 1, 1, ..., 0, 0, 0],\n",
      "       [1, 1, 1, ..., 0, 0, 0],\n",
      "       [1, 1, 1, ..., 0, 0, 0],\n",
      "       ...,\n",
      "       [1, 1, 1, ..., 0, 0, 0],\n",
      "       [1, 1, 1, ..., 0, 0, 0],\n",
      "       [1, 1, 1, ..., 0, 0, 0]]), 'label_ids': array([[10,  2,  2, ...,  0,  0,  0],\n",
      "       [10,  2,  2, ...,  0,  0,  0],\n",
      "       [10,  2,  2, ...,  0,  0,  0],\n",
      "       ...,\n",
      "       [10,  2,  2, ...,  0,  0,  0],\n",
      "       [10,  5,  4, ...,  0,  0,  0],\n",
      "       [10,  2,  2, ...,  0,  0,  0]]), 'segment_ids': array([[0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       ...,\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0]])}\n",
      "43\n"
     ]
    }
   ],
   "source": [
    "# meta_predict_data=predict_input\n",
    "with tf.Session() as sess:\n",
    "    predict_data=sess.run(meta_predict_data)\n",
    "    print(predict_data)\n",
    "    print(len(predict_data['input_ids']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': array([[ 101, 3862, 7157, ...,    0,    0,    0],\n",
      "       [ 101, 6821, 2429, ...,    0,    0,    0],\n",
      "       [ 101,  852,  868, ...,    0,    0,    0],\n",
      "       ...,\n",
      "       [ 101, 1059, 1744, ...,    0,    0,    0],\n",
      "       [ 101, 1963, 3634, ...,    0,    0,    0],\n",
      "       [ 101, 4507,  754, ...,    0,    0,    0]]), 'input_mask': array([[1, 1, 1, ..., 0, 0, 0],\n",
      "       [1, 1, 1, ..., 0, 0, 0],\n",
      "       [1, 1, 1, ..., 0, 0, 0],\n",
      "       ...,\n",
      "       [1, 1, 1, ..., 0, 0, 0],\n",
      "       [1, 1, 1, ..., 0, 0, 0],\n",
      "       [1, 1, 1, ..., 0, 0, 0]]), 'label_ids': array([[11,  1,  1, ...,  0,  0,  0],\n",
      "       [11,  1,  1, ...,  0,  0,  0],\n",
      "       [11,  1,  1, ...,  0,  0,  0],\n",
      "       ...,\n",
      "       [11,  6,  7, ...,  0,  0,  0],\n",
      "       [11,  1,  1, ...,  0,  0,  0],\n",
      "       [11,  1,  1, ...,  0,  0,  0]]), 'segment_ids': array([[0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       ...,\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0]])}\n"
     ]
    },
    {
     "ename": "OutOfRangeError",
     "evalue": "End of sequence\n\t [[node IteratorGetNext_4 (defined at <ipython-input-14-b589965b271a>:2) ]]\n\nCaused by op 'IteratorGetNext_4', defined at:\n  File \"c:\\python36\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"c:\\python36\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"c:\\python36\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 505, in start\n    self.io_loop.start()\n  File \"c:\\python36\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 148, in start\n    self.asyncio_loop.run_forever()\n  File \"c:\\python36\\lib\\asyncio\\base_events.py\", line 421, in run_forever\n    self._run_once()\n  File \"c:\\python36\\lib\\asyncio\\base_events.py\", line 1425, in _run_once\n    handle._run()\n  File \"c:\\python36\\lib\\asyncio\\events.py\", line 127, in _run\n    self._callback(*self._args)\n  File \"c:\\python36\\lib\\site-packages\\tornado\\ioloop.py\", line 690, in <lambda>\n    lambda f: self._run_callback(functools.partial(callback, future))\n  File \"c:\\python36\\lib\\site-packages\\tornado\\ioloop.py\", line 743, in _run_callback\n    ret = callback()\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 781, in inner\n    self.run()\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 742, in run\n    yielded = self.gen.send(value)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 357, in process_one\n    yield gen.maybe_future(dispatch(*args))\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 267, in dispatch_shell\n    yield gen.maybe_future(handler(stream, idents, msg))\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in execute_request\n    user_expressions, allow_stdin,\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 294, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 536, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2848, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2874, in _run_cell\n    return runner(coro)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 67, in _pseudo_sync_runner\n    coro.send(None)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3049, in run_cell_async\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3214, in run_ast_nodes\n    if (yield from self.run_code(code, result)):\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3296, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-14-b589965b271a>\", line 2, in <module>\n    meta_eval_data = eval_input.get_next()\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\data\\ops\\iterator_ops.py\", line 414, in get_next\n    output_shapes=self._structure._flat_shapes, name=name)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\ops\\gen_dataset_ops.py\", line 1711, in iterator_get_next\n    output_shapes=output_shapes, name=name)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3300, in create_op\n    op_def=op_def)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1801, in __init__\n    self._traceback = tf_stack.extract_stack()\n\nOutOfRangeError (see above for traceback): End of sequence\n\t [[node IteratorGetNext_4 (defined at <ipython-input-14-b589965b271a>:2) ]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOutOfRangeError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[1;32mc:\\python36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1333\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1334\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1335\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1319\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1320\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1406\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1407\u001b[1;33m         run_metadata)\n\u001b[0m\u001b[0;32m   1408\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mOutOfRangeError\u001b[0m: End of sequence\n\t [[{{node IteratorGetNext_4}}]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mOutOfRangeError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-b589965b271a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m         \u001b[0meval_data\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmeta_eval_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0meval_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    927\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 929\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    930\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1150\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1152\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1153\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1326\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1328\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1329\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1330\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1346\u001b[0m           \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1347\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0merror_interpolation\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minterpolate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1348\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1349\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1350\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mOutOfRangeError\u001b[0m: End of sequence\n\t [[node IteratorGetNext_4 (defined at <ipython-input-14-b589965b271a>:2) ]]\n\nCaused by op 'IteratorGetNext_4', defined at:\n  File \"c:\\python36\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"c:\\python36\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"c:\\python36\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 505, in start\n    self.io_loop.start()\n  File \"c:\\python36\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 148, in start\n    self.asyncio_loop.run_forever()\n  File \"c:\\python36\\lib\\asyncio\\base_events.py\", line 421, in run_forever\n    self._run_once()\n  File \"c:\\python36\\lib\\asyncio\\base_events.py\", line 1425, in _run_once\n    handle._run()\n  File \"c:\\python36\\lib\\asyncio\\events.py\", line 127, in _run\n    self._callback(*self._args)\n  File \"c:\\python36\\lib\\site-packages\\tornado\\ioloop.py\", line 690, in <lambda>\n    lambda f: self._run_callback(functools.partial(callback, future))\n  File \"c:\\python36\\lib\\site-packages\\tornado\\ioloop.py\", line 743, in _run_callback\n    ret = callback()\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 781, in inner\n    self.run()\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 742, in run\n    yielded = self.gen.send(value)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 357, in process_one\n    yield gen.maybe_future(dispatch(*args))\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 267, in dispatch_shell\n    yield gen.maybe_future(handler(stream, idents, msg))\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in execute_request\n    user_expressions, allow_stdin,\n  File \"c:\\python36\\lib\\site-packages\\tornado\\gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 294, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"c:\\python36\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 536, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2848, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2874, in _run_cell\n    return runner(coro)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 67, in _pseudo_sync_runner\n    coro.send(None)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3049, in run_cell_async\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3214, in run_ast_nodes\n    if (yield from self.run_code(code, result)):\n  File \"c:\\python36\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3296, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-14-b589965b271a>\", line 2, in <module>\n    meta_eval_data = eval_input.get_next()\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\data\\ops\\iterator_ops.py\", line 414, in get_next\n    output_shapes=self._structure._flat_shapes, name=name)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\ops\\gen_dataset_ops.py\", line 1711, in iterator_get_next\n    output_shapes=output_shapes, name=name)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3300, in create_op\n    op_def=op_def)\n  File \"c:\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1801, in __init__\n    self._traceback = tf_stack.extract_stack()\n\nOutOfRangeError (see above for traceback): End of sequence\n\t [[node IteratorGetNext_4 (defined at <ipython-input-14-b589965b271a>:2) ]]\n"
     ]
    }
   ],
   "source": [
    "eval_input=eval_input_fn.make_one_shot_iterator()\n",
    "meta_eval_data = eval_input.get_next()\n",
    "with tf.Session() as sess:\n",
    "    for i in range(2):\n",
    "        eval_data=sess.run(meta_eval_data)\n",
    "        print(eval_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[11  4  5 ...  0  0  0]\n",
      " [11  1  1 ...  0  0  0]\n",
      " [11  1  1 ...  0  0  0]\n",
      " ...\n",
      " [11  6  7 ...  1  1 12]\n",
      " [11  1  1 ...  0  0  0]\n",
      " [11  1  1 ...  0  0  0]]\n"
     ]
    }
   ],
   "source": [
    "for i in range(max_step):\n",
    "\n",
    "    #把tensor转化为numpy输入\n",
    "    train_data=sess.run([meta_train_data])[0]\n",
    "    if i%10==0:\n",
    "        print(train_data['label_ids'])\n",
    "        break\n",
    "#     sess.run(train_op,feed_dict={input_ids:train_data['input_ids'],input_mask:train_data['input_mask'],\n",
    "#                                  segment_ids:train_data['segment_ids'],label_ids:train_data['label_ids']})\n",
    "#     if i%10==0:\n",
    "#         train_summary = sess.run(merged, feed_dict={input_ids:train_data['input_ids'],input_mask:train_data['input_mask'],\n",
    "#                                  segment_ids:train_data['segment_ids'],label_ids:train_data['label_ids']})\n",
    "#         train_writer.add_summary(train_summary, i)\n",
    "# #         print('accuracy %s at %s'%(accu,i))\n",
    "#         print(sess.run(tf.shape(pred_ids),feed_dict={input_ids:train_data['input_ids'],input_mask:train_data['input_mask'],\n",
    "#                                  segment_ids:train_data['segment_ids'],label_ids:train_data['label_ids']}))\n",
    "#         print(sess.run(tf.shape(train_data['label_ids']),feed_dict={input_ids:train_data['input_ids'],input_mask:train_data['input_mask'],\n",
    "#                                  segment_ids:train_data['segment_ids'],label_ids:train_data['label_ids']}))\n",
    "train_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}