哪個網址得實作.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "Keras requires TensorFlow 2.2 or higher. Install TensorFlow via `pip install tensorflow`",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32mC:\\Users\\User\\Anaconda3\\envs\\tf2.0\\lib\\site-packages\\keras\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m     \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperimental\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mRandomRotation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mImportError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow.keras.layers.experimental.preprocessing'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-187752bc6e5a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mKFold\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mkeras_bert\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mload_trained_model_from_checkpoint\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mTokenizer\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtop_k_categorical_accuracy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\User\\Anaconda3\\envs\\tf2.0\\lib\\site-packages\\keras_bert\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mbert\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mloader\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mtokenizer\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mTokenizer\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0moptimizers\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mutil\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\User\\Anaconda3\\envs\\tf2.0\\lib\\site-packages\\keras_bert\\bert.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mkeras_pos_embd\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mPositionEmbedding\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mkeras_layer_normalization\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mLayerNormalization\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mkeras_transformer\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mget_encoders\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgelu\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mkeras_transformer\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mget_custom_objects\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mget_encoder_custom_objects\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\User\\Anaconda3\\envs\\tf2.0\\lib\\site-packages\\keras_pos_embd\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mpos_embd\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mPositionEmbedding\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mtrig_pos_embd\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mTrigPosEmbedding\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\User\\Anaconda3\\envs\\tf2.0\\lib\\site-packages\\keras_pos_embd\\pos_embd.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mbackend\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mbackend\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mK\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mPositionEmbedding\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mLayer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\User\\Anaconda3\\envs\\tf2.0\\lib\\site-packages\\keras_pos_embd\\backend.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[1;32mimport\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[0mutils\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\User\\Anaconda3\\envs\\tf2.0\\lib\\site-packages\\keras\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mImportError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     raise ImportError(\n\u001b[1;32m----> 6\u001b[1;33m         \u001b[1;34m'Keras requires TensorFlow 2.2 or higher. '\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m         'Install TensorFlow via `pip install tensorflow`')\n\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mImportError\u001b[0m: Keras requires TensorFlow 2.2 or higher. Install TensorFlow via `pip install tensorflow`"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import codecs, gc\n",
    "import numpy as np\n",
    "from sklearn.model_selection import KFold\n",
    "from keras_bert import load_trained_model_from_checkpoint, Tokenizer\n",
    "from keras.metrics import top_k_categorical_accuracy\n",
    "from keras.layers import *\n",
    "from keras.callbacks import *\n",
    "from keras.models import Model\n",
    "import keras.backend as K\n",
    "from keras.optimizers import Adam\n",
    "from keras.utils import to_categorical "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-de529eaa976c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#读取训练集和测试集\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mdata_path\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'./data/'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mdf_train\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata_path\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"train_x_c.txt\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mdf_test\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata_path\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"test_x_c.txt\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mtrain_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"sentence\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"label\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集\n",
    "data_path = './data/'\n",
    "df_train = pd.read_csv(data_path + \"train_x_c.txt\")\n",
    "df_test = pd.read_csv(data_path + \"test_x_c.txt\")\n",
    "train_df = pd.DataFrame(df_train, columns=[\"sentence\", \"label\"]).astype(str)\n",
    "test_df = pd.DataFrame(df_test, columns=[\"sentence\", \"lebel\"]).astype(str)\n",
    "maxlen = 100  #设置序列长度为120，要保证序列长度不超过512\n",
    "#预训练好的模型\n",
    "config_path = 'chinese_roberta_wwm_large_ext_L-24_H-1024_A-16/bert_config.json'\n",
    "checkpoint_path = 'chinese_roberta_wwm_large_ext_L-24_H-1024_A-16/bert_model.ckpt'\n",
    "dict_path = 'chinese_roberta_wwm_large_ext_L-24_H-1024_A-16/vocab.txt'\n",
    " \n",
    "#将词表中的词编号转换为字典\n",
    "token_dict = {}\n",
    "with codecs.open(dict_path, 'r', 'utf8') as reader:\n",
    "    for line in reader:\n",
    "        token = line.strip()\n",
    "        token_dict[token] = len(token_dict)\n",
    " \n",
    "#重写tokenizer        \n",
    "class OurTokenizer(Tokenizer):\n",
    "    def _tokenize(self, text):\n",
    "        R = []\n",
    "        for c in text:\n",
    "            if c in self._token_dict:\n",
    "                R.append(c)\n",
    "            elif self._is_space(c):\n",
    "                R.append('[unused1]')  # 用[unused1]来表示空格类字符\n",
    "            else:\n",
    "                R.append('[UNK]')  # 不在列表的字符用[UNK]表示\n",
    "        return R\n",
    "tokenizer = OurTokenizer(token_dict)\n",
    " \n",
    "#让每条文本的长度相同，用0填充\n",
    "def seq_padding(X, padding=0):\n",
    "    L = [len(x) for x in X]\n",
    "    ML = max(L)\n",
    "    return np.array([\n",
    "        np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X\n",
    "    ])\n",
    " \n",
    "#data_generator只是一种为了节约内存的数据方式\n",
    "class data_generator:\n",
    "    def __init__(self, data, batch_size=32, shuffle=True):\n",
    "        self.data = data\n",
    "        self.batch_size = batch_size\n",
    "        self.shuffle = shuffle\n",
    "        self.steps = len(self.data) // self.batch_size\n",
    "        if len(self.data) % self.batch_size != 0:\n",
    "            self.steps += 1\n",
    " \n",
    "    def __len__(self):\n",
    "        return self.steps\n",
    " \n",
    "    def __iter__(self):\n",
    "        while True:\n",
    "            idxs = list(range(len(self.data)))\n",
    " \n",
    "            if self.shuffle:\n",
    "                np.random.shuffle(idxs)\n",
    " \n",
    "            X1, X2, Y = [], [], []\n",
    "            for i in idxs:\n",
    "                d = self.data[i]\n",
    "                text = d[0][:maxlen]\n",
    "                x1, x2 = tokenizer.encode(first=text)\n",
    "                y = d[1]\n",
    "                X1.append(x1)\n",
    "                X2.append(x2)\n",
    "                Y.append([y])\n",
    "                if len(X1) == self.batch_size or i == idxs[-1]:\n",
    "                    X1 = seq_padding(X1)\n",
    "                    X2 = seq_padding(X2)\n",
    "                    Y = seq_padding(Y)\n",
    "                    yield [X1, X2], Y[:, 0, :]\n",
    "                    [X1, X2, Y] = [], [], []\n",
    " \n",
    "#计算top-k正确率,当预测值的前k个值中存在目标类别即认为预测正确                 \n",
    "#def acc_top2(y_true, y_pred):\n",
    "#    return top_k_categorical_accuracy(y_true, y_pred, k=2)\n",
    " \n",
    "#bert模型设置\n",
    "def build_bert(nclass):\n",
    "    bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None)  #加载预训练模型\n",
    " \n",
    "    for l in bert_model.layers:\n",
    "        l.trainable = True\n",
    " \n",
    "    x1_in = Input(shape=(None,))\n",
    "    x2_in = Input(shape=(None,))\n",
    " \n",
    "    x = bert_model([x1_in, x2_in])\n",
    "    x = Lambda(lambda x: x[:, 0])(x) # 取出[CLS]对应的向量用来做分类\n",
    "    p = Dense(nclass, activation='softmax')(x)\n",
    " \n",
    "    model = Model([x1_in, x2_in], p)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer=Adam(1e-5),    #用足够小的学习率\n",
    "                  metrics=['accuracy'])\n",
    "    print(model.summary())\n",
    "    return model\n",
    " \n",
    "#训练数据、测试数据和标签转化为模型输入格式\n",
    "DATA_LIST = []\n",
    "for data_row in train_df.iloc[:].itertuples():\n",
    "    DATA_LIST.append((data_row.contents, to_categorical(data_row.labels, 3)))\n",
    "DATA_LIST = np.array(DATA_LIST)\n",
    " \n",
    "DATA_LIST_TEST = []\n",
    "for data_row in test_df.iloc[:].itertuples():\n",
    "    DATA_LIST_TEST.append((data_row.contents, to_categorical(0, 3)))\n",
    "DATA_LIST_TEST = np.array(DATA_LIST_TEST)\n",
    " \n",
    "#交叉验证训练和测试模型\n",
    "def run_cv(nfold, data, data_labels, data_test):\n",
    "    kf = KFold(n_splits=nfold, shuffle=True, random_state=520).split(data)\n",
    "    train_model_pred = np.zeros((len(data), 3))\n",
    "    test_model_pred = np.zeros((len(data_test), 3))\n",
    " \n",
    "    for i, (train_fold, test_fold) in enumerate(kf):\n",
    "        X_train, X_valid, = data[train_fold, :], data[test_fold, :]\n",
    " \n",
    "        model = build_bert(3)\n",
    "        early_stopping = EarlyStopping(monitor='val_acc', patience=3)   #早停法，防止过拟合\n",
    "        plateau = ReduceLROnPlateau(monitor=\"val_acc\", verbose=1, mode='max', factor=0.5, patience=2) #当评价指标不在提升时，减少学习率\n",
    "        checkpoint = ModelCheckpoint('./bert_dump/' + str(i) + '.hdf5', monitor='val_acc',verbose=2, save_best_only=True, mode='max', save_weights_only=True) #保存最好的模型\n",
    " \n",
    "        train_D = data_generator(X_train, shuffle=True)\n",
    "        valid_D = data_generator(X_valid, shuffle=True)\n",
    "        test_D = data_generator(data_test, shuffle=False)\n",
    "        #模型训练\n",
    "        model.fit_generator(\n",
    "            train_D.__iter__(),\n",
    "            steps_per_epoch=len(train_D),\n",
    "            epochs=5,\n",
    "            validation_data=valid_D.__iter__(),\n",
    "            validation_steps=len(valid_D),\n",
    "            callbacks=[early_stopping, plateau, checkpoint],\n",
    "        )\n",
    " \n",
    "        # model.load_weights('./bert_dump/' + str(i) + '.hdf5')\n",
    " \n",
    "        # return model\n",
    "        train_model_pred[test_fold, :] = model.predict_generator(valid_D.__iter__(), steps=len(valid_D), verbose=1)\n",
    "        test_model_pred += model.predict_generator(test_D.__iter__(), steps=len(test_D), verbose=1)\n",
    " \n",
    "        del model\n",
    "        gc.collect()   #清理内存\n",
    "        K.clear_session()   #clear_session就是清除一个session\n",
    "        # break\n",
    " \n",
    "    return train_model_pred, test_model_pred\n",
    " \n",
    "#n折交叉验证\n",
    "train_model_pred, test_model_pred = run_cv(2, DATA_LIST, None, DATA_LIST_TEST)\n",
    " \n",
    "test_pred = [np.argmax(x) for x in test_model_pred]\n",
    " \n",
    "#将测试集预测结果写入文件\n",
    "output=pd.DataFrame({'id':test_df.id,'sentiment':test_pred})\n",
    "output.to_csv('data/results.csv', index=None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Tensorflow2.0",
   "language": "python",
   "name": "tf2.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}