examples/question_answer/bidaf_squad_concise.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "\n",
    "import mindspore\n",
    "import mindspore.nn as nn\n",
    "from mindspore import ops\n",
    "from mindspore import Parameter, Tensor\n",
    "from mindspore.common.initializer import Uniform, HeUniform, initializer\n",
    "from mindspore.common import dtype as mstype\n",
    "\n",
    "from mindnlp import load_dataset, process, Vocab\n",
    "from mindnlp._legacy.abc import Seq2vecModel\n",
    "from mindnlp.engine import Trainer\n",
    "from mindnlp.metrics import Accuracy\n",
    "from mindnlp.modules import Glove, StaticLSTM\n",
    "from mindnlp.transforms import BasicTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['id', 'context', 'question', 'answers', 'answer_start']\n"
     ]
    }
   ],
   "source": [
    "# load datasets\n",
    "squad_train, squad_dev = load_dataset('squad1')\n",
    "print(squad_train.get_col_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load vocab and embedding\n",
    "char_dic = {\"<unk>\": 0, \"<pad>\": 1, \"e\": 2, \"t\": 3, \"a\": 4, \"i\": 5, \"n\": 6,\\\n",
    "                    \"o\": 7, \"s\": 8, \"r\": 9, \"h\": 10, \"l\": 11, \"d\": 12, \"c\": 13, \"u\": 14,\\\n",
    "                    \"m\": 15, \"f\": 16, \"p\": 17, \"g\": 18, \"w\": 19, \"y\": 20, \"b\": 21, \",\": 22,\\\n",
    "                    \"v\": 23, \".\": 24, \"k\": 25, \"1\": 26, \"0\": 27, \"x\": 28, \"2\": 29, \"\\\"\": 30, \\\n",
    "                    \"-\": 31, \"j\": 32, \"9\": 33, \"'\": 34, \")\": 35, \"(\": 36, \"?\": 37, \"z\": 38,\\\n",
    "                    \"5\": 39, \"8\": 40, \"q\": 41, \"3\": 42, \"4\": 43, \"7\": 44, \"6\": 45, \";\": 46,\\\n",
    "                    \":\": 47, \"\\u2013\": 48, \"%\": 49, \"/\": 50, \"]\": 51, \"[\": 52}\n",
    "char_vocab = Vocab(char_dic)\n",
    "word_vocab = Vocab.from_pretrained(name=\"glove.6B.100d\")\n",
    "word_embeddings = Glove.from_pretrained('6B', 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=============ready to process dataset===========\n",
      "===============process over=====================\n"
     ]
    }
   ],
   "source": [
    "# process dataset\n",
    "tokenizer = BasicTokenizer(True)\n",
    "print(\"=============ready to process dataset===========\")\n",
    "squad_train = process('squad1', squad_train, char_vocab, word_vocab, tokenizer=tokenizer,\\\n",
    "                   max_context_len=768, max_question_len=64, max_char_len=48,\\\n",
    "                   batch_size=4, drop_remainder=False )\n",
    "squad_dev = process('squad1', squad_dev, char_vocab, word_vocab, tokenizer=tokenizer,\\\n",
    "                   max_context_len=768, max_question_len=64, max_char_len=48,\\\n",
    "                   batch_size=4, drop_remainder=False )\n",
    "print(\"===============process over=====================\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "# construct bidirectional attention flow model(BiDAF)\n",
    "def arange(start, stop, step, dtype):\n",
    "    return Tensor(np.arange(start, stop, step), dtype)\n",
    "\n",
    "def sequence_mask(lengths, maxlen):\n",
    "    \"\"\"generate mask matrix by seq_length\"\"\"\n",
    "    range_vector = arange(0, maxlen, 1, lengths.dtype)\n",
    "    result = range_vector < lengths.view(lengths.shape + (1,))\n",
    "    result = result.transpose((1, 0))\n",
    "    return result.astype(lengths.dtype)\n",
    "\n",
    "def select_by_mask(inputs, mask):\n",
    "    \"\"\"mask hiddens by mask matrix\"\"\"\n",
    "    return mask.view(mask.shape + (1,)).swapaxes(0, 1) \\\n",
    "        .expand_as(inputs).astype(mstype.bool_)  * inputs\n",
    "\n",
    "def get_hidden(output, seq_length):\n",
    "    \"\"\"get hidden state by seq_length\"\"\"\n",
    "    batch_index = arange(0, seq_length.shape[0], 1, seq_length.dtype)\n",
    "    indices = ops.concat((seq_length.view(-1, 1) - 1, batch_index.view(-1, 1)), 1)\n",
    "    return ops.gather_nd(output, indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build encoder\n",
    "class Encoder(nn.Module):\n",
    "    \"\"\"\n",
    "    Encoder for BiDAF model\n",
    "    \"\"\"\n",
    "    def __init__(self, char_vocab_size, char_vocab, char_dim, char_channel_size, char_channel_width, word_vocab,\n",
    "                  word_embeddings, hidden_size, dropout):\n",
    "        super().__init__()\n",
    "        self.char_vocab = char_vocab\n",
    "        self.char_dim = char_dim\n",
    "        self.char_channel_width = char_channel_width\n",
    "        self.char_channel_size = char_channel_size\n",
    "        self.word_vocab = word_vocab\n",
    "        self.hidden_size = hidden_size\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        self.init_embed = initializer(Uniform(0.001), [char_vocab_size, char_dim])\n",
    "        self.embed = Parameter(self.init_embed, name='embed')\n",
    "\n",
    "        # 1. Character Embedding Layer\n",
    "        self.char_emb = Glove(init_embed=self.embed, dropout=0.0)\n",
    "        self.char_conv = nn.Sequential(\n",
    "            nn.Conv2d(1, char_channel_size, (char_dim, char_channel_width), pad_mode=\"pad\",\n",
    "                      weight_init=HeUniform(math.sqrt(5)), bias_init=Uniform(1 / math.sqrt(1))),\n",
    "            nn.ReLU()\n",
    "            )\n",
    "\n",
    "        # 2. Word Embedding Layer\n",
    "        self.word_emb = word_embeddings\n",
    "\n",
    "        # highway network\n",
    "        self.highway_linear0 = nn.Dense(hidden_size * 2, hidden_size * 2,\n",
    "                                        weight_init=HeUniform(math.sqrt(5)),\n",
    "                                        bias_init=Uniform(1 / math.sqrt(hidden_size * 2)),\n",
    "                                        activation=nn.ReLU())\n",
    "        self.highway_linear1 = nn.Dense(hidden_size * 2, hidden_size * 2,\n",
    "                                        weight_init=HeUniform(math.sqrt(5)),\n",
    "                                        bias_init=Uniform(1 / math.sqrt(hidden_size * 2)),\n",
    "                                        activation=nn.ReLU())\n",
    "        self.highway_gate0 = nn.Dense(hidden_size * 2, hidden_size * 2,\n",
    "                                      weight_init=HeUniform(math.sqrt(5)),\n",
    "                                      bias_init=Uniform(1 / math.sqrt(hidden_size * 2)),\n",
    "                                      activation=nn.Sigmoid())\n",
    "        self.highway_gate1 = nn.Dense(hidden_size * 2, hidden_size * 2,\n",
    "                                      weight_init=HeUniform(math.sqrt(5)),\n",
    "                                      bias_init=Uniform(1 / math.sqrt(hidden_size * 2)),\n",
    "                                      activation=nn.Sigmoid())\n",
    "\n",
    "        # 3. Contextual Embedding Layer\n",
    "        self.context_LSTM = StaticLSTM(input_size=hidden_size * 2, hidden_size=hidden_size,\n",
    "                                    bidirectional=True, batch_first=True, dropout=dropout)\n",
    "    \n",
    "    def construct(self, c_char, q_char, c_word, q_word, c_lens, q_lens):\n",
    "        # 1. Character Embedding Layer\n",
    "        c_char = self.char_emb_layer(c_char)\n",
    "        q_char = self.char_emb_layer(q_char)\n",
    "\n",
    "        # 2. Word Embedding Layer\n",
    "        c_word = self.word_emb(c_word)\n",
    "        q_word = self.word_emb(q_word)\n",
    "\n",
    "        # Highway network\n",
    "        c = self.highway_network(c_char, c_word)\n",
    "        q = self.highway_network(q_char, q_word)\n",
    "        \n",
    "        # 3. Contextual Embedding Layer\n",
    "        c, _ = self.context_LSTM(c)\n",
    "        mask = sequence_mask(c_lens, c.shape[1])\n",
    "        c = select_by_mask(c, mask)\n",
    "\n",
    "        q, _ = self.context_LSTM(q)\n",
    "        mask = sequence_mask(q_lens, q.shape[1])\n",
    "        q = select_by_mask(q, mask)\n",
    "\n",
    "        return c, q\n",
    "\n",
    "    def char_emb_layer(self, x):\n",
    "        \"\"\"\n",
    "        param x: (batch, seq_len, word_len)\n",
    "        return: (batch, seq_len, char_channel_size)\n",
    "        \"\"\"\n",
    "        batch_size = x.shape[0]\n",
    "        # x: [batch, seq_len, word_len, char_dim]\n",
    "        x = self.dropout(self.char_emb(x))\n",
    "        # x: [batch, seq_len, char_dim, word_len]\n",
    "        x = ops.transpose(x, (0, 1, 3, 2))\n",
    "        # x: [batch * seq_len, 1, char_dim, word_len]\n",
    "        x = x.view(-1, self.char_dim, x.shape[3]).expand_dims(1)\n",
    "        # x: [batch * seq_len, char_channel_size, 1, conv_len] -> [batch * seq_len, char_channel_size, conv_len]\n",
    "        x = self.char_conv(x).squeeze(2)\n",
    "        # x: [batch * seq_len, char_channel_size]\n",
    "        x = ops.max(x, axis=2)[1]\n",
    "        # x: [batch, seq_len, char_channel_size]\n",
    "        x = x.view(batch_size, -1, self.char_channel_size)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def highway_network(self, x1, x2):\n",
    "        \"\"\"\n",
    "        param x1: (batch, seq_len, char_channel_size)\n",
    "        param x2: (batch, seq_len, word_dim)\n",
    "        return: (batch, seq_len, hidden_size * 2)\n",
    "        \"\"\"\n",
    "        # [batch, seq_len, char_channel_size + word_dim]\n",
    "        x = ops.concat((x1, x2), axis=-1)\n",
    "        h = self.highway_linear0(x)\n",
    "        g = self.highway_gate0(x)\n",
    "        x = g * h + (1 - g) * x\n",
    "        h = self.highway_linear1(x)\n",
    "        g = self.highway_gate1(x)\n",
    "        x = g * h + (1 - g) * x\n",
    "\n",
    "        # [batch, seq_len, hidden_size * 2]\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build head\n",
    "class Head(nn.Module):\n",
    "    \"\"\"\n",
    "    Head for BiDAF model\n",
    "    \"\"\"\n",
    "    def __init__(self, hidden_size, dropout):\n",
    "        super().__init__()\n",
    "        # 4. Attention Flow Layer\n",
    "        self.att_weight_c = nn.Dense(hidden_size * 2, 1,\n",
    "                                     weight_init=HeUniform(math.sqrt(5)),\n",
    "                                     bias_init=Uniform(1 / math.sqrt(hidden_size * 2)))\n",
    "        self.att_weight_q = nn.Dense(hidden_size * 2, 1,\n",
    "                                     weight_init=HeUniform(math.sqrt(5)),\n",
    "                                     bias_init=Uniform(1 / math.sqrt(hidden_size * 2)))\n",
    "        self.att_weight_cq = nn.Dense(hidden_size * 2, 1,\n",
    "                                      weight_init=HeUniform(math.sqrt(5)),\n",
    "                                      bias_init=Uniform(1 / math.sqrt(hidden_size * 2)))\n",
    "        self.softmax = nn.Softmax(axis=-1)\n",
    "        self.batch_matmul = ops.BatchMatMul()\n",
    "\n",
    "        # 5. Modeling Layer\n",
    "        self.modeling_LSTM1 = StaticLSTM(input_size=hidden_size * 8, hidden_size=hidden_size,\n",
    "                                      bidirectional=True, batch_first=True, dropout=dropout)\n",
    "        self.modeling_LSTM2 = StaticLSTM(input_size=hidden_size * 2, hidden_size=hidden_size,\n",
    "                                      bidirectional=True, batch_first=True, dropout=dropout)\n",
    "        \n",
    "        # 6. Output Layer\n",
    "        self.p1_weight_g = nn.Dense(hidden_size * 8, 1,\n",
    "                                    weight_init=HeUniform(math.sqrt(5)),\n",
    "                                    bias_init=Uniform(1 / math.sqrt(hidden_size * 8)))\n",
    "        self.p1_weight_m = nn.Dense(hidden_size * 2, 1,\n",
    "                                    weight_init=HeUniform(math.sqrt(5)),\n",
    "                                    bias_init=Uniform(1 / math.sqrt(hidden_size * 2)))\n",
    "        self.p2_weight_g = nn.Dense(hidden_size * 8, 1,\n",
    "                                    weight_init=HeUniform(math.sqrt(5)),\n",
    "                                    bias_init=Uniform(1 / math.sqrt(hidden_size * 8)))\n",
    "        self.p2_weight_m = nn.Dense(hidden_size * 2, 1,\n",
    "                                    weight_init=HeUniform(math.sqrt(5)),\n",
    "                                    bias_init=Uniform(1 / math.sqrt(hidden_size * 2)))\n",
    "\n",
    "        self.output_LSTM = StaticLSTM(input_size=hidden_size * 2, hidden_size=hidden_size,\n",
    "                                   bidirectional=True, batch_first=True, dropout=dropout)\n",
    "\n",
    "    def construct(self, c, q, c_lens):\n",
    "        # 4. Attention Flow Layer\n",
    "        g = self.att_flow_layer(c, q)  #c, q are generated from Contextual Embedding Layer in Encoder\n",
    "        \n",
    "        # 5. Modeling Layer\n",
    "        m, _ = self.modeling_LSTM1(g)\n",
    "        mask = sequence_mask(c_lens, g.shape[1])\n",
    "        m = select_by_mask(m, mask)\n",
    "\n",
    "        m, _ = self.modeling_LSTM2(m)\n",
    "        mask = sequence_mask(c_lens, m.shape[1])\n",
    "        m = select_by_mask(m, mask)\n",
    "\n",
    "        # 6. Output Layer\n",
    "        p1, p2 = self.output_layer(g, m, c_lens)\n",
    "\n",
    "        # [batch, c_len], [batch, c_len]\n",
    "        return p1, p2\n",
    "\n",
    "    def att_flow_layer(self, c, q):\n",
    "        \"\"\"\n",
    "        param c: (batch, c_len, hidden_size * 2)\n",
    "        param q: (batch, q_len, hidden_size * 2)\n",
    "        return: (batch, c_len, q_len)\n",
    "        \"\"\"\n",
    "        c_len = c.shape[1]\n",
    "        q_len = q.shape[1]\n",
    "\n",
    "        cq = []\n",
    "        for i in range(q_len):\n",
    "            # qi: [batch, 1, hidden_size * 2]\n",
    "            qi = q.gather(mindspore.Tensor(i), axis=1).expand_dims(1)\n",
    "            # ci: [batch, c_len, 1] -> [batch, c_len]\n",
    "            ci = self.att_weight_cq(c * qi).squeeze(2)\n",
    "            cq.append(ci)\n",
    "        # cq: [batch, c_len, q_len]\n",
    "        cq = ops.stack(cq, -1)\n",
    "\n",
    "        # s: [batch, c_len, q_len]\n",
    "        s = self.att_weight_c(c).broadcast_to((-1, -1, q_len)) + \\\n",
    "            self.att_weight_q(q).transpose((0, 2, 1)).broadcast_to((-1, c_len, -1)) + cq\n",
    "\n",
    "        # a: [batch, c_len, q_len]\n",
    "        a = self.softmax(s)\n",
    "        # c2q_att: [batch, c_len, hidden_size * 2]\n",
    "        c2q_att = self.batch_matmul(a, q)\n",
    "        # b: [batch, 1, c_len]\n",
    "        b = self.softmax(ops.max(s, axis=2)[1]).expand_dims(1)\n",
    "        # q2c_att: [batch, hidden_size * 2]\n",
    "        q2c_att = self.batch_matmul(b, c).squeeze(1)\n",
    "        # q2c_att: [batch, c_len, hidden_size * 2]\n",
    "        q2c_att = q2c_att.expand_dims(1).broadcast_to((-1, c_len, -1))\n",
    "\n",
    "        # x: [batch, c_len, hidden_size * 8]\n",
    "        x = ops.concat([c, c2q_att, c * c2q_att, c * q2c_att], axis=-1)\n",
    "        return x\n",
    "\n",
    "    def output_layer(self, g, m, l):\n",
    "        \"\"\"\n",
    "        param g: (batch, c_len, hidden_size * 8)\n",
    "        param m: (batch, c_len ,hidden_size * 2)\n",
    "        return: p1: (batch, c_len), p2: (batch, c_len)\n",
    "        \"\"\"\n",
    "        # p1: [batch, c_len]\n",
    "        p1 = (self.p1_weight_g(g) + self.p1_weight_m(m)).squeeze(2)\n",
    "        # m2: [batch, c_len, hidden_size * 2]\n",
    "        m2, _ = self.output_LSTM(m)\n",
    "        mask = sequence_mask(l, m.shape[1])\n",
    "        m2 = select_by_mask(m2, mask)\n",
    "        # p2: [batch, c_len]\n",
    "        p2 = (self.p2_weight_g(g) + self.p2_weight_m(m2)).squeeze(2)\n",
    "\n",
    "        return p1, p2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiDAF(Seq2vecModel):\n",
    "    def __init__(self, encoder, head):\n",
    "        super().__init__(encoder, head)\n",
    "        self.encoder = encoder\n",
    "        self.head = head\n",
    "\n",
    "    def construct(self, c_char, q_char, c_word, q_word, c_lens, q_lens):\n",
    "        c, q = self.encoder(c_char, q_char, c_word, q_word, c_lens, q_lens)\n",
    "        p1, p2 = self.head(c, q, c_lens)\n",
    "        return p1, p2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define some parameters\n",
    "char_vocab_size = len(char_vocab)\n",
    "char_dim = 8\n",
    "char_channel_width = 5\n",
    "char_channel_size = 100\n",
    "hidden_size = 100\n",
    "dropout = 0.2\n",
    "lr = 0.5\n",
    "epoch = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WARNING] ME(708482:139912655406912,MainProcess):2023-03-23-13:19:31.282.05 [mindspore/nn/layer/basic.py:162] For Dropout, this parameter `keep_prob` will be deprecated, please use `p` instead.\n",
      "[WARNING] ME(708482:139912655406912,MainProcess):2023-03-23-13:19:31.321.69 [mindspore/nn/layer/conv.py:104] Value of 'has_bias' is False, value of 'bias_init' will be ignored.\n",
      "[WARNING] ME(708482:139912655406912,MainProcess):2023-03-23-13:19:31.419.57 [/home/daiyuxin/ytt/mindnlp/mindnlp/modules/rnns.py:365] dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
      "[WARNING] ME(708482:139912655406912,MainProcess):2023-03-23-13:19:31.519.77 [/home/daiyuxin/ytt/mindnlp/mindnlp/modules/rnns.py:365] dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
      "[WARNING] ME(708482:139912655406912,MainProcess):2023-03-23-13:19:31.586.01 [/home/daiyuxin/ytt/mindnlp/mindnlp/modules/rnns.py:365] dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
      "[WARNING] ME(708482:139912655406912,MainProcess):2023-03-23-13:19:31.691.23 [/home/daiyuxin/ytt/mindnlp/mindnlp/modules/rnns.py:365] dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n"
     ]
    }
   ],
   "source": [
    "# net\n",
    "encoder = Encoder(char_vocab_size, char_vocab, char_dim, char_channel_size, char_channel_width, word_vocab,\n",
    "                  word_embeddings, hidden_size, dropout)                  \n",
    "head = Head(hidden_size, dropout)\n",
    "net = BiDAF(encoder, head)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WARNING] ME(708482:139912655406912,MainProcess):2023-03-23-13:19:33.427.21 [mindspore/nn/layer/basic.py:188] For Dropout, this parameter `keep_prob` will be deprecated, please use `p` instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BiDAF<\n",
       "  (encoder): Encoder<\n",
       "    (dropout): Dropout<keep_prob=0.8>\n",
       "    (char_emb): Glove<\n",
       "      (dropout_layer): Dropout<p=0.0>\n",
       "      >\n",
       "    (char_conv): Sequential<\n",
       "      (0): Conv2d<input_channels=1, output_channels=100, kernel_size=(8, 5), stride=(1, 1), pad_mode=pad, padding=0, dilation=(1, 1), group=1, has_bias=False, weight_init=<mindspore.common.initializer.HeUniform object at 0x7f3963efec90>, bias_init=<mindspore.common.initializer.Uniform object at 0x7f3963efe610>, format=NCHW>\n",
       "      (1): ReLU<>\n",
       "      >\n",
       "    (word_emb): Glove<\n",
       "      (dropout_layer): Dropout<p=0.0>\n",
       "      >\n",
       "    (highway_linear0): Dense<\n",
       "      input_channels=200, output_channels=200, has_bias=True, activation=ReLU<>\n",
       "      (activation): ReLU<>\n",
       "      >\n",
       "    (highway_linear1): Dense<\n",
       "      input_channels=200, output_channels=200, has_bias=True, activation=ReLU<>\n",
       "      (activation): ReLU<>\n",
       "      >\n",
       "    (highway_gate0): Dense<\n",
       "      input_channels=200, output_channels=200, has_bias=True, activation=Sigmoid<>\n",
       "      (activation): Sigmoid<>\n",
       "      >\n",
       "    (highway_gate1): Dense<\n",
       "      input_channels=200, output_channels=200, has_bias=True, activation=Sigmoid<>\n",
       "      (activation): Sigmoid<>\n",
       "      >\n",
       "    (context_LSTM): StaticLSTM<\n",
       "      (rnn): MultiLayerRNN<\n",
       "        (cell_list): CellList<\n",
       "          (0): SingleLSTMLayer_GPU<>\n",
       "          >\n",
       "        (dropout): Dropout<p=0.2>\n",
       "        >\n",
       "      >\n",
       "    >\n",
       "  (head): Head<\n",
       "    (att_weight_c): Dense<input_channels=200, output_channels=1, has_bias=True>\n",
       "    (att_weight_q): Dense<input_channels=200, output_channels=1, has_bias=True>\n",
       "    (att_weight_cq): Dense<input_channels=200, output_channels=1, has_bias=True>\n",
       "    (softmax): Softmax<>\n",
       "    (modeling_LSTM1): StaticLSTM<\n",
       "      (rnn): MultiLayerRNN<\n",
       "        (cell_list): CellList<\n",
       "          (0): SingleLSTMLayer_GPU<>\n",
       "          >\n",
       "        (dropout): Dropout<p=0.2>\n",
       "        >\n",
       "      >\n",
       "    (modeling_LSTM2): StaticLSTM<\n",
       "      (rnn): MultiLayerRNN<\n",
       "        (cell_list): CellList<\n",
       "          (0): SingleLSTMLayer_GPU<>\n",
       "          >\n",
       "        (dropout): Dropout<p=0.2>\n",
       "        >\n",
       "      >\n",
       "    (p1_weight_g): Dense<input_channels=800, output_channels=1, has_bias=True>\n",
       "    (p1_weight_m): Dense<input_channels=200, output_channels=1, has_bias=True>\n",
       "    (p2_weight_g): Dense<input_channels=800, output_channels=1, has_bias=True>\n",
       "    (p2_weight_m): Dense<input_channels=200, output_channels=1, has_bias=True>\n",
       "    (output_LSTM): StaticLSTM<\n",
       "      (rnn): MultiLayerRNN<\n",
       "        (cell_list): CellList<\n",
       "          (0): SingleLSTMLayer_GPU<>\n",
       "          >\n",
       "        (dropout): Dropout<p=0.2>\n",
       "        >\n",
       "      >\n",
       "    >\n",
       "  >"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define Loss & Optimizer\n",
    "class Loss(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "    def construct(self, logit1, logit2, s_idx, e_idx):\n",
    "        loss_fn = nn.CrossEntropyLoss()\n",
    "        loss = loss_fn(logit1, s_idx) + loss_fn(logit2, e_idx)\n",
    "        return loss\n",
    "\n",
    "loss = Loss()\n",
    "optimizer = nn.Adadelta(net.trainable_params(), learning_rate=lr)\n",
    "\n",
    "metric = Accuracy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "The train will start from the checkpoint saved in ck.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0:   0%|          | 0/331 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "SyncHostToDevice failed\n\n----------------------------------------------------\n- C++ Call Stack: (For framework developers)\n----------------------------------------------------\nmindspore/ccsrc/runtime/pynative/run_op_helper.cc:170 CopyTensorDataToDevice\n",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mRuntimeError\u001B[0m                              Traceback (most recent call last)",
      "\u001B[0;32m<ipython-input-128-bcc1709b1985>\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      5\u001B[0m \u001B[0;31m# define trainer\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      6\u001B[0m \u001B[0mtrainer\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mTrainer\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mnetwork\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mnet\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtrain_dataset\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0msquad_dev\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mepochs\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mepoch\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mloss_fn\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mloss\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moptimizer\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0moptimizer\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mcallbacks\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mckpoint_cb\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mjit\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 7\u001B[0;31m \u001B[0mtrainer\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtgt_columns\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"s_idx\"\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m\"e_idx\"\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[0;32m~/ytt/mindnlp/mindnlp/engine/trainer.py\u001B[0m in \u001B[0;36mrun\u001B[0;34m(self, tgt_columns)\u001B[0m\n\u001B[1;32m    175\u001B[0m         \u001B[0mrun_context\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mRunContext\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0margs_dict\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    176\u001B[0m         \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcallback_manager\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrain_begin\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mrun_context\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 177\u001B[0;31m         \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_run\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mrun_context\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtgt_columns\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    178\u001B[0m         \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcallback_manager\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrain_end\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mrun_context\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    179\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/ytt/mindnlp/mindnlp/engine/trainer.py\u001B[0m in \u001B[0;36m_run\u001B[0;34m(self, run_context, tgt_columns)\u001B[0m\n\u001B[1;32m    203\u001B[0m                     \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcur_step_nums\u001B[0m \u001B[0;34m+=\u001B[0m \u001B[0;36m1\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    204\u001B[0m                     \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcallback_manager\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrain_step_begin\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mrun_context\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 205\u001B[0;31m                     \u001B[0mloss\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtrain_fn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0minputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtgts\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    206\u001B[0m                     \u001B[0mloss_total\u001B[0m \u001B[0;34m+=\u001B[0m \u001B[0mloss\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    207\u001B[0m                     \u001B[0mrun_context\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mloss\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mloss_total\u001B[0m\u001B[0;34m/\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcur_step_nums\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/ytt/mindnlp/mindnlp/engine/trainer.py\u001B[0m in \u001B[0;36m_run_step\u001B[0;34m(inputs, labels)\u001B[0m\n\u001B[1;32m    102\u001B[0m         \u001B[0;32mdef\u001B[0m \u001B[0m_run_step\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0minputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mlabels\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    103\u001B[0m             \u001B[0;34m\"\"\"Core process of each step, including the forward propagation process and back propagation of data.\"\"\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 104\u001B[0;31m             \u001B[0;34m(\u001B[0m\u001B[0mloss\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0m_\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mgrads\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mgrad_fn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0minputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mlabels\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    105\u001B[0m             \u001B[0moptimizer\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mgrads\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    106\u001B[0m             \u001B[0;32mreturn\u001B[0m \u001B[0mloss\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/ops/composite/base.py\u001B[0m in \u001B[0;36mafter_grad\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    602\u001B[0m                 \u001B[0;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mget_by_list\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    603\u001B[0m                     \u001B[0;32mdef\u001B[0m \u001B[0mafter_grad\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 604\u001B[0;31m                         \u001B[0;32mreturn\u001B[0m \u001B[0mgrad_\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfn_\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mweights\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    605\u001B[0m                 \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    606\u001B[0m                     \u001B[0;32mdef\u001B[0m \u001B[0mafter_grad\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/common/api.py\u001B[0m in \u001B[0;36mwrapper\u001B[0;34m(*arg, **kwargs)\u001B[0m\n\u001B[1;32m     99\u001B[0m     \u001B[0;34m@\u001B[0m\u001B[0mwraps\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfn\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    100\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mwrapper\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0marg\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 101\u001B[0;31m         \u001B[0mresults\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mfn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0marg\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    102\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0m_convert_python_data\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mresults\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    103\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/ops/composite/base.py\u001B[0m in \u001B[0;36mafter_grad\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    579\u001B[0m             \u001B[0;34m@\u001B[0m\u001B[0m_wrap_func\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    580\u001B[0m             \u001B[0;32mdef\u001B[0m \u001B[0mafter_grad\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 581\u001B[0;31m                 \u001B[0mres\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_pynative_forward_run\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfn\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mgrad_\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mweights\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    582\u001B[0m                 \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgrad\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfn\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mgrad_\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mweights\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mgrad_position\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    583\u001B[0m                 \u001B[0mout\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/ops/composite/base.py\u001B[0m in \u001B[0;36m_pynative_forward_run\u001B[0;34m(self, fn, grad, weights, args, kwargs)\u001B[0m\n\u001B[1;32m    627\u001B[0m                 \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mset_grad_flag\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    628\u001B[0m                 \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnew_graph\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfn\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mnew_kwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 629\u001B[0;31m                 \u001B[0moutputs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mfn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mnew_kwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    630\u001B[0m                 \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mend_graph\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mfn\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mnew_kwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    631\u001B[0m                 \u001B[0;32mreturn\u001B[0m \u001B[0moutputs\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/ops/composite/base.py\u001B[0m in \u001B[0;36maux_fn\u001B[0;34m(*args)\u001B[0m\n\u001B[1;32m    543\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    544\u001B[0m         \u001B[0;32mdef\u001B[0m \u001B[0maux_fn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 545\u001B[0;31m             \u001B[0moutputs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mfn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    546\u001B[0m             \u001B[0;32mif\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0misinstance\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0moutputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtuple\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mor\u001B[0m \u001B[0mlen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0moutputs\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;34m<\u001B[0m \u001B[0;36m2\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    547\u001B[0m                 \u001B[0;32mraise\u001B[0m \u001B[0mValueError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"When has_aux is True, origin fn requires more than one outputs.\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/ytt/mindnlp/mindnlp/engine/trainer.py\u001B[0m in \u001B[0;36mforward_fn\u001B[0;34m(inputs, labels)\u001B[0m\n\u001B[1;32m     81\u001B[0m         \u001B[0;32mdef\u001B[0m \u001B[0mforward_fn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0minputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mlabels\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     82\u001B[0m             \u001B[0mlogits_list\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 83\u001B[0;31m             \u001B[0mlogits\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mnetwork\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0minputs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     84\u001B[0m             \u001B[0;32mif\u001B[0m \u001B[0misinstance\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mlogits\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtuple\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     85\u001B[0m                 \u001B[0mlogits_list\u001B[0m \u001B[0;34m+=\u001B[0m \u001B[0mlogits\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/nn/cell.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m    655\u001B[0m         \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0merr\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    656\u001B[0m             \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mclear_res\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 657\u001B[0;31m             \u001B[0;32mraise\u001B[0m \u001B[0merr\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    658\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    659\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0misinstance\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0moutput\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mParameter\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/nn/cell.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m    651\u001B[0m         \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    652\u001B[0m             \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnew_graph\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 653\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_run_construct\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    654\u001B[0m             \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mend_graph\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    655\u001B[0m         \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0merr\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/nn/cell.py\u001B[0m in \u001B[0;36m_run_construct\u001B[0;34m(self, cast_inputs, kwargs)\u001B[0m\n\u001B[1;32m    439\u001B[0m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_shard_fn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0mcast_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    440\u001B[0m         \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 441\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mconstruct\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0mcast_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    442\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_enable_forward_hook\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    443\u001B[0m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_run_forward_hook\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcast_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m<ipython-input-123-29d95b645ccc>\u001B[0m in \u001B[0;36mconstruct\u001B[0;34m(self, c_char, q_char, c_word, q_word, c_lens, q_lens)\u001B[0m\n\u001B[1;32m      6\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      7\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mconstruct\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mc_char\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq_char\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mc_word\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq_word\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mc_lens\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq_lens\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 8\u001B[0;31m         \u001B[0mc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mencoder\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mc_char\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq_char\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mc_word\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq_word\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mc_lens\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq_lens\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      9\u001B[0m         \u001B[0mp1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mp2\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mhead\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mc_lens\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     10\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0mp1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mp2\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/nn/cell.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m    655\u001B[0m         \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0merr\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    656\u001B[0m             \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mclear_res\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 657\u001B[0;31m             \u001B[0;32mraise\u001B[0m \u001B[0merr\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    658\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    659\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0misinstance\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0moutput\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mParameter\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/nn/cell.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m    651\u001B[0m         \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    652\u001B[0m             \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnew_graph\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 653\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_run_construct\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    654\u001B[0m             \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mend_graph\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    655\u001B[0m         \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0merr\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/nn/cell.py\u001B[0m in \u001B[0;36m_run_construct\u001B[0;34m(self, cast_inputs, kwargs)\u001B[0m\n\u001B[1;32m    439\u001B[0m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_shard_fn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0mcast_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    440\u001B[0m         \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 441\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mconstruct\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0mcast_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    442\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_enable_forward_hook\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    443\u001B[0m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_run_forward_hook\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcast_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m<ipython-input-121-068c5fb31527>\u001B[0m in \u001B[0;36mconstruct\u001B[0;34m(self, c_char, q_char, c_word, q_word, c_lens, q_lens)\u001B[0m\n\u001B[1;32m     52\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mconstruct\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mc_char\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq_char\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mc_word\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq_word\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mc_lens\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mq_lens\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     53\u001B[0m         \u001B[0;31m# 1. Character Embedding Layer\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 54\u001B[0;31m         \u001B[0mc_char\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchar_emb_layer\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mc_char\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     55\u001B[0m         \u001B[0mq_char\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchar_emb_layer\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mq_char\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     56\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m<ipython-input-121-068c5fb31527>\u001B[0m in \u001B[0;36mchar_emb_layer\u001B[0;34m(self, x)\u001B[0m\n\u001B[1;32m     81\u001B[0m         \u001B[0mbatch_size\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mx\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mshape\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     82\u001B[0m         \u001B[0;31m# x: [batch, seq_len, word_len, char_dim]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 83\u001B[0;31m         \u001B[0mx\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdropout\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mchar_emb\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mx\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     84\u001B[0m         \u001B[0;31m# x: [batch, seq_len, char_dim, word_len]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     85\u001B[0m         \u001B[0mx\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mops\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtranspose\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mx\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m(\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;36m1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;36m3\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;36m2\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/nn/cell.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m    655\u001B[0m         \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0merr\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    656\u001B[0m             \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mclear_res\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 657\u001B[0;31m             \u001B[0;32mraise\u001B[0m \u001B[0merr\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    658\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    659\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0misinstance\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0moutput\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mParameter\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/nn/cell.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m    651\u001B[0m         \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    652\u001B[0m             \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnew_graph\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 653\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_run_construct\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    654\u001B[0m             \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mend_graph\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    655\u001B[0m         \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0merr\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/nn/cell.py\u001B[0m in \u001B[0;36m_run_construct\u001B[0;34m(self, cast_inputs, kwargs)\u001B[0m\n\u001B[1;32m    439\u001B[0m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_shard_fn\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0mcast_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    440\u001B[0m         \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 441\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mconstruct\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0mcast_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    442\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_enable_forward_hook\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    443\u001B[0m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_run_forward_hook\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcast_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/ytt/mindnlp/mindnlp/modules/embeddings/glove_embedding.py\u001B[0m in \u001B[0;36mconstruct\u001B[0;34m(self, ids)\u001B[0m\n\u001B[1;32m    135\u001B[0m         \"\"\"\n\u001B[1;32m    136\u001B[0m         \u001B[0mout_shape\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mids\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mshape\u001B[0m \u001B[0;34m+\u001B[0m \u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_embed_dim\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 137\u001B[0;31m         \u001B[0mflat_ids\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mids\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mreshape\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m-\u001B[0m\u001B[0;36m1\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    138\u001B[0m         \u001B[0moutput_for_reshape\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mops\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgather\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0membed\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mflat_ids\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    139\u001B[0m         \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mops\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mreshape\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0moutput_for_reshape\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mout_shape\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/common/tensor.py\u001B[0m in \u001B[0;36mreshape\u001B[0;34m(self, *shape)\u001B[0m\n\u001B[1;32m   1514\u001B[0m         \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_init_check\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1515\u001B[0m         \u001B[0mnew_shape\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mvalidator\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcheck_reshape_shp\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mshape\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1516\u001B[0;31m         \u001B[0;32mreturn\u001B[0m \u001B[0mtensor_operator_registry\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mget\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m'reshape'\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mnew_shape\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1517\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1518\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mreshape_as\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mother\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/ops/primitive.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, *args)\u001B[0m\n\u001B[1;32m    315\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0mshould_elim\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    316\u001B[0m             \u001B[0;32mreturn\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 317\u001B[0;31m         \u001B[0;32mreturn\u001B[0m \u001B[0m_run_op\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mname\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    318\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    319\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0m__getstate__\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/ops/primitive.py\u001B[0m in \u001B[0;36m_run_op\u001B[0;34m(obj, op_name, args)\u001B[0m\n\u001B[1;32m    881\u001B[0m     \u001B[0;34m\"\"\"Single op execution function supported by ge in PyNative mode.\"\"\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    882\u001B[0m     \u001B[0;32mif\u001B[0m \u001B[0m_RUN_OP_ASYNC\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 883\u001B[0;31m         \u001B[0mstub\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0m_pynative_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun_op_async\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mobj\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    884\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0m_convert_stub\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mstub\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    885\u001B[0m     \u001B[0;32mreturn\u001B[0m \u001B[0m_run_op_sync\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mobj\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mop_name\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/ytt/lib/python3.7/site-packages/mindspore/common/api.py\u001B[0m in \u001B[0;36mrun_op_async\u001B[0;34m(self, prim, args)\u001B[0m\n\u001B[1;32m   1050\u001B[0m             \u001B[0mStubNode\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mresult\u001B[0m \u001B[0mof\u001B[0m \u001B[0mrun\u001B[0m \u001B[0mop\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1051\u001B[0m         \"\"\"\n\u001B[0;32m-> 1052\u001B[0;31m         \u001B[0;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_executor\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mrun_op_async\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mprim\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1053\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1054\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mnew_graph\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mobj\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;31mRuntimeError\u001B[0m: SyncHostToDevice failed\n\n----------------------------------------------------\n- C++ Call Stack: (For framework developers)\n----------------------------------------------------\nmindspore/ccsrc/runtime/pynative/run_op_helper.cc:170 CopyTensorDataToDevice\n"
     ]
    }
   ],
   "source": [
    "# save checkpoint\n",
    "from mindnlp.engine.callbacks import CheckpointCallback\n",
    "ckpoint_cb = CheckpointCallback(save_path=\"ck\", epochs=1, keep_checkpoint_max=5)\n",
    "\n",
    "# define trainer\n",
    "trainer = Trainer(network=net, train_dataset=squad_train, epochs=epoch, loss_fn=loss, optimizer=optimizer, callbacks=[ckpoint_cb], jit=False)\n",
    "trainer.run(tgt_columns=[\"s_idx\", \"e_idx\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ytt",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}