publishing a new model for clf

MANIFEST.in

@@ -1,4 +1,4 @@
-include tableqa/Question_Classifier.h5
+include tableqa/Question_Classifier_Bert.h5
 prune tableqa/cleaned_data
 prune tableqa/schema
 prune tableqa/data

setup.py

@@ -3,7 +3,7 @@
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-install_requires = ['responder', 'graphql-core==2.3', 'graphene==2.1.8', 'transformers[tf-cpu]==3.0.2', 'rake_nltk', 'nltk','sqlalchemy','sentence_transformers==0.3.0']
+install_requires = ['responder', 'graphql-core==2.3', 'graphene==2.1.8', 'transformers[tf-cpu]==3.0.2', 'rake_nltk', 'nltk','sqlalchemy']
 
 setuptools.setup(
     name="tableqa", # Replace with your own username

tableqa/clauses.py

@@ -1,17 +1,28 @@
-
+import tensorflow as tf
 from tensorflow.keras.models import load_model
-from sentence_transformers import SentenceTransformer
+from transformers import DistilBertTokenizer, TFDistilBertModel
 from numpy import asarray
 import os
 
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+bert_model = TFDistilBertModel.from_pretrained('distilbert-base-uncased')
+
 class Clause:
     def __init__(self):
-        self.bert_model = SentenceTransformer('bert-base-nli-mean-tokens')
-        self.model=load_model(os.path.join(os.path.abspath(os.path.dirname(__file__)),"Question_Classifier.h5"))
+        self.model=load_model(os.path.join(os.path.abspath(os.path.dirname(__file__)),"Question_Classifier_Bert.h5"))
         self.types={0:'SELECT {} FROM {}', 1:'SELECT MAX({}) FROM {}', 2:'SELECT MIN({}) FROM {}', 3:'SELECT COUNT({}) FROM {}', 4:'SELECT SUM({}) FROM {}', 5:'SELECT AVG({}) FROM {}'}
+
+    def get_bert_embeddings(self,x):
+        emb=[]
+        for i,sentence in enumerate(x):
+            input_ids = tf.constant(tokenizer.encode(sentence))[None, :]  # Batch size 1
+            outputs = bert_model(input_ids)
+            last_hidden_states = outputs[0][0][1] 
+            emb.append(last_hidden_states)
+        return asarray(emb)
 
     def adapt(self,q,inttype=False,summable=False):
-        emb=asarray(self.bert_model.encode(q))
+        emb=self.get_bert_embeddings([q])
         self.clause=self.types[self.model.predict_classes(emb)[0]]
 
         if summable and inttype  and "COUNT" in self.clause:

tableqa/clf.py

@@ -4,12 +4,15 @@
 import numpy as np
 from numpy import asarray
 from nltk.tokenize import sent_tokenize
-from sentence_transformers import SentenceTransformer
 from sklearn.model_selection import train_test_split
 from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout, Lambda, Flatten
 from tensorflow.keras.models import Sequential, load_model, model_from_config
 from tensorflow.keras.layers import Dense, Flatten, LSTM, Conv1D, MaxPooling1D, Dropout, Activation
+import tensorflow as tf
+from transformers import DistilBertTokenizer, TFDistilBertModel
 
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+bert_model = TFDistilBertModel.from_pretrained('distilbert-base-uncased')
 np.random.seed(7)
 
 def get_keras_model():
@@ -23,31 +26,34 @@ def get_keras_model():
     return model
 
 data=pd.read_csv("wikidata.csv",usecols=["questions","types"])
-
 categories=data["types"]
 #print(categories.tolist().count(0))
 
 
 x_train, x_test, y_train,y_test =train_test_split(data["questions"],categories)
 
-bert_model = SentenceTransformer('bert-base-nli-mean-tokens')
-# bert_model = SentenceTransformer('average_word_embeddings_glove.6B.300d')
 
-train_embeddings = asarray(bert_model.encode(x_train.tolist()), dtype = "float32")
-test_embeddings = asarray(bert_model.encode(x_test.tolist()), dtype = "float32")
+# bert_model = SentenceTransformer('average_word_embeddings_glove.6B.300d')
+def get_bert_embeddings(x):
+    emb=[]
+    for i,sentence in enumerate(x):
+        print(i)
+        input_ids = tf.constant(tokenizer.encode(sentence))[None, :]  # Batch size 1
+        outputs = bert_model(input_ids)
+        last_hidden_states = outputs[0][0][1] 
+        emb.append(last_hidden_states)
+    return asarray(emb)
+train_embeddings =get_bert_embeddings(x_train.tolist())
+test_embeddings = get_bert_embeddings(x_test.tolist())
 y_train=asarray(y_train,dtype="float32")
 y_test=asarray(y_test,dtype="float32")
 
 model = get_keras_model()
-model.fit(train_embeddings, y_train, epochs=100,validation_data=(test_embeddings,y_test))
-
-print('\n# Evaluate on test data')
-results = model.evaluate(test_embeddings, y_test, batch_size=128)
-print('test loss, test acc:', results)
-
+print(train_embeddings.shape)
+model.fit(train_embeddings, y_train, epochs=200,validation_data=(test_embeddings,y_test))
 
+model.save("Question_Classifier_Bert.h5")
 
-model.save("Question_Classifier.h5")
 
 
 

tableqa/requirements.txt

@@ -5,4 +5,3 @@ transformers[tf-cpu]==3.0.2
 rake_nltk
 nltk
 sqlalchemy
-sentence_transformers==0.3.0