I am trying DNN- RNN on a text data set. It is a simple dummy data and I think the code can be used with most of text data. However I am getting error when I am trying to load the trained model and then retrain it. Please tell me If I am doing it wrong.
def convert_docs(documents,no_class=2,MAX_DOCUMENT_LENGTH=200):
'''Takes list of docs and associated clas list as input.
Prepares it for the tflearn library. documents should be a list of strings and
clas should be a numbered list of classes encoded into 0,1,2 etc.
no_classes is the number of classes that are going to be used in the model
this is defaulted to 2'''
if MAX_DOCUMENT_LENGTH is None:
list_docs = []
for x in documents:
list_docs.append(x.split())
MAX_DOCUMENT_LENGTH = max(len(l) for l in list_docs)
print(MAX_DOCUMENT_LENGTH)
else:
MAX_DOCUMENT_LENGTH=MAX_DOCUMENT_LENGTH
vocab_processor = VocabularyProcessor(MAX_DOCUMENT_LENGTH,min_frequency=5,vocabulary=None)
data = np.array(list(vocab_processor.fit_transform(documents)))
n_words = len(vocab_processor.vocabulary_)
return data,vocab_processor, n_words, MAX_DOCUMENT_LENGTH
This is for converting a list of text docs into the format required
def model_RNN(MAX_DOCUMENT_LENGTH,n_words):
net = input_data(shape=[None, MAX_DOCUMENT_LENGTH])
net = embedding(net, input_dim=n_words, output_dim=128)
net = bidirectional_rnn(net, BasicLSTMCell(128), BasicLSTMCell(128))
net = dropout(net, 0.5)
net = fully_connected(net, 2, activation='softmax')
net = regression(net, optimizer='adam', loss='categorical_crossentropy')
model = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2)
return model
Here we initialize the RNN model
def classify_DNN(data,clas,model):
from sklearn.cross_validation import StratifiedKFold
folds = 10 #number of folds for the cv
skf = StratifiedKFold(n_folds=folds,y=clas)
fold = 1
cms = np.array([[0,0],[0,0]])
accs = []
aucs=[]
for train_index, test_index in skf:
X_train, X_test = data[train_index], data[test_index]
y_train, y_test = clas[train_index], clas[test_index]
trainy= to_categorical(y_train, nb_classes=2)
model.fit(X_train, trainy, n_epoch = 10, shuffle=True)
prediction = model.predict(X_test)
pred=np.argmax(prediction,axis=1)
acc = accuracy_score(pred, y_test)
cm = confusion_matrix(y_test,pred)
fpr, tpr, thresholds = metrics.roc_curve(y_test, pred)
print('Test Accuracy for fold {} : {}'.format(fold,round((acc*100),2)))
au = metrics.auc(fpr, tpr)
#au=roc_auc_score(testY, pred)
print('AUC for fold {} : {}'.format(fold,round((au*100),2)))
fold +=1
cms += cm
accs.append(acc)
aucs.append(au)
#print('CV test accuracy: {}\n{}'.format(round((np.mean(accs)*100),2),cms))
#print('\nCV AUC: {}'.format(round(np.mean(aucs)*100),2))
print('\nCV accuracy: %.3f +/- %.3f' % (round((np.mean(accs)*100),2),round((np.std(accs)*100),2)))
print('\nCV ROC AUC: %.3f +/- %.3f' % (round((np.mean(aucs)*100),2),round((np.std(aucs)*100),2)))
return model, round(np.mean(accs)*100,2), round(np.mean(aucs)*100,2)
This is for training the model.. I know this might not be the best way to do it but it was an experiment..
def pred_user_dnn(user_transformed, clf, y=None):
'''
Used for predicting the class of the user string given the transformed user input and the pretrained classifier
Arguments:
user_transformed= the transformed doc using the one used on the training data.. Must have same dimension as the training data
clf= classifier pre trained on the training data of the one returned from cros_val()
y= the training labels
returns:
string- Yes if the predicted label is 0
No is the predicted label is 1
'''
usr_p = clf.predict(user_transformed)
usr_p= np.argmax(usr_p,1)
print('\nUser class'+str(usr_p))
for x in usr_p:
if x==0:
print("Case recovery eligibility is: Yes")
return 'Yes'
elif x==1:
print("Case recovery eligibility is: No")
return 'No'
This function predicts the new strings
tf.reset_default_graph()
data,vocab_processor, n_words, MAX_DOCUMENT_LENGTH = convert_docs(documents,no_class=2,MAX_DOCUMENT_LENGTH=200)
model = model_RNN(MAX_DOCUMENT_LENGTH,n_words)
clf, acc, roc_auc =classify_DNN(data,clas,model)
final_name = 'LSTM'.lower()+'_'+now+'.clf'
clf.save(os.path.join(trained,final_name))
This is for saving the trained model
tf.reset_default_graph()
model_name=model_name.lower()
data,vocab_processor, n_words, MAX_DOCUMENT_LENGTH = convert_docs(documents,no_class=2,MAX_DOCUMENT_LENGTH=200)
model = model_RNN(MAX_DOCUMENT_LENGTH,n_words)
path_clf= #path where the model is saved
model.load(os.path.join(trained,path_clf))
user_transformed = np.array(list(vocab_processor.transform(clean_user_list)))
#using it for prediction
user_transformed =pad_sequences(sequences=user_transformed,maxlen=MAX_DOCUMENT_LENGTH, value=0.)
result = pred_user_dnn(user_transformed, model)
And here I am loading the saved model and i am getting this error.
model.load(os.path.join(trained,path_clf))
Traceback (most recent call last):
File "<ipython-input-28-d4cf3784bb15>", line 1, in <module>
model.load(os.path.join(trained,path_clf))
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tflearn\models\dnn.py", line 260, in load
self.trainer.restore(model_file, weights_only, **optargs)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tflearn\helpers\trainer.py", line 449, in restore
self.restorer.restore(self.session, model_file)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1388, in restore
{self.saver_def.filename_tensor_name: save_path})
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 766, in run
run_metadata_ptr)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 964, in _run
feed_dict_string, options, run_metadata)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1014, in _do_run
target_list, options, run_metadata)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1034, in _do_call
raise type(e)(node_def, op, message)
NotFoundError: Key val_loss_2 not found in checkpoint
[[Node: save_5/RestoreV2_122 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:localhost/replica:0/task:0/cpu:0"](_recv_save_5/Const_0, save_5/RestoreV2_122/tensor_names, save_5/RestoreV2_122/shape_and_slices)]]
Caused by op 'save_5/RestoreV2_122', defined at:
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 223, in <module>
main()
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 219, in main
kernel.start()
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 474, in start
ioloop.IOLoop.instance().start()
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\zmq\eventloop\ioloop.py", line 162, in start
super(ZMQIOLoop, self).start()
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tornado\ioloop.py", line 887, in start
handler_func(fd_obj, events)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tornado\stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tornado\stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 390, in execute_request
user_expressions, allow_stdin)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 501, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2717, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2821, in run_ast_nodes
if self.run_code(code, result):
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2881, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-18-395d2873044e>", line 2, in <module>
model = model_bi_LSTM(MAX_DOCUMENT_LENGTH,n_words)
File "C:\Users\kkothari\Desktop\text_mining\deep_learning.py", line 112, in model_bi_LSTM
model = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tflearn\models\dnn.py", line 63, in __init__
best_val_accuracy=best_val_accuracy)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tflearn\helpers\trainer.py", line 145, in __init__
keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1000, in __init__
self.build()
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1030, in build
restore_sequentially=self._restore_sequentially)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 624, in build
restore_sequentially, reshape)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 361, in _AddRestoreOps
tensors = self.restore_op(filename_tensor, saveable, preferred_shard)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 200, in restore_op
[spec.tensor.dtype])[0])
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py", line 441, in restore_v2
dtypes=dtypes, name=name)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 759, in apply_op
op_def=op_def)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2240, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Users\kkothari\AppData\Local\Continuum\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1128, in __init__
self._traceback = _extract_stack()
NotFoundError (see above for traceback): Key val_loss_2 not found in checkpoint
[[Node: save_5/RestoreV2_122 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:localhost/replica:0/task:0/cpu:0"](_recv_save_5/Const_0, save_5/RestoreV2_122/tensor_names, save_5/RestoreV2_122/shape_and_slices)]]
This is to create a graph and save it
graph1 = tf.Graph()
with graph1.as_default():
network = input_data(shape=[None, MAX_DOCUMENT_LENGTH])
network = tflearn.embedding(network, input_dim=n_words, output_dim=128)
branch1 = conv_1d(network, 128, 3, padding='valid', activation='relu', regularizer="L2")
branch2 = conv_1d(network, 128, 4, padding='valid', activation='relu', regularizer="L2")
branch3 = conv_1d(network, 128, 5, padding='valid', activation='relu', regularizer="L2")
network = merge([branch1, branch2, branch3], mode='concat', axis=1)
network = tf.expand_dims(network, 2)
network = global_max_pool(network)
network = dropout(network, 0.5)
network = fully_connected(network, 2, activation='softmax')
network = regression(network, optimizer='adam', learning_rate=0.001,loss='categorical_crossentropy', name='target')
model = tflearn.DNN(network, tensorboard_verbose=0)
clf, acc, roc_auc,fpr,tpr =classify_DNN(data,clas,model)
clf.save(model_path)
To reload and retrain or use it for prediction
MODEL = None
with tf.Graph().as_default():
## Building deep neural network
network = input_data(shape=[None, MAX_DOCUMENT_LENGTH])
network = tflearn.embedding(network, input_dim=n_words, output_dim=128)
branch1 = conv_1d(network, 128, 3, padding='valid', activation='relu', regularizer="L2")
branch2 = conv_1d(network, 128, 4, padding='valid', activation='relu', regularizer="L2")
branch3 = conv_1d(network, 128, 5, padding='valid', activation='relu', regularizer="L2")
network = merge([branch1, branch2, branch3], mode='concat', axis=1)
network = tf.expand_dims(network, 2)
network = global_max_pool(network)
network = dropout(network, 0.5)
network = fully_connected(network, 2, activation='softmax')
network = regression(network, optimizer='adam', learning_rate=0.001,loss='categorical_crossentropy', name='target')
new_model = tflearn.DNN(network, tensorboard_verbose=3)
new_model.load(model_path)
MODEL = new_model
Use the MODEL for prediction or retraining. The 1st line and the with loop was important. For anyone who might need help
来源：https://stackoverflow.com/questions/41616292/how-to-load-and-retrain-tflean-model