Tensorflow的LSTM输入 [英] LSTM inputs for Tensorflow
问题描述
我正在尝试在Tensorflow中创建一个LSTM网络，但我对术语/基础知识一无所知.我有 n 个时间序列示例，所以 X = x n ，其中 x i = [[x 1 1 x 1 2 ,x13],...,[xm1xm 2 ，x m 3 ]]，其中x i i 是一个浮点数.首先，我想训练一个给定序列开始([x 1 1 x 1 2 ，x 1 3 ])，我可以预测序列的其余部分.然后，我希望以后包含一个分类器，以预测每个 x i 属于哪个二进制类.
所以我的问题是，我该如何引入模型的开始并退出模型的末尾?到目前为止，我有类似下面的内容
class ETLSTM(object):
"""docstring for ETLSTM"""
def __init__(self, isTraining, config):
super(ETLSTM, self).__init__()
# This needs to be tidied
self.batchSize = batchSize = config.batchSize
self.numSteps = numSteps = config.numSteps
self.numInputs = numInputs = config.numInputs
self.numLayers = numLayers = config.numLayers
lstmSize = config.lstm_size
DORate = config.keep_prob
self.input_data = tf.placeholder(tf.float32, [batchSize, numSteps,
numInputs])
self.targets = tf.placeholder(tf.float32, [batchSize, numSteps,
numInputs])
lstmCell = rnn_cell.BasicLSTMCell(lstmSize, forgetbias=0.0)
if(isTraining and DORate < 1):
lstmCell = tf.nn.rnn_cell.DropoutWrapper(lstmCell,
output_keep_prob=DORate)
cell = tf.nn.rnn_cell.MultiRNNCell([lstmCell]*numLayers)
self._initial_state = cell.zero_state(batchSize, tf.float32)
# This won't work with my data, need to find what goes in...
with tf.device("/cpu:0"):
embedding = tf.get_variable("embedding", [vocab_size, size])
inputs = tf.nn.embedding_lookup(embedding, self._input_data)
if(isTraining and DORate < 1):
inputs = tf.nn.dropout(inputs, DORate)
编辑:
具体来说，如何完成__init__函数，使其与我的数据兼容?
到目前为止，RNN会根据给定的1到N值来预测N + 1的值. (LSTM只是实现RNN单元的一种方法.)
简短的答案是:
- 使用反向传播对完整序列[[x 1 1 x 1 2 "进行训练， x 1 3 ]，...，[x m 1 x m 2 ，x m 3 ]]
- 在序列[x 1 1 x 1 2 "的开始处向前运行您训练有素的模型， x 1 3 ，...]，然后从模型中采样以预测序列的其余部分[x m 1 x m 2 ，x m 3 ，...].
更长的答案是:
您的示例仅显示了模型的初始化.您还需要实现一个训练函数来反向传播，以及一个用于预测结果的样本函数.
以下代码段是混合& ;;匹配，仅用于说明目的...
要进行培训，只需在数据迭代器中输入完整的序列，并在其中添加start + rest.
例如，在示例代码tensorflow/models/rnn/ptb_word_lm.py中，训练循环针对目标(输入数据以一个时间步移位)为输入数据批次计算成本函数
# compute a learning rate decay
session.run(tf.assign(self.learning_rate_variable, learning_rate))
logger.info("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(self.learning_rate_variable)))
"""Runs the model on the given data."""
epoch_size = ((len(training_data) // self.batch_size) - 1) // self.num_steps
costs = 0.0
iters = 0
state = self.initial_state.eval()
for step, (x, y) in enumerate(self.data_iterator(training_data, self.batch_size, self.num_steps)):
# x and y should have shape [batch_size, num_steps]
cost, state, _ = session.run([self.cost_function, self.final_state, self.train_op],
{self.input_data: x,
self.targets: y,
self.initial_state: state})
costs += cost
iters += self.num_steps
请注意tensorflow/models/rnn/reader.py中的数据迭代器将输入数据作为'x'返回，并将目标数据作为'y'从x向前移了一步. (您将需要创建一个像这样的数据迭代器来打包您的训练序列集.)
def ptb_iterator(raw_data, batch_size, num_steps):
raw_data = np.array(raw_data, dtype=np.int32)
data_len = len(raw_data)
batch_len = data_len // batch_size
data = np.zeros([batch_size, batch_len], dtype=np.int32)
for i in range(batch_size):
data[i] = raw_data[batch_len * i:batch_len * (i + 1)]
epoch_size = (batch_len - 1) // num_steps
if epoch_size == 0:
raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
for i in range(epoch_size):
x = data[:, i*num_steps:(i+1)*num_steps]
y = data[:, i*num_steps+1:(i+1)*num_steps+1]
yield (x, y)
经过训练后，您可以通过输入序列的开头开始运行模型以对序列进行预测start_x = [X1，X2，X3，...] ...此代码段假定二进制值代表类，您可以d必须调整浮点值的采样功能.
def sample(self, sess, num=25, start_x):
# return state tensor with batch size 1 set to zeros, eval
state = self.rnn_layers.zero_state(1, tf.float32).eval()
# run model forward through the start of the sequence
for char in start_x:
# create a 1,1 tensor/scalar set to zero
x = np.zeros((1, 1))
# set to the vocab index
x[0, 0] = char
# fetch: final_state
# input_data = x, initial_state = state
[state] = sess.run([self.final_state], {self.input_data: x, self.initial_state:state})
def weighted_pick(weights):
# an array of cummulative sum of weights
t = np.cumsum(weights)
# scalar sum of tensor
s = np.sum(weights)
# randomly selects a value from the probability distribution
return(int(np.searchsorted(t, np.random.rand(1)*s)))
# PREDICT REST OF SEQUENCE
rest_x = []
# get last character in init
char = start_x[-1]
# sample next num chars in the sequence after init
score = 0.0
for n in xrange(num):
# init input to zeros
x = np.zeros((1, 1))
# lookup character index
x[0, 0] = char
# probs = tf.nn.softmax(self.logits)
# fetch: probs, final_state
# input_data = x, initial_state = state
[probs, state] = sess.run([self.output_layer, self.final_state], {self.input_data: x, self.initial_state:state})
p = probs[0]
logger.info("output=%s" % np.shape(p))
# sample = int(np.random.choice(len(p), p=p))
# select a random value from the probability distribution
sample = weighted_pick(p)
score += p[sample]
# look up the key with the index
logger.debug("sample[%d]=%d" % (n, sample))
pred = self.vocabulary[sample]
logger.debug("pred=%s" % pred)
# add the car to the output
rest_x.append(pred)
# set the next input character
char = pred
return rest_x, score
I'm trying to create an LSTM network in Tensorflow and I'm lost in terminology/basics. I have n time series examples so X=xn, where xi=[[x11x12,x13],...,[xm1xm2,xm3]] and where xii is a float. First of all I want to train a model that given the start of a sequence ([x11x12,x13]) I can predict the rest of the sequence. Then later I hope to include a classifier to predict which binary class each xi belongs to.
So my problem is what do I feed in to the start and pull out the end of my model? So far I have something that looks like the below
class ETLSTM(object):
"""docstring for ETLSTM"""
def __init__(self, isTraining, config):
super(ETLSTM, self).__init__()
# This needs to be tidied
self.batchSize = batchSize = config.batchSize
self.numSteps = numSteps = config.numSteps
self.numInputs = numInputs = config.numInputs
self.numLayers = numLayers = config.numLayers
lstmSize = config.lstm_size
DORate = config.keep_prob
self.input_data = tf.placeholder(tf.float32, [batchSize, numSteps,
numInputs])
self.targets = tf.placeholder(tf.float32, [batchSize, numSteps,
numInputs])
lstmCell = rnn_cell.BasicLSTMCell(lstmSize, forgetbias=0.0)
if(isTraining and DORate < 1):
lstmCell = tf.nn.rnn_cell.DropoutWrapper(lstmCell,
output_keep_prob=DORate)
cell = tf.nn.rnn_cell.MultiRNNCell([lstmCell]*numLayers)
self._initial_state = cell.zero_state(batchSize, tf.float32)
# This won't work with my data, need to find what goes in...
with tf.device("/cpu:0"):
embedding = tf.get_variable("embedding", [vocab_size, size])
inputs = tf.nn.embedding_lookup(embedding, self._input_data)
if(isTraining and DORate < 1):
inputs = tf.nn.dropout(inputs, DORate)
EDIT:
Specifically, how to I finish the __init__ function so that it is compatible with my data?
An RNN predicts the value of N+1 given the values from 1 to N so far. (LSTM is just one way to implement an RNN cell.)
The short answer is:
- train your model using back propagation on your complete sequences [[x11x12,x13],...,[xm1xm2,xm3]]
- run your trained model forward on the start of your sequence [x11x12,x13,...] then sample from the model to predict the rest of your sequence [xm1xm2,xm3,...].
The longer answer is:
Your example just shows the initialization of the model. You also need to implement a training function to run back propagation as well as a sample function that predicts the results.
The following code snippets are mix & match and are for illustration purposes only...
For training just feed in your complete sequences with start + rest in your data iterator.
For example in the sample code tensorflow/models/rnn/ptb_word_lm.py the training loop computes a cost function for batches of input_data against targets (which are the input_data shifted by one timestep)
# compute a learning rate decay
session.run(tf.assign(self.learning_rate_variable, learning_rate))
logger.info("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(self.learning_rate_variable)))
"""Runs the model on the given data."""
epoch_size = ((len(training_data) // self.batch_size) - 1) // self.num_steps
costs = 0.0
iters = 0
state = self.initial_state.eval()
for step, (x, y) in enumerate(self.data_iterator(training_data, self.batch_size, self.num_steps)):
# x and y should have shape [batch_size, num_steps]
cost, state, _ = session.run([self.cost_function, self.final_state, self.train_op],
{self.input_data: x,
self.targets: y,
self.initial_state: state})
costs += cost
iters += self.num_steps
Note the data iterator in tensorflow/models/rnn/reader.py returns the input data as 'x' and the targets as 'y' which are just shifted one step forward from x. (You would need to create a data iterator like this that packages your set of training sequences.)
def ptb_iterator(raw_data, batch_size, num_steps):
raw_data = np.array(raw_data, dtype=np.int32)
data_len = len(raw_data)
batch_len = data_len // batch_size
data = np.zeros([batch_size, batch_len], dtype=np.int32)
for i in range(batch_size):
data[i] = raw_data[batch_len * i:batch_len * (i + 1)]
epoch_size = (batch_len - 1) // num_steps
if epoch_size == 0:
raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
for i in range(epoch_size):
x = data[:, i*num_steps:(i+1)*num_steps]
y = data[:, i*num_steps+1:(i+1)*num_steps+1]
yield (x, y)
After training, you run the model forward to make predictions for sequences by feeding in the start of your sequence start_x=[X1, X2, X3,...]...this snippets assumes binary values representing classes, you'd have to adjust the sampling function for float values.
def sample(self, sess, num=25, start_x):
# return state tensor with batch size 1 set to zeros, eval
state = self.rnn_layers.zero_state(1, tf.float32).eval()
# run model forward through the start of the sequence
for char in start_x:
# create a 1,1 tensor/scalar set to zero
x = np.zeros((1, 1))
# set to the vocab index
x[0, 0] = char
# fetch: final_state
# input_data = x, initial_state = state
[state] = sess.run([self.final_state], {self.input_data: x, self.initial_state:state})
def weighted_pick(weights):
# an array of cummulative sum of weights
t = np.cumsum(weights)
# scalar sum of tensor
s = np.sum(weights)
# randomly selects a value from the probability distribution
return(int(np.searchsorted(t, np.random.rand(1)*s)))
# PREDICT REST OF SEQUENCE
rest_x = []
# get last character in init
char = start_x[-1]
# sample next num chars in the sequence after init
score = 0.0
for n in xrange(num):
# init input to zeros
x = np.zeros((1, 1))
# lookup character index
x[0, 0] = char
# probs = tf.nn.softmax(self.logits)
# fetch: probs, final_state
# input_data = x, initial_state = state
[probs, state] = sess.run([self.output_layer, self.final_state], {self.input_data: x, self.initial_state:state})
p = probs[0]
logger.info("output=%s" % np.shape(p))
# sample = int(np.random.choice(len(p), p=p))
# select a random value from the probability distribution
sample = weighted_pick(p)
score += p[sample]
# look up the key with the index
logger.debug("sample[%d]=%d" % (n, sample))
pred = self.vocabulary[sample]
logger.debug("pred=%s" % pred)
# add the car to the output
rest_x.append(pred)
# set the next input character
char = pred
return rest_x, score
这篇关于Tensorflow的LSTM输入的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！
