Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
drop_2 = tf.nn.dropout(img_layer_2, _dropout)
#layer 2 out merge
if _tr == 1:
tp1 = tf.split(0, batch_size, aud_layer_2)
tp2 = tf.split(0, batch_size, drop_2)
factor = calculatCA(tp1, tp2)
merge_sum = tf.nn.relu(tf.add(aud_layer_2, drop_2))
facmat = tf.nn.relu(tf.matmul(factor, merge_sum))
else:
facmat = tf.nn.relu(tf.add(aud_layer_2, drop_2))
#output layer
out_layer_1 = tf.nn.relu(tf.add(tf.matmul(facmat, _w_out['h1']), _b_out['b1'])) #Hidden layer with RELU activation
out_layer_2 = tf.nn.relu(tf.add(tf.matmul(out_layer_1, _w_out['h2']), _b_out['b2'])) #Hidden layer with RELU activation
return tf.matmul(out_layer_2, _w_out['out']) + _b_out['out']
y = tf.placeholder(tf.int64, [batch_size*(args.sequence_length-1)*(args.total_digits+1)])
rnn = LSTMBlockCell(hidden_size)
if use_act:
inputs = [tf.squeeze(xx) for xx in tf.split(x, args.sequence_length, 1)]
act = ACTCell(num_units=args.hidden_size, cell=rnn,
max_computation=20, batch_size=batch_size, state_is_tuple=True, return_ponders=args.return_ponders)
outputs, final_state = static_rnn(act, inputs, dtype=tf.float32, initial_state=act.zero_state(args.batch_size, tf.float32))
outputs = tf.stack(outputs, 1)
else:
outputs, final_state = tf.nn.dynamic_rnn(rnn, x, dtype=tf.float32)
output = tf.reshape(outputs[:, 1:, :], [-1, hidden_size])
softmax_w = tf.get_variable("softmax_w", [hidden_size, output_size])
softmax_b = tf.get_variable("softmax_b", [output_size])
logits = tf.reshape(tf.matmul(output, softmax_w) + softmax_b, [-1, 11])
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
loss = tf.reduce_mean(tf.reshape(loss, [batch_size, -1]), axis=1)
if use_act:
if args.return_ponders:
ponder, ponders_tensor = act.calculate_ponder_cost()
ponders_tensor = tf.reduce_mean(ponders_tensor, axis=0)
else:
ponder = act.calculate_ponder_cost()
ponder_mean = tf.reduce_mean(ponder)
tf.summary.scalar('Ponder', ponder_mean)
loss += args.tau*ponder
loss = tf.reduce_mean(loss)
def gram_matrix(input_tensor):
# if input tensor is a 3D array of size Nh x Nw X Nc
# we reshape it to a 2D array of Nc x (Nh*Nw)
channels = int(input_tensor.shape[-1])
a = tf.reshape(input_tensor, [-1, channels])
n = tf.shape(a)[0]
# get gram matrix
gram = tf.matmul(a, a, transpose_a=True)
return gram
def project_output(self, outputs):
"""
Project outputs into a distribution with same dimensions as the targets
:param outputs: a tensor of shape (batch_size, output_size)
:return: softmax distributions on vocab_size, a Tensor of shape (batch_size, vocab_size)
"""
if self.has_project:
target_size = self.target_size if self.target_size is not None else self.vocab_size
with tf.variable_scope('project', initializer=self.initializer):
project_w = tf.get_variable(
"project_w", [self.cell_list[-1].output_size, target_size], dtype=data_type())
projcet_b = tf.get_variable("project_b", [target_size], dtype=data_type())
return tf.matmul(outputs, project_w) + projcet_b
else:
return None
def dot_product_attention(q, k, v, bias, dropout_rate = 0.0, name = None):
with tf.variable_scope(name, default_name = 'dot_product_attention'):
logits = tf.matmul(q, k, transpose_b = True)
if bias is not None:
logits += bias
weights = tf.nn.softmax(logits, name = 'attention_weights')
weights = tf.nn.dropout(weights, 1.0 - dropout_rate)
return tf.matmul(weights, v)
layer1_size = LAYER1_SIZE
layer2_size = LAYER2_SIZE
state_input = tf.placeholder("float", [None, state_dim])
is_training = tf.placeholder(tf.bool)
W1 = self.variable([state_dim, layer1_size], state_dim)
b1 = self.variable([layer1_size], state_dim)
W2 = self.variable([layer1_size, layer2_size], layer1_size)
b2 = self.variable([layer2_size], layer1_size)
W3 = tf.Variable(tf.random_uniform([layer2_size, action_dim], -3e-3, 3e-3))
b3 = tf.Variable(tf.random_uniform([action_dim], -3e-3, 3e-3))
layer0_bn = self.batch_norm_layer(state_input, training_phase=is_training, scope_bn='batch_norm_0',
activation=tf.identity)
layer1 = tf.matmul(layer0_bn, W1) + b1
layer1_bn = self.batch_norm_layer(layer1, training_phase=is_training, scope_bn='batch_norm_1',
activation=tf.nn.relu)
layer2 = tf.matmul(layer1_bn, W2) + b2
layer2_bn = self.batch_norm_layer(layer2, training_phase=is_training, scope_bn='batch_norm_2',
activation=tf.nn.relu)
action_output = tf.tanh(tf.matmul(layer2_bn, W3) + b3)
return state_input, action_output, [W1, b1, W2, b2, W3, b3], is_training
# Overfit since our data generating model is quadratic
# Ap = tf.concat(1,[tf.concat(1,[tf.concat(1,[a*a*a,a*a]),a]),tf.ones_like(a)])
# Inputs to form y = a*a*xcoeff[0] + a*xcoeff[1] + xcoeff[2]
A = tf.concat([tf.concat([a*a,a],1),tf.ones_like(a)],1)
# Initial guess on coefficients of predicted linear model
xp = tf.Variable(tf.random_uniform([5,1], -1.0, 1.0))
# xp = tf.Variable(tf.random_uniform([4,1], -1.0, 1.0))
# Predicted Model
yp = tf.matmul(Ap,xp)
# Observed outputs
y = tf.matmul(A,xcoeff)
# noise = tf.random_normal(y.get_shape(),stddev=0.8)
noise = tf.sin(math.pi*a)
y = tf.add(y,noise)
# The smaller the loss, the closer our prediction to the observed outputs
# The loss model used is square of error (yp - y)
# Miinimization of loss is done by Gradient Descent
loss = tf.reduce_mean(tf.square(yp - y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
train_step = optimizer.minimize(loss)
init = tf.global_variables_initializer()
with tf.Session() as session:
session.run(init)
for i in range(10000):
session.run(train_step)
def gru_single_step_word_level(self, Xt, h_t_minus_1):
"""
single step of gru for word level
:param Xt: Xt:[batch_size*num_sentences,embed_size]
:param h_t_minus_1:[batch_size*num_sentences,embed_size]
:return:
"""
# update gate: decides how much past information is kept and how much new information is added.
z_t = tf.nn.sigmoid(tf.matmul(Xt, self.W_z) + tf.matmul(h_t_minus_1, self.U_z) + self.b_z) # z_t:[batch_size*num_sentences,self.hidden_size]
# reset gate: controls how much the past state contributes to the candidate state.
r_t = tf.nn.sigmoid(tf.matmul(Xt, self.W_r) + tf.matmul(h_t_minus_1, self.U_r) + self.b_r) # r_t:[batch_size*num_sentences,self.hidden_size]
# candiate state h_t~
h_t_candiate = tf.nn.tanh(tf.matmul(Xt, self.W_h) + r_t * (tf.matmul(h_t_minus_1, self.U_h)) + self.b_h) # h_t_candiate:[batch_size*num_sentences,self.hidden_size]
# new state: a linear combine of pervious hidden state and the current new state h_t~
h_t = (1 - z_t) * h_t_minus_1 + z_t * h_t_candiate # h_t:[batch_size*num_sentences,hidden_size]
return h_t
def _build_c(self, s, a, scope, trainable):
with tf.variable_scope(scope):
n_l1 = 100
w1_s = tf.get_variable('w1_s', [self.s_dim, n_l1], trainable=trainable)
w1_a = tf.get_variable('w1_a', [self.a_dim, n_l1], trainable=trainable)
b1 = tf.get_variable('b1', [1, n_l1], trainable=trainable)
net = tf.nn.relu(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1)
return tf.layers.dense(net, 1, trainable=trainable) # Q(s,a)
conv12 = conv2d(conv11, weights['wc12'], biases['bc12'])
conv13 = conv2d(conv12, weights['wc13'], biases['bc13'])
# Max Pooling
pool5 = maxpool2d(conv13, k=2)
print(pool5.shape) #(4,4,512)
# Fully connected layer
# Reshape conv2 output to fit fully connected layer input
fc1 = tf.reshape(pool5, [-1, weights['wd1'].get_shape().as_list()[0]])
fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])
fc1 = tf.nn.relu(fc1)
# Apply Dropout
fc1 = tf.nn.dropout(fc1, dropout)
#fc2 = tf.reshape(fc1, [-1, weights['wd2'].get_shape().as_list()[0]])
fc2 = tf.add(tf.matmul(fc1, weights['wd2']), biases['bd2'])
fc2 = tf.nn.relu(fc2)
# Apply Dropout
fc2 = tf.nn.dropout(fc2, dropout)
'''
fc3 = tf.reshape(fc2, [-1, weights['out'].get_shape().as_list()[0]])
fc3 = tf.add(tf.matmul(fc2, weights['out']), biases['bd2'])
fc3 = tf.nn.relu(fc2)
'''
# Output, class prediction
out = tf.add(tf.matmul(fc2, weights['out']), biases['out'])
out = tf.nn.softmax(out)
return out