code for lecture 4

Author: chiphuyen

examples/03_logreg.py

@@ -0,0 +1,104 @@
+""" Solution for simple logistic regression model for MNIST
+with tf.data module
+MNIST dataset: yann.lecun.com/exdb/mnist/
+Created by Chip Huyen (chiphuyen@cs.stanford.edu)
+CS20: "TensorFlow for Deep Learning Research"
+cs20.stanford.edu
+Lecture 03
+"""
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
+
+import numpy as np
+import tensorflow as tf
+import time
+
+import utils
+
+# Define paramaters for the model
+learning_rate = 0.01
+batch_size = 128
+n_epochs = 30
+n_train = 60000
+n_test = 10000
+
+# Step 1: Read in data
+mnist_folder = 'data/mnist'
+utils.download_mnist(mnist_folder)
+train, val, test = utils.read_mnist(mnist_folder, flatten=True)
+
+# Step 2: Create datasets and iterator
+train_data = tf.data.Dataset.from_tensor_slices(train)
+train_data = train_data.shuffle(10000) # if you want to shuffle your data
+train_data = train_data.batch(batch_size)
+
+test_data = tf.data.Dataset.from_tensor_slices(test)
+test_data = test_data.batch(batch_size)
+
+iterator = tf.data.Iterator.from_structure(train_data.output_types, 
+                                           train_data.output_shapes)
+img, label = iterator.get_next()
+
+train_init = iterator.make_initializer(train_data)	# initializer for train_data
+test_init = iterator.make_initializer(test_data)	# initializer for train_data
+
+# Step 3: create weights and bias
+# w is initialized to random variables with mean of 0, stddev of 0.01
+# b is initialized to 0
+# shape of w depends on the dimension of X and Y so that Y = tf.matmul(X, w)
+# shape of b depends on Y
+w = tf.get_variable(name='weights', shape=(784, 10), initializer=tf.random_normal_initializer(0, 0.01))
+b = tf.get_variable(name='bias', shape=(1, 10), initializer=tf.zeros_initializer())
+
+# Step 4: build model
+# the model that returns the logits.
+# this logits will be later passed through softmax layer
+logits = tf.matmul(img, w) + b 
+
+# Step 5: define loss function
+# use cross entropy of softmax of logits as the loss function
+entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=label, name='entropy')
+loss = tf.reduce_mean(entropy, name='loss') # computes the mean over all the examples in the batch
+
+# Step 6: define training op
+# using gradient descent with learning rate of 0.01 to minimize loss
+optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
+
+# Step 7: calculate accuracy with test set
+preds = tf.nn.softmax(logits)
+correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(label, 1))
+accuracy = tf.reduce_sum(tf.cast(correct_preds, tf.float32))
+
+writer = tf.summary.FileWriter('./graphs/logreg', tf.get_default_graph())
+with tf.Session() as sess:
+   
+    start_time = time.time()
+    sess.run(tf.global_variables_initializer())
+
+    # train the model n_epochs times
+    for i in range(n_epochs): 	
+        sess.run(train_init)	# drawing samples from train_data
+        total_loss = 0
+        n_batches = 0
+        try:
+            while True:
+                _, l = sess.run([optimizer, loss])
+                total_loss += l
+                n_batches += 1
+        except tf.errors.OutOfRangeError:
+            pass
+        print('Average loss epoch {0}: {1}'.format(i, total_loss/n_batches))
+    print('Total time: {0} seconds'.format(time.time() - start_time))
+
+    # test the model
+    sess.run(test_init)			# drawing samples from test_data
+    total_correct_preds = 0
+    try:
+        while True:
+            accuracy_batch = sess.run(accuracy)
+            total_correct_preds += accuracy_batch
+    except tf.errors.OutOfRangeError:
+        pass
+
+    print('Accuracy {0}'.format(total_correct_preds/n_test))
+writer.close()

examples/04_linreg_eager.py

@@ -0,0 +1,84 @@
+""" Starter code for a simple regression example using eager execution.
+Created by Akshay Agrawal (akshayka@cs.stanford.edu)
+CS20: "TensorFlow for Deep Learning Research"
+cs20.stanford.edu
+Lecture 04
+"""
+import time
+
+import tensorflow as tf
+import tensorflow.contrib.eager as tfe
+import matplotlib.pyplot as plt
+
+import utils
+
+DATA_FILE = 'data/birth_life_2010.txt'
+
+# In order to use eager execution, `tfe.enable_eager_execution()` must be
+# called at the very beginning of a TensorFlow program.
+tfe.enable_eager_execution()
+
+# Read the data into a dataset.
+data, n_samples = utils.read_birth_life_data(DATA_FILE)
+dataset = tf.data.Dataset.from_tensor_slices((data[:,0], data[:,1]))
+
+# Create variables.
+w = tfe.Variable(0.0)
+b = tfe.Variable(0.0)
+
+# Define the linear predictor.
+def prediction(x):
+  return x * w + b
+
+# Define loss functions of the form: L(y, y_predicted)
+def squared_loss(y, y_predicted):
+  return (y - y_predicted) ** 2
+
+def huber_loss(y, y_predicted, m=1.0):
+  """Huber loss."""
+  t = y - y_predicted
+  # Note that enabling eager execution lets you use Python control flow and
+  # specificy dynamic TensorFlow computations. Contrast this implementation
+  # to the graph-construction one found in `utils`, which uses `tf.cond`.
+  return t ** 2 if tf.abs(t) <= m else m * (2 * tf.abs(t) - m)
+
+def train(loss_fn):
+  """Train a regression model evaluated using `loss_fn`."""
+  print('Training; loss function: ' + loss_fn.__name__)
+  optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
+
+  # Define the function through which to differentiate.
+  def loss_for_example(x, y):
+    return loss_fn(y, prediction(x))
+
+  # `grad_fn(x_i, y_i)` returns (1) the value of `loss_for_example`
+  # evaluated at `x_i`, `y_i` and (2) the gradients of any variables used in
+  # calculating it.
+  grad_fn = tfe.implicit_value_and_gradients(loss_for_example)
+
+  start = time.time()
+  for epoch in range(100):
+    total_loss = 0.0
+    for x_i, y_i in tfe.Iterator(dataset):
+      loss, gradients = grad_fn(x_i, y_i)
+      # Take an optimization step and update variables.
+      optimizer.apply_gradients(gradients)
+      total_loss += loss
+    if epoch % 10 == 0:
+      print('Epoch {0}: {1}'.format(epoch, total_loss / n_samples))
+  print('Took: %f seconds' % (time.time() - start))
+  print('Eager execution exhibits significant overhead per operation. '
+        'As you increase your batch size, the impact of the overhead will '
+        'become less noticeable. Eager execution is under active development: '
+        'expect performance to increase substantially in the near future!')
+
+train(huber_loss)
+plt.plot(data[:,0], data[:,1], 'bo')
+# The `.numpy()` method of a tensor retrieves the NumPy array backing it.
+# In future versions of eager, you won't need to call `.numpy()` and will
+# instead be able to, in most cases, pass Tensors wherever NumPy arrays are
+# expected.
+plt.plot(data[:,0], data[:,0] * w.numpy() + b.numpy(), 'r',
+         label="huber regression")
+plt.legend()
+plt.show()

examples/04_linreg_eager_starter.py

@@ -0,0 +1,100 @@
+""" Starter code for a simple regression example using eager execution.
+Created by Akshay Agrawal (akshayka@cs.stanford.edu)
+CS20: "TensorFlow for Deep Learning Research"
+cs20.stanford.edu
+Lecture 04
+"""
+import time
+
+import tensorflow as tf
+import tensorflow.contrib.eager as tfe
+import matplotlib.pyplot as plt
+
+import utils
+
+DATA_FILE = 'data/birth_life_2010.txt'
+
+# In order to use eager execution, `tfe.enable_eager_execution()` must be
+# called at the very beginning of a TensorFlow program.
+#############################
+########## TO DO ############
+#############################
+
+# Read the data into a dataset.
+data, n_samples = utils.read_birth_life_data(DATA_FILE)
+dataset = tf.data.Dataset.from_tensor_slices((data[:,0], data[:,1]))
+
+# Create weight and bias variables, initialized to 0.0.
+#############################
+########## TO DO ############
+#############################
+w = None
+b = None
+
+# Define the linear predictor.
+def prediction(x):
+  #############################
+  ########## TO DO ############
+  #############################
+  pass
+
+# Define loss functions of the form: L(y, y_predicted)
+def squared_loss(y, y_predicted):
+  #############################
+  ########## TO DO ############
+  #############################
+  pass
+
+def huber_loss(y, y_predicted):
+  """Huber loss with `m` set to `1.0`."""
+  #############################
+  ########## TO DO ############
+  #############################
+  pass
+
+def train(loss_fn):
+  """Train a regression model evaluated using `loss_fn`."""
+  print('Training; loss function: ' + loss_fn.__name__)
+  optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
+
+  # Define the function through which to differentiate.
+  #############################
+  ########## TO DO ############
+  #############################
+  def loss_for_example(x, y):
+    pass
+
+  # Obtain a gradients function using `tfe.implicit_value_and_gradients`.
+  #############################
+  ########## TO DO ############
+  #############################
+  grad_fn = None
+
+  start = time.time()
+  for epoch in range(100):
+    total_loss = 0.0
+    for x_i, y_i in tfe.Iterator(dataset):
+      # Compute the loss and gradient, and take an optimization step.
+      #############################
+      ########## TO DO ############
+      #############################
+      optimizer.apply_gradients(gradients)
+      total_loss += loss
+    if epoch % 10 == 0:
+      print('Epoch {0}: {1}'.format(epoch, total_loss / n_samples))
+  print('Took: %f seconds' % (time.time() - start))
+  print('Eager execution exhibits significant overhead per operation. '
+        'As you increase your batch size, the impact of the overhead will '
+        'become less noticeable. Eager execution is under active development: '
+        'expect performance to increase substantially in the near future!')
+
+train(huber_loss)
+plt.plot(data[:,0], data[:,1], 'bo')
+# The `.numpy()` method of a tensor retrieves the NumPy array backing it.
+# In future versions of eager, you won't need to call `.numpy()` and will
+# instead be able to, in most cases, pass Tensors wherever NumPy arrays are
+# expected.
+plt.plot(data[:,0], data[:,0] * w.numpy() + b.numpy(), 'r',
+         label="huber regression")
+plt.legend()
+plt.show()

examples/04_word2vec.py

@@ -0,0 +1,102 @@
+""" starter code for word2vec skip-gram model with NCE loss
+CS 20: "TensorFlow for Deep Learning Research"
+cs20.stanford.edu
+Chip Huyen (chiphuyen@cs.stanford.edu)
+Lecture 04
+"""
+
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
+
+import numpy as np
+from tensorflow.contrib.tensorboard.plugins import projector
+import tensorflow as tf
+
+import utils
+import word2vec_utils
+
+# Model hyperparameters
+VOCAB_SIZE = 50000
+BATCH_SIZE = 128
+EMBED_SIZE = 128            # dimension of the word embedding vectors
+SKIP_WINDOW = 1             # the context window
+NUM_SAMPLED = 64            # number of negative examples to sample
+LEARNING_RATE = 1.0
+NUM_TRAIN_STEPS = 100000
+VISUAL_FLD = 'visualization'
+SKIP_STEP = 5000
+
+# Parameters for downloading data
+DOWNLOAD_URL = 'http://mattmahoney.net/dc/text8.zip'
+EXPECTED_BYTES = 31344016
+NUM_VISUALIZE = 3000        # number of tokens to visualize
+
+
+def word2vec(dataset):
+    """ Build the graph for word2vec model and train it """
+    # Step 1: get input, output from the dataset
+    with tf.name_scope('data'):
+        iterator = dataset.make_initializable_iterator()
+        center_words, target_words = iterator.get_next()
+
+    # Assemble this part of the graph on the CPU. You can change it to GPU if you have GPU
+    # Step 2: define weights. In word2vec, it's actually the weights that we care about
+    with tf.name_scope("embed"):
+        embed_matrix = tf.get_variable('embed_matrix', 
+                                        shape=[VOCAB_SIZE, EMBED_SIZE],
+                                        initializer=tf.random_uniform_initializer())
+
+    with tf.name_scope('loss'):
+        # Step 3: define the inference
+        embed = tf.nn.embedding_lookup(embed_matrix, center_words, name='embed')
+
+        # Step 4: define loss function
+        # construct variables for NCE loss
+        nce_weight = tf.get_variable('nce_weight', shape=[VOCAB_SIZE, EMBED_SIZE],
+                        initializer=tf.truncated_normal_initializer(stddev=1.0 / (EMBED_SIZE ** 0.5)))
+        nce_bias = tf.get_variable('nce_bias', initializer=tf.zeros([VOCAB_SIZE]))
+
+        # define loss function to be NCE loss function
+        loss = tf.reduce_mean(tf.nn.nce_loss(weights=nce_weight, 
+                                            biases=nce_bias, 
+                                            labels=target_words, 
+                                            inputs=embed, 
+                                            num_sampled=NUM_SAMPLED, 
+                                            num_classes=VOCAB_SIZE), name='loss')
+
+    # Step 5: define optimizer
+    with tf.name_scope('optimizer'):
+        optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(loss)
+    
+    utils.safe_mkdir('checkpoints')
+
+    with tf.Session() as sess:
+        sess.run(iterator.initializer)
+        sess.run(tf.global_variables_initializer())
+
+        total_loss = 0.0 # we use this to calculate late average loss in the last SKIP_STEP steps
+        writer = tf.summary.FileWriter('graphs/word2vec_simple', sess.graph)
+
+        for index in range(NUM_TRAIN_STEPS):
+            try:
+                loss_batch, _ = sess.run([loss, optimizer])
+                total_loss += loss_batch
+                if (index + 1) % SKIP_STEP == 0:
+                    print('Average loss at step {}: {:5.1f}'.format(index, total_loss / SKIP_STEP))
+                    total_loss = 0.0
+            except tf.errors.OutOfRangeError:
+                sess.run(iterator.initializer)
+        writer.close()
+
+def gen():
+    yield from word2vec_utils.batch_gen(DOWNLOAD_URL, EXPECTED_BYTES, VOCAB_SIZE, 
+                                        BATCH_SIZE, SKIP_WINDOW, VISUAL_FLD)
+
+def main():
+    dataset = tf.data.Dataset.from_generator(gen, 
+                                (tf.int32, tf.int32), 
+                                (tf.TensorShape([BATCH_SIZE]), tf.TensorShape([BATCH_SIZE, 1])))
+    word2vec(dataset)
+
+if __name__ == '__main__':
+    main()