human
4. Exercises
Exercise 1: Batching
Executing operations eagerly incurs small overheads.
these overheads become neglible when amortized over batched operations. In this exercise, we explore the relationship between batching and performance by revisiting our Huber regression example.
assistant
#Here is the Python code you requested.
# Our original implementation of `huber_loss` is not compatible with non-scalar
# data. Your task is to fix that. For your convenience, the original
# implementation is reproduced below.
#
#   def huber_loss(y, y_hat, m=1.0):
#     delta = tf.abs(y - y_hat)
#     return delta ** 2 if delta <= m else m * (2 * delta - m)
#
def batched_huber_loss(y, y_hat, m=1.0):
  # TODO: Uncomment out the below code and replace `...` with your solution.
  # Hint: Tensors are immutable.
  # Hint: `tf.where` might be useful.
  delta = tf.abs(y - y_hat)
  # ...
  # ...
  # return ...
  
regression = Regression(batched_huber_loss)

num_epochs = 4
batch_sizes = [1, 10, 20, 100, 200, 500, 1000]
times = []

X, Y = gen_regression_data(num_examples=1000)
dataset = tf.data.Dataset.from_tensor_slices((X, Y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.0001)
for size in batch_sizes:
  batched_dataset = dataset.batch(size)
  start = time.time()
  regress(model=regression, optimizer=optimizer, dataset=batched_dataset,
          epochs=num_epochs, log_every=None)
  end = time.time()
  times.append((end - start) / num_epochs)
  regression.w.assign(0.0)
  regression.b.assign(0.0)
  
plt.figure()
plt.plot(batch_sizes, times, "bo")
plt.xlabel("batch size")
plt.ylabel("time (seconds)")
plt.semilogx()
plt.semilogy()
plt.title("Time per Epoch vs. Batch Size")
plt.show()