<NME> functions.py
<BEF> # Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#      Unless required by applicable law or agreed to in writing, software
#      distributed under the License is distributed on an "AS IS" BASIS,
#      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#      See the License for the specific language governing permissions and
#      limitations under the License.
"""Functions to test.

Notes
-----
Functions can have up to 3 scalar arguments with names a, b and c. If the
function has an optional integer argument it must be called n. Vector inputs
have the name x.

"""
from __future__ import division

import numpy as np

import tangent
from tangent import insert_grad_of

try:
  import tensorflow as tf
except ImportError:
  tf = None


def id_(a):
  return a


def unary_sub(a):
  return -a


def sum_(x):
  return np.sum(x)


def cast_sum_(x):
  return np.sum(np.array(x))


def overwrite_call(x):
  x = np.sum(x)
  x = x * x
  return x


def tanh(a):
  return np.tanh(a)


def identity_assign(a):
  b = a
  return b


def increment(a):
  return a + 1


def saxpy(a, b, c):
  return a * b + c


def mul(a, b):
  return a * b


def add(a, b):
  return a + b


def saxpy_call(a, b, c):
  return add(mul(a, b), c)


def saxpy_anf(a, b, c):
  t1 = a * b
  t2 = t1 + c
  return t2


def saxpy_overwrite(a, b, c):
  t = a * b
  t = t + c
  return t


def gradname_aliasing(a):
  da = a * a
  return da


def overwrite_if(a):
  if a > 0:
    a = a * 3.0
  else:
    a = a * 2.0
  return a


def test_binop_mul1(a):
  return a * 3.0


def test_binop_mul2(a):
  return 3.0 * a


def test_binop_mul3(a):
  return a * a


def test_binop_div1(a):
  return a / 3.0


def test_binop_div2(a):
  return 3.0 / a


def test_binop_div3(a):
  return a / a


def test_binop_sub1(a):
  return a - 3.0


def test_binop_sub2(a):
  return 3.0 - a


def test_binop_sub3(a):
  return a - a


def test_binop_add1(a):
  return a + 3.0


def test_binop_add2(a):
  return 3.0 + a


def test_binop_add3(a):
  return a + a


def nested_if(a):
  if a > 0:
    if a < 10:
      a = a * a
  else:
    a = 3.0 * a
  return a


def multiarg_if(a):
  # A bunch of spammy nonsense to try to break our system
  if a * a / 3.0 > 0:
    b = 3.0
  else:
    b = 4.0
  a = a * b
  if a < b - a * b:
    a = a * a
  elif a > b + 3.0 * a:
    if a / 2.0 < 0:
      a = a / b
    else:
      a = a * b
  else:
    a = -a
  return a


def fn_multiply(a):
  return np.multiply(a, 3.0)


def fn_multiple_return(a):
  return 2 * a, a


def test_anf_list(a):
  b = [1, a * a, 2]
  return np.sum(b)


def test_deep_anf_list(x):
  b = [1, x[0] * x[1] * x[2], 3]
  return np.sum(b)

  # TODO: needs equivalent for all iterable collections


def test_subscript1(x):
  a = x[0]
  b = x[1]
  return a * b


def test_subscript2(x):
  a = x[0]
  b = x[1]
  x[0] = a * b
  return np.sum(x)


def test_subscript3(x):
  y = x ** 2.0
  x[0] = y[1] * y[2]
  return np.sum(x * y)


def test_list_and_subscript(a):
  x = [1.0, a, 3.0]
  return x[0] * x[1]

def test_implicit_indexing(x):
  res = 0.0
  for a in x:
    res += a
  return res

# TODO: needs a numpy equivalent, and all iterables collections too
# def test_subscript_overwrite(a):
#     x = [1,a*a,3]
#     x[1] = x[0]*x[1]
#     return x[1]+x[1]


def serial_if(a):
  if a > 0:
    a = a * a
  a = a + a
  if a < 0:
    a = 3.0 * a
  return a


def multivar_if(a, b):
  if a > b:
    a = a * a
  else:
    a = a * b
  return a

# TODO: split this into a bunch of tinier tests


def serial_for(a, n):
  for i in range(n):
    a = a * a
  a = 2.0 * a
  for i in range(n + 1):
    a = a * 3.0
  return a


def serial_ifthenfor(a, n):
  if a > 0:
    a = a * a
  a = 2.0 * a
  for i in range(n):
    a = a * a
  return a


def serial_forthenif(a, n):
  for i in range(n):
    a = a * a
  a = 2.0 * a
  if a > 0:
    a = a * a
  return a


def devilish_nested_if(a):
  if a > 0:
    a = a * 3.0
    if a < 10:
      a = a * a
    else:
      a = a * 2.0
  return a


def overwrite_ifelse(a):
  if a > 0:
    a = a * 3.0
  elif 0 > a:
    a = a * 2.0
  else:
    a = 1.0 * a
  return a


def overwrite_arg(a):
  a = a * 3.0
  return a


def overwrite_non_arg(a):
  b = a
  b = b * 3.0
  return b


def third_pow(a):
  return a * a * a


def direct_third_pow(a):
  return a ** 3


def iter_third_pow1(a):
  out = 1
  for i in range(3):
    out = out * a
  return out


def iter_third_pow2(a):
  out = a
  for i in range(3 - 1):
    out = out * a
  return out


def iterpower_static(a):
  for i in range(3):
    a = a * a
  return a


def iterpower(a, n):
  for i in range(n):
    a = a * a
  return a


def cond_iterpower1(a):
  for i in range(3):
    if a < 20:
      a = a * a
  return a


def cond_iterpower2(a):
  if a < 20:
    for i in range(3):
      a = a * a
  return a


def superfor_iterpower(a):
  for i in range(2):
    for j in range(3):
      a = a * a
  return a


def super_iterpower(a):
  # Tests ANF in a for loop
  for i in range(3):
    a = a * a * a
  return a

# ================================================
# Numpy grads
# ================================================


def numpy_sum(x):
  return np.sum(x)


def numpy_mean(x):
  return np.mean(x)


def numpy_exp(a):
  return np.exp(a)


def numpy_exp2(a):
  return np.exp(np.exp(a))


def numpy_sqrt(a):
  if a >= 0:
    r = np.sqrt(a)
  else:
    r = np.sqrt(-a)
  return r


def numpy_cos(a):
  return np.cos(a)


def numpy_sin(a):
  return np.sin(a)


def numpy_tan(a):
  return np.tan(a)


def numpy_cosh(a):
  return np.cosh(a)


def numpy_sinh(a):
  return np.sinh(a)


def numpy_tanh(a):
  return np.tanh(a)


  return np.arctan(a)


# Label is 0 or 1




def numpy_atleast_3d(x):
  return np.sum(np.atleast_3d(x))


# Label is 0 or 1


def logistic_regression(input, label, W, b):
  h1 = np.dot(input, W) + b
  prediction = 0.5 * (np.tanh(h1) + 1)
  label_probabilities = prediction * label + (1 - prediction) * (1 - label)
  loss = -np.sum(np.log(label_probabilities))
  return loss


def det(sqm):
  return np.linalg.det(sqm)


# ================================================
# TFE grads
# ================================================


def tfe_negative(t):
  return tf.negative(t)


def tfe_exp(t):
  return tf.exp(t)


def tfe_log(t):
  return tf.log(t)


def tfe_tanh(t):
  return tf.tanh(t)


def tfe_cosh(t):
  return tf.cosh(t)


def tfe_sinh(t):
  return tf.sinh(t)


def tfe_rsqrt(t):
  return tf.rsqrt(t)


def tfe_expand_dims_before(t):
  return tf.expand_dims(t, 0)


def tfe_expand_dims_after(t):
  return tf.expand_dims(t, 1)


def tfe_squeeze_before(t):
  return tf.squeeze(tf.expand_dims(t, 0), 0)


def tfe_squeeze_before(t):
  return tf.squeeze(tf.expand_dims(t, 1), 1)


def tfe_reshape_flat(t):
  return tf.reshape(t, (-1,))


def tfe_reshape_noop(t):
  # TODO: Why doesn't test_forward see that shape is non-differentiable?
  return tf.reshape(tf.reshape(t, (-1,)), tf.shape(t))


def tfe_reduce_sum(timage, boolean):
  return tf.reduce_sum(timage, None, boolean)


def tfe_reduce_sum_axis(timage, boolean):
  return tf.reduce_sum(timage, [0, 1, 2], boolean)


def tfe_reduce_mean(timage, boolean):
  return tf.reduce_mean(timage, None, boolean)


def tfe_reduce_mean_axis(timage, boolean):
  return tf.reduce_mean(timage, [0, 1, 2], boolean)


def tfe_reduce_max(timage, boolean):
  return tf.reduce_max(timage, None, boolean)


def tfe_reduce_max_axis(timage, boolean):
  return tf.reduce_max(timage, [0, 1, 2], boolean)


def tfe_add(t1, t2):
  return tf.add(t1, t2)


def tfe_add_bcast(s, t):
  return tf.add(s, t)


def tfe_subtract(t1, t2):
  return tf.subtract(t1, t2)


def tfe_multiply(t1, t2):
  return tf.multiply(t1, t2)


def tfe_divide(t1, t2):
  return tf.divide(t1, t2)


def tfe_maximum(t1, t2):
  return tf.maximum(t1, t2)


def tfe_squared_difference(t1, t2):
  return tf.squared_difference(t1, t2)


def tfe_matmul(mat1, mat2, boolean1, boolean2):
  return tf.matmul(mat1, mat2, transpose_a=boolean1, transpose_b=boolean2)


def tfe_matmul_highdim(timage1, timage2, boolean1, boolean2):
  return tf.matmul(timage1, timage2, transpose_a=boolean1, transpose_b=boolean2)


def tfe_conv2d(timage, tkernel, conv2dstrides):
  return tf.nn.conv2d(timage, tkernel, conv2dstrides, 'SAME')


def tfe_max_pool(timage, pool2dsizes, conv2dstrides):
  return tf.nn.max_pool(timage, pool2dsizes, conv2dstrides, 'SAME')


def tfe_avg_pool(timage, pool2dsizes, conv2dstrides):
  return tf.nn.avg_pool(timage, pool2dsizes, conv2dstrides, 'SAME')


# ================================================
# Traced TFE calls
# ================================================


@tangent.trace
def _trace_mul(a, b):
  out = tf.multiply(a, b)
  c = 3
  del c  # invalid Tangent syntax, but invisible to tracing
  return out


def tfe_trace_fn(t):
  out = _trace_mul(t, t)
  result = tf.reduce_sum(out)
  return result


def _nontrace_mul(a, b):
  out = tf.multiply(a, b)
  return out


def tfe_notrace_fn(t):
  out = _nontrace_mul(t, t)
  result = tf.reduce_sum(out)
  return result


# ================================================
# Function calls
# ================================================


def _inner_fn(a, b):
  return a * b


def mul_with_call(a, b):
  return _inner_fn(a, b)


def if_mul_with_call(a, b):
  if a > b:
    o = _inner_fn(a, b)
  else:
    o = _inner_fn(b, a)
  return o


def iterpower_with_call(a, n):
  for i in range(n):
    a = _inner_fn(a, a)
  return a


def nested_call(a, b):
  return mul_with_call(a, b)


def iterpower_with_nested_def(a, n):
  def _fn(a, n):
    return a * n

  for i in range(n):
    a = _fn(a, a)
  return a


# ================================================
# Subscripts
# ================================================


def unpacking_args_saxpy(abc_packed_in_tuple):
  a, b, c = abc_packed_in_tuple
  return a * b + c


def dict_saxpy(val):
  return val['a'] * val['b'] + val['c']


def dict_wrapper(abc_packed_in_tuple):
  a, b, c = abc_packed_in_tuple
  return dict_saxpy(dict(a=a, b=b, c=c))


def passthru_pack(a, b, c, i):
  x = a, b, c
  return x[i]


def passthru_unpack(abc_packed_in_tuple):
  a, b, c = abc_packed_in_tuple
  return a

# ================================================
# Misc
# ================================================


def cart2polar(a, b):
  r = np.sqrt(a**2.0 + b**2.0)
  theta = np.arctan(b, a)  # Should really be arctan2
  return r, theta


def inlining_contextmanager(a):
  b = a * a
  with insert_grad_of(a) as g:
    g = g * 0.9
  c = b * a
  return c


def listcomp(a):
  return np.sum([i * 3 for i in a])


def while_big(a):
  while np.abs(a) > 0.1:
    a = a * 0.5
  return a


def active_subscript(x):
  y = np.zeros_like(x)
  for i in range(len(x)):
    y[i] = x[i]
  return np.sum(y)


def init_array_grad_maybe_active(x):
  h = np.zeros((len(x), 3))
  for t in range(len(h)):
    h[t] = x[t]
  return np.sum(h)


def slicing(x):
  return np.sum(x[1:])


def rnn(inputs, W):
  h = np.zeros((4, 3))
  for t in range(3):
    h[t + 1] = np.dot(inputs[t], W) + h[t]
  return np.sum(h[1:])


def augassign(a):
  x = a * a
  x *= x
  return x


def bilinear(x, h, U, W, b):
  y = np.dot(x, W)
  y = y + np.dot(h, U)
  y = y + b
  y = np.tanh(y)
  return np.sum(y)


# Functions were problematic in the development of grad(grad(f))
# They stress differentiation through stack pushes and pops,
# which can complicate dataflow analysis
def stack_pushing(a):
  stack = tangent.Stack()
  y = a * a
  tangent.push(stack, a, 'abc')
  aa = tangent.pop(stack, 'abc')
  z = y * aa
  z = z * aa
  return z


def gradgrad_pow(a, b):
  if a >= 0:
    a = a ** b
  else:
    a = (-a) ** b
  return a


def gradgrad_pow_twice(a):
  a = a * a
  y = a * a
  return y


def gradgrad_chain_of_mults(a):
  b = a  # this line forces overwrite protection
  b = a * a
  c = b * b
  d = c * c
  return d


def gradgrad_chain_of_multcalls(a):
  b = mul(a, a)
  c = mul(b, b)
  d = mul(c, c)
  return d


def useless_stack_ops(a):
  _stack = tangent.Stack()
  b = a * a
  tangent.push(_stack, b, 'abc')
  b = tangent.pop(_stack, 'abc')
  return a


def redefining_var_as_list(a):
  # Initialize the tape
  _stack = tangent.Stack()
  x = a
  tangent.push(_stack, x, 'abc')
  x = [a]
  x = tangent.pop(_stack, 'abc')
  return a


def nested_dict(p):
  return p['i']['j'] * p['i']['k']


def extended_slice(x):
  return np.sum(x[::2, np.newaxis])

<MSG> Implement adjoints for atleast_*d (#48)

* Implement adjoints for atleast_1,2,3-d

* Test with vector input

<DFF> @@ -435,6 +435,18 @@ def numpy_arctan(a):
   return np.arctan(a)
 
 
+def numpy_atleast_1d(x):
+  return np.sum(np.atleast_1d(x))
+
+
+def numpy_atleast_2d(x):
+  return np.sum(np.atleast_2d(x))
+
+
+def numpy_atleast_3d(x):
+  return np.sum(np.atleast_3d(x))
+
+
 # Label is 0 or 1
 
 

{'commit': 'diff --git a/tangent/grads.py b/tangent/grads.py\nindex c681bc4..e865002 100644\n--- a/tangent/grads.py\n+++ b/tangent/grads.py\n@@ -276,6 +276,21 @@ def dot(y, x1, x2):\n                                            numpy.transpose(x1)))\n \n \n+@adjoint(numpy.atleast_1d)\n+def atleast_1d(y, x):\n+  d[x] = numpy.reshape(d[y], numpy.shape(x))\n+\n+\n+@adjoint(numpy.atleast_2d)\n+def atleast_2d(y, x):\n+  d[x] = numpy.reshape(d[y], numpy.shape(x))\n+\n+\n+@adjoint(numpy.atleast_3d)\n+def atleast_3d(y, x):\n+  d[x] = numpy.reshape(d[y], numpy.shape(x))\n+\n+\n @adjoint(numpy.reshape)\n def reshape(y, x, y_shape):\n   d[x] = numpy.reshape(d[y], numpy.shape(x))\ndiff --git a/tests/functions.py b/tests/functions.py\nindex dd5e785..3ad0bc6 100644\n--- a/tests/functions.py\n+++ b/tests/functions.py\n@@ -435,6 +435,18 @@ def numpy_arctan(a):\n   return np.arctan(a)\n \n \n+def numpy_atleast_1d(x):\n+  return np.sum(np.atleast_1d(x))\n+\n+\n+def numpy_atleast_2d(x):\n+  return np.sum(np.atleast_2d(x))\n+\n+\n+def numpy_atleast_3d(x):\n+  return np.sum(np.atleast_3d(x))\n+\n+\n # Label is 0 or 1\n \n \n', 'commit_subject': 'Implement adjoints for atleast_*d (#48)', 'repo_name': 'google/tangent', 'committer': {'name': 'Dan Moldovan', 'email': '02bd3f004a63c78990e4cfa97922eaaaa6d78ea5@google.com', 'time_sec': '1511989993', 'tz_offset': '-300', 'date': {'seconds': '1511989993'}}, 'commit_tree': '91cbe64169cff216dcfdbb1726b361132dc1f8d9', 'commit_message': 'Implement adjoints for atleast_*d (#48)\n\n* Implement adjoints for atleast_1,2,3-d\r\n\r\n* Test with vector input\r\n', 'language_name': 'Python', 'license': 'apache-2.0', 'hunks': '@@ -435,6 +435,18 @@ def numpy_arctan(a):\n   return np.arctan(a)\n \n \n+def numpy_atleast_1d(x):\n+  return np.sum(np.atleast_1d(x))\n+\n+\n+def numpy_atleast_2d(x):\n+  return np.sum(np.atleast_2d(x))\n+\n+\n+def numpy_atleast_3d(x):\n+  return np.sum(np.atleast_3d(x))\n+\n+\n # Label is 0 or 1\n \n \n', 'addition_count': 12, 'deletion_count': 0, 'src_file': 'a/tests/functions.py', 'tgt_file': 'b/tests/functions.py', 'file_extension': '.py', 'before_file': '# Copyright 2017 Google Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the "License");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n#      Unless required by applicable law or agreed to in writing, software\n#      distributed under the License is distributed on an "AS IS" BASIS,\n#      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#      See the License for the specific language governing permissions and\n#      limitations under the License.\n"""Functions to test.\n\nNotes\n-----\nFunctions can have up to 3 scalar arguments with names a, b and c. If the\nfunction has an optional integer argument it must be called n. Vector inputs\nhave the name x.\n\n"""\nfrom __future__ import division\n\nimport numpy as np\n\nimport tangent\nfrom tangent import insert_grad_of\n\ntry:\n  import tensorflow as tf\nexcept ImportError:\n  tf = None\n\n\ndef id_(a):\n  return a\n\n\ndef unary_sub(a):\n  return -a\n\n\ndef sum_(x):\n  return np.sum(x)\n\n\ndef cast_sum_(x):\n  return np.sum(np.array(x))\n\n\ndef overwrite_call(x):\n  x = np.sum(x)\n  x = x * x\n  return x\n\n\ndef tanh(a):\n  return np.tanh(a)\n\n\ndef identity_assign(a):\n  b = a\n  return b\n\n\ndef increment(a):\n  return a + 1\n\n\ndef saxpy(a, b, c):\n  return a * b + c\n\n\ndef mul(a, b):\n  return a * b\n\n\ndef add(a, b):\n  return a + b\n\n\ndef saxpy_call(a, b, c):\n  return add(mul(a, b), c)\n\n\ndef saxpy_anf(a, b, c):\n  t1 = a * b\n  t2 = t1 + c\n  return t2\n\n\ndef saxpy_overwrite(a, b, c):\n  t = a * b\n  t = t + c\n  return t\n\n\ndef gradname_aliasing(a):\n  da = a * a\n  return da\n\n\ndef overwrite_if(a):\n  if a > 0:\n    a = a * 3.0\n  else:\n    a = a * 2.0\n  return a\n\n\ndef test_binop_mul1(a):\n  return a * 3.0\n\n\ndef test_binop_mul2(a):\n  return 3.0 * a\n\n\ndef test_binop_mul3(a):\n  return a * a\n\n\ndef test_binop_div1(a):\n  return a / 3.0\n\n\ndef test_binop_div2(a):\n  return 3.0 / a\n\n\ndef test_binop_div3(a):\n  return a / a\n\n\ndef test_binop_sub1(a):\n  return a - 3.0\n\n\ndef test_binop_sub2(a):\n  return 3.0 - a\n\n\ndef test_binop_sub3(a):\n  return a - a\n\n\ndef test_binop_add1(a):\n  return a + 3.0\n\n\ndef test_binop_add2(a):\n  return 3.0 + a\n\n\ndef test_binop_add3(a):\n  return a + a\n\n\ndef nested_if(a):\n  if a > 0:\n    if a < 10:\n      a = a * a\n  else:\n    a = 3.0 * a\n  return a\n\n\ndef multiarg_if(a):\n  # A bunch of spammy nonsense to try to break our system\n  if a * a / 3.0 > 0:\n    b = 3.0\n  else:\n    b = 4.0\n  a = a * b\n  if a < b - a * b:\n    a = a * a\n  elif a > b + 3.0 * a:\n    if a / 2.0 < 0:\n      a = a / b\n    else:\n      a = a * b\n  else:\n    a = -a\n  return a\n\n\ndef fn_multiply(a):\n  return np.multiply(a, 3.0)\n\n\ndef fn_multiple_return(a):\n  return 2 * a, a\n\n\ndef test_anf_list(a):\n  b = [1, a * a, 2]\n  return np.sum(b)\n\n\ndef test_deep_anf_list(x):\n  b = [1, x[0] * x[1] * x[2], 3]\n  return np.sum(b)\n\n  # TODO: needs equivalent for all iterable collections\n\n\ndef test_subscript1(x):\n  a = x[0]\n  b = x[1]\n  return a * b\n\n\ndef test_subscript2(x):\n  a = x[0]\n  b = x[1]\n  x[0] = a * b\n  return np.sum(x)\n\n\ndef test_subscript3(x):\n  y = x ** 2.0\n  x[0] = y[1] * y[2]\n  return np.sum(x * y)\n\n\ndef test_list_and_subscript(a):\n  x = [1.0, a, 3.0]\n  return x[0] * x[1]\n\ndef test_implicit_indexing(x):\n  res = 0.0\n  for a in x:\n    res += a\n  return res\n\n# TODO: needs a numpy equivalent, and all iterables collections too\n# def test_subscript_overwrite(a):\n#     x = [1,a*a,3]\n#     x[1] = x[0]*x[1]\n#     return x[1]+x[1]\n\n\ndef serial_if(a):\n  if a > 0:\n    a = a * a\n  a = a + a\n  if a < 0:\n    a = 3.0 * a\n  return a\n\n\ndef multivar_if(a, b):\n  if a > b:\n    a = a * a\n  else:\n    a = a * b\n  return a\n\n# TODO: split this into a bunch of tinier tests\n\n\ndef serial_for(a, n):\n  for i in range(n):\n    a = a * a\n  a = 2.0 * a\n  for i in range(n + 1):\n    a = a * 3.0\n  return a\n\n\ndef serial_ifthenfor(a, n):\n  if a > 0:\n    a = a * a\n  a = 2.0 * a\n  for i in range(n):\n    a = a * a\n  return a\n\n\ndef serial_forthenif(a, n):\n  for i in range(n):\n    a = a * a\n  a = 2.0 * a\n  if a > 0:\n    a = a * a\n  return a\n\n\ndef devilish_nested_if(a):\n  if a > 0:\n    a = a * 3.0\n    if a < 10:\n      a = a * a\n    else:\n      a = a * 2.0\n  return a\n\n\ndef overwrite_ifelse(a):\n  if a > 0:\n    a = a * 3.0\n  elif 0 > a:\n    a = a * 2.0\n  else:\n    a = 1.0 * a\n  return a\n\n\ndef overwrite_arg(a):\n  a = a * 3.0\n  return a\n\n\ndef overwrite_non_arg(a):\n  b = a\n  b = b * 3.0\n  return b\n\n\ndef third_pow(a):\n  return a * a * a\n\n\ndef direct_third_pow(a):\n  return a ** 3\n\n\ndef iter_third_pow1(a):\n  out = 1\n  for i in range(3):\n    out = out * a\n  return out\n\n\ndef iter_third_pow2(a):\n  out = a\n  for i in range(3 - 1):\n    out = out * a\n  return out\n\n\ndef iterpower_static(a):\n  for i in range(3):\n    a = a * a\n  return a\n\n\ndef iterpower(a, n):\n  for i in range(n):\n    a = a * a\n  return a\n\n\ndef cond_iterpower1(a):\n  for i in range(3):\n    if a < 20:\n      a = a * a\n  return a\n\n\ndef cond_iterpower2(a):\n  if a < 20:\n    for i in range(3):\n      a = a * a\n  return a\n\n\ndef superfor_iterpower(a):\n  for i in range(2):\n    for j in range(3):\n      a = a * a\n  return a\n\n\ndef super_iterpower(a):\n  # Tests ANF in a for loop\n  for i in range(3):\n    a = a * a * a\n  return a\n\n# ================================================\n# Numpy grads\n# ================================================\n\n\ndef numpy_sum(x):\n  return np.sum(x)\n\n\ndef numpy_mean(x):\n  return np.mean(x)\n\n\ndef numpy_exp(a):\n  return np.exp(a)\n\n\ndef numpy_exp2(a):\n  return np.exp(np.exp(a))\n\n\ndef numpy_sqrt(a):\n  if a >= 0:\n    r = np.sqrt(a)\n  else:\n    r = np.sqrt(-a)\n  return r\n\n\ndef numpy_cos(a):\n  return np.cos(a)\n\n\ndef numpy_sin(a):\n  return np.sin(a)\n\n\ndef numpy_tan(a):\n  return np.tan(a)\n\n\ndef numpy_cosh(a):\n  return np.cosh(a)\n\n\ndef numpy_sinh(a):\n  return np.sinh(a)\n\n\ndef numpy_tanh(a):\n  return np.tanh(a)\n\n\n  return np.arctan(a)\n\n\n# Label is 0 or 1\n\n\n\n\ndef numpy_atleast_3d(x):\n  return np.sum(np.atleast_3d(x))\n\n\n# Label is 0 or 1\n\n\ndef logistic_regression(input, label, W, b):\n  h1 = np.dot(input, W) + b\n  prediction = 0.5 * (np.tanh(h1) + 1)\n  label_probabilities = prediction * label + (1 - prediction) * (1 - label)\n  loss = -np.sum(np.log(label_probabilities))\n  return loss\n\n\ndef det(sqm):\n  return np.linalg.det(sqm)\n\n\n# ================================================\n# TFE grads\n# ================================================\n\n\ndef tfe_negative(t):\n  return tf.negative(t)\n\n\ndef tfe_exp(t):\n  return tf.exp(t)\n\n\ndef tfe_log(t):\n  return tf.log(t)\n\n\ndef tfe_tanh(t):\n  return tf.tanh(t)\n\n\ndef tfe_cosh(t):\n  return tf.cosh(t)\n\n\ndef tfe_sinh(t):\n  return tf.sinh(t)\n\n\ndef tfe_rsqrt(t):\n  return tf.rsqrt(t)\n\n\ndef tfe_expand_dims_before(t):\n  return tf.expand_dims(t, 0)\n\n\ndef tfe_expand_dims_after(t):\n  return tf.expand_dims(t, 1)\n\n\ndef tfe_squeeze_before(t):\n  return tf.squeeze(tf.expand_dims(t, 0), 0)\n\n\ndef tfe_squeeze_before(t):\n  return tf.squeeze(tf.expand_dims(t, 1), 1)\n\n\ndef tfe_reshape_flat(t):\n  return tf.reshape(t, (-1,))\n\n\ndef tfe_reshape_noop(t):\n  # TODO: Why doesn\'t test_forward see that shape is non-differentiable?\n  return tf.reshape(tf.reshape(t, (-1,)), tf.shape(t))\n\n\ndef tfe_reduce_sum(timage, boolean):\n  return tf.reduce_sum(timage, None, boolean)\n\n\ndef tfe_reduce_sum_axis(timage, boolean):\n  return tf.reduce_sum(timage, [0, 1, 2], boolean)\n\n\ndef tfe_reduce_mean(timage, boolean):\n  return tf.reduce_mean(timage, None, boolean)\n\n\ndef tfe_reduce_mean_axis(timage, boolean):\n  return tf.reduce_mean(timage, [0, 1, 2], boolean)\n\n\ndef tfe_reduce_max(timage, boolean):\n  return tf.reduce_max(timage, None, boolean)\n\n\ndef tfe_reduce_max_axis(timage, boolean):\n  return tf.reduce_max(timage, [0, 1, 2], boolean)\n\n\ndef tfe_add(t1, t2):\n  return tf.add(t1, t2)\n\n\ndef tfe_add_bcast(s, t):\n  return tf.add(s, t)\n\n\ndef tfe_subtract(t1, t2):\n  return tf.subtract(t1, t2)\n\n\ndef tfe_multiply(t1, t2):\n  return tf.multiply(t1, t2)\n\n\ndef tfe_divide(t1, t2):\n  return tf.divide(t1, t2)\n\n\ndef tfe_maximum(t1, t2):\n  return tf.maximum(t1, t2)\n\n\ndef tfe_squared_difference(t1, t2):\n  return tf.squared_difference(t1, t2)\n\n\ndef tfe_matmul(mat1, mat2, boolean1, boolean2):\n  return tf.matmul(mat1, mat2, transpose_a=boolean1, transpose_b=boolean2)\n\n\ndef tfe_matmul_highdim(timage1, timage2, boolean1, boolean2):\n  return tf.matmul(timage1, timage2, transpose_a=boolean1, transpose_b=boolean2)\n\n\ndef tfe_conv2d(timage, tkernel, conv2dstrides):\n  return tf.nn.conv2d(timage, tkernel, conv2dstrides, \'SAME\')\n\n\ndef tfe_max_pool(timage, pool2dsizes, conv2dstrides):\n  return tf.nn.max_pool(timage, pool2dsizes, conv2dstrides, \'SAME\')\n\n\ndef tfe_avg_pool(timage, pool2dsizes, conv2dstrides):\n  return tf.nn.avg_pool(timage, pool2dsizes, conv2dstrides, \'SAME\')\n\n\n# ================================================\n# Traced TFE calls\n# ================================================\n\n\n@tangent.trace\ndef _trace_mul(a, b):\n  out = tf.multiply(a, b)\n  c = 3\n  del c  # invalid Tangent syntax, but invisible to tracing\n  return out\n\n\ndef tfe_trace_fn(t):\n  out = _trace_mul(t, t)\n  result = tf.reduce_sum(out)\n  return result\n\n\ndef _nontrace_mul(a, b):\n  out = tf.multiply(a, b)\n  return out\n\n\ndef tfe_notrace_fn(t):\n  out = _nontrace_mul(t, t)\n  result = tf.reduce_sum(out)\n  return result\n\n\n# ================================================\n# Function calls\n# ================================================\n\n\ndef _inner_fn(a, b):\n  return a * b\n\n\ndef mul_with_call(a, b):\n  return _inner_fn(a, b)\n\n\ndef if_mul_with_call(a, b):\n  if a > b:\n    o = _inner_fn(a, b)\n  else:\n    o = _inner_fn(b, a)\n  return o\n\n\ndef iterpower_with_call(a, n):\n  for i in range(n):\n    a = _inner_fn(a, a)\n  return a\n\n\ndef nested_call(a, b):\n  return mul_with_call(a, b)\n\n\ndef iterpower_with_nested_def(a, n):\n  def _fn(a, n):\n    return a * n\n\n  for i in range(n):\n    a = _fn(a, a)\n  return a\n\n\n# ================================================\n# Subscripts\n# ================================================\n\n\ndef unpacking_args_saxpy(abc_packed_in_tuple):\n  a, b, c = abc_packed_in_tuple\n  return a * b + c\n\n\ndef dict_saxpy(val):\n  return val[\'a\'] * val[\'b\'] + val[\'c\']\n\n\ndef dict_wrapper(abc_packed_in_tuple):\n  a, b, c = abc_packed_in_tuple\n  return dict_saxpy(dict(a=a, b=b, c=c))\n\n\ndef passthru_pack(a, b, c, i):\n  x = a, b, c\n  return x[i]\n\n\ndef passthru_unpack(abc_packed_in_tuple):\n  a, b, c = abc_packed_in_tuple\n  return a\n\n# ================================================\n# Misc\n# ================================================\n\n\ndef cart2polar(a, b):\n  r = np.sqrt(a**2.0 + b**2.0)\n  theta = np.arctan(b, a)  # Should really be arctan2\n  return r, theta\n\n\ndef inlining_contextmanager(a):\n  b = a * a\n  with insert_grad_of(a) as g:\n    g = g * 0.9\n  c = b * a\n  return c\n\n\ndef listcomp(a):\n  return np.sum([i * 3 for i in a])\n\n\ndef while_big(a):\n  while np.abs(a) > 0.1:\n    a = a * 0.5\n  return a\n\n\ndef active_subscript(x):\n  y = np.zeros_like(x)\n  for i in range(len(x)):\n    y[i] = x[i]\n  return np.sum(y)\n\n\ndef init_array_grad_maybe_active(x):\n  h = np.zeros((len(x), 3))\n  for t in range(len(h)):\n    h[t] = x[t]\n  return np.sum(h)\n\n\ndef slicing(x):\n  return np.sum(x[1:])\n\n\ndef rnn(inputs, W):\n  h = np.zeros((4, 3))\n  for t in range(3):\n    h[t + 1] = np.dot(inputs[t], W) + h[t]\n  return np.sum(h[1:])\n\n\ndef augassign(a):\n  x = a * a\n  x *= x\n  return x\n\n\ndef bilinear(x, h, U, W, b):\n  y = np.dot(x, W)\n  y = y + np.dot(h, U)\n  y = y + b\n  y = np.tanh(y)\n  return np.sum(y)\n\n\n# Functions were problematic in the development of grad(grad(f))\n# They stress differentiation through stack pushes and pops,\n# which can complicate dataflow analysis\ndef stack_pushing(a):\n  stack = tangent.Stack()\n  y = a * a\n  tangent.push(stack, a, \'abc\')\n  aa = tangent.pop(stack, \'abc\')\n  z = y * aa\n  z = z * aa\n  return z\n\n\ndef gradgrad_pow(a, b):\n  if a >= 0:\n    a = a ** b\n  else:\n    a = (-a) ** b\n  return a\n\n\ndef gradgrad_pow_twice(a):\n  a = a * a\n  y = a * a\n  return y\n\n\ndef gradgrad_chain_of_mults(a):\n  b = a  # this line forces overwrite protection\n  b = a * a\n  c = b * b\n  d = c * c\n  return d\n\n\ndef gradgrad_chain_of_multcalls(a):\n  b = mul(a, a)\n  c = mul(b, b)\n  d = mul(c, c)\n  return d\n\n\ndef useless_stack_ops(a):\n  _stack = tangent.Stack()\n  b = a * a\n  tangent.push(_stack, b, \'abc\')\n  b = tangent.pop(_stack, \'abc\')\n  return a\n\n\ndef redefining_var_as_list(a):\n  # Initialize the tape\n  _stack = tangent.Stack()\n  x = a\n  tangent.push(_stack, x, \'abc\')\n  x = [a]\n  x = tangent.pop(_stack, \'abc\')\n  return a\n\n\ndef nested_dict(p):\n  return p[\'i\'][\'j\'] * p[\'i\'][\'k\']\n\n\ndef extended_slice(x):\n  return np.sum(x[::2, np.newaxis])\n', 'message': 'Implement adjoints for atleast_*d (#48)\n\n* Implement adjoints for atleast_1,2,3-d\r\n\r\n* Test with vector input\r\n', 'author_name': 'Dan Moldovan', 'lang': 'py'}