matMulGradGrad :: Test
matMulGradGrad = testCase "matMulGradGrad" $ do
  let width = 2 :: Int64
      batch = 4 :: Int64

  let tower = do
        x <- TF.render $ TF.zeros $ TF.Shape [batch, 1]
        w <- TF.zeroInitializedVariable $ TF.Shape [1, width]
        let f = x `TF.matMul` TF.readValue w
        l1 <- TF.gradients f [x]
        let dfdx = head l1 -- avoid MonadFail
        let f'x = TF.reduceSum dfdx
        l2 <- TF.gradients f'x [w] -- take gradient again (this time over w)
        let dfdw = head l2
        return [TF.readValue w, TF.expr dfdw]

  TF.runSession $ do
    l <- TF.build tower
    (w, dfdw) <-
      case l of
        [w, dfdw] -> pure (w, dfdw)
        _ -> liftIO $ assertFailure "pattern-match failure in matMulGradMad"
    (wShape, dfdwShape) <- TF.run (TF.shape w, TF.shape dfdw)
    liftIO $ assertEqual "Shape of gradient must match input" wShape (dfdwShape :: V.Vector Int32)

    let step = w `TF.add` dfdw
    w0 <- TF.run step
    liftIO $ V.fromList [4, 4 :: Float] @=? w0


-- test that gradient of matMul deals correctly with transpose_a and transpose_b
matMulGradGrad :: Test
matMulGradGrad = testCase "matMulGradGrad" $ do
  let width = 2 :: Int64
      batch = 4 :: Int64

  let tower = do
        x <- TF.render $ TF.zeros $ TF.Shape [batch, 1]
        w <- TF.zeroInitializedVariable $ TF.Shape [1, width]
        let f = x `TF.matMul` TF.readValue w
        l1 <- TF.gradients f [x]
        let dfdx = head l1 -- avoid MonadFail
        let f'x = TF.reduceSum dfdx
        l2 <- TF.gradients f'x [w] -- take gradient again (this time over w)
        let dfdw = head l2
        return [TF.readValue w, TF.expr dfdw]

  TF.runSession $ do
    l <- TF.build tower
    (w, dfdw) <-
      case l of
        [w, dfdw] -> pure (w, dfdw)
        _ -> liftIO $ assertFailure "pattern-match failure in matMulGradMad"
    (wShape, dfdwShape) <- TF.run (TF.shape w, TF.shape dfdw)
    liftIO $ assertEqual "Shape of gradient must match input" wShape (dfdwShape :: V.Vector Int32)

    let step = w `TF.add` dfdw
    w0 <- TF.run step
    liftIO $ V.fromList [4, 4 :: Float] @=? w0


-- test that gradient of matMul deals correctly with transpose_a and transpose_b
matMulGradGrad = testCase "matMulGradGrad" $ do
  let width = 2 :: Int64
      batch = 4 :: Int64

  let tower = do
        x <- TF.render $ TF.zeros $ TF.Shape [batch, 1]
        w <- TF.zeroInitializedVariable $ TF.Shape [1, width]
        let f = x `TF.matMul` TF.readValue w
        l1 <- TF.gradients f [x]
        let dfdx = head l1 -- avoid MonadFail
        let f'x = TF.reduceSum dfdx
        l2 <- TF.gradients f'x [w] -- take gradient again (this time over w)
        let dfdw = head l2
        return [TF.readValue w, TF.expr dfdw]

  TF.runSession $ do
    l <- TF.build tower
    (w, dfdw) <-
      case l of
        [w, dfdw] -> pure (w, dfdw)
        _ -> liftIO $ assertFailure "pattern-match failure in matMulGradMad"
    (wShape, dfdwShape) <- TF.run (TF.shape w, TF.shape dfdw)
    liftIO $ assertEqual "Shape of gradient must match input" wShape (dfdwShape :: V.Vector Int32)

    let step = w `TF.add` dfdw
    w0 <- TF.run step
    liftIO $ V.fromList [4, 4 :: Float] @=? w0


-- test that gradient of matMul deals correctly with transpose_a and transpose_b