anchor.diff:              if bias then r.hidden <- r.hidden + b.value
               r.hidden <- nonlinearity r.hidden
               output.[i] <- r.hidden
  -        dsharp.stack output
  +        let output = dsharp.stack output
  +        if batchFirst then output.transpose(0, 1) else output
   
   
  -type RNN(inFeatures, outFeatures, ?nonlinearity, ?numLayers, ?bias, ?dropout, ?bidirectional) =
  +type RNN(inFeatures, outFeatures, ?numLayers, ?nonlinearity, ?bias, ?batchFirst, ?dropout, ?bidirectional) =
       inherit Model()
       let numLayers = defaultArg numLayers 1
       let dropout = defaultArg dropout 0.
       let bidirectional = defaultArg bidirectional false
  +    let batchFirst = defaultArg batchFirst false
       let numDirections = if bidirectional then 2 else 1
       let makeLayers () = Array.init numLayers (fun i -> if i = 0 then RNNCell(inFeatures, outFeatures, ?nonlinearity=nonlinearity, ?bias=bias) else RNNCell(outFeatures, outFeatures, ?nonlinearity=nonlinearity, ?bias=bias))
       let layers = makeLayers()
  
positive.code:     let hs = Parameter <| dsharp.tensor([])
      do 
          base.add(layers |> Array.map box, Array.init numLayers (fun i -> sprintf "RNN-layer-%A" i))
          if dropout > 0. then base.add([dropoutLayer], ["RNN-dropout"])
  
      member _.hidden
  
negative.code:                 dsharp.cat([hFwd; hRev], 2)
              else hFwd
          r.hidden <- dsharp.stack(newhs)
          if batchFirst then output.transpose(0, 1) else output
  
  
  // let rnn = RNN(32, 10, bidirectional=true, numLayers=2, bias=false, dropout=0.3)
  // let x = dsharp.randn([4; 16; 32])
  // print rnn.hidden.shape
  // let h = x --> rnn
  // print rnn.hidden.shape
  // print h.shape 
  // rnn.reset()
  
  let text = "A merry little surge of electricity piped by automatic alarm from the mood organ beside his bed awakened Rick Deckard."
  
  type TextTokenizer(?sampleText) = 
      let sampleText = defaultArg sampleText """0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;?@[\\]^_`{|}~ """
      let chars = sampleText |> Seq.distinct |> Seq.toArray
      member _.length = chars.Length
      member _.charToIndex(c) =
          let i = 
              try
                  Array.findIndex ((=) c) chars
              with
              | _ -> failwithf "Given char '%A' is not a part of this tokenizer %A" c chars
          i
      member t.textToIndices(text) = text |> Seq.map t.charToIndex |> Seq.toArray
      member t.indicesToTensor(indices) = indices |> Array.map (fun i -> dsharp.onehot(t.length, i)) |> dsharp.stack
      member t.textToTensor(text) = t.textToIndices(text) |> t.indicesToTensor
      member t.indexToChar(index) = chars.[index]
      member t.tensorToText(tensor:Tensor) =
          if tensor.dim <> 2 then failwithf "Expecting a 2d tensor with shape seqLen x features, received tensor with shape %A" tensor.shape
          [|for i in 0..tensor.shape.[0]-1 do tensor.[i].argmax().[0]|] |> Array.map t.indexToChar |> System.String
      member t.dataset(text:string, seqLength) =
          if seqLength > text.Length then failwithf "Expecting text.Length (%A) >= seqLength (%A)" text.Length seqLength
          let sequences = [|for i in 0..(text.Length - seqLength + 1)-1 do text.Substring(i, seqLength)|] |> Array.map t.textToIndices
          let data = sequences |> Array.map t.indicesToTensor |> dsharp.stack
          let target = sequences |> Array.map dsharp.tensor |> dsharp.stack
          TensorDataset(data, target)
  
  
  let seqLen = 32
  let tok = TextTokenizer()
  let dataset = tok.dataset(text, seqLen)
  let loader = dataset.loader(batchSize=4)
  
  let rnn = RNN(tok.length, tok.length, numLayers=1, batchFirst=true)
  print rnn
  let optimizer = Adam(rnn, lr=dsharp.tensor(0.001))
  
  let epochs = 5
  let start = System.DateTime.Now
  for epoch = 1 to epochs do
      for i, x, t in loader.epoch() do
          let input =  x.[*,..seqLen-2]
          let target = t.[*,1..]
          // printfn "input  %A" input.shape
          // printfn "target %A" target.shape
          rnn.reverseDiff()
          let output = input --> rnn
          // printfn "output %A" output.shape
          // printfn ""
          let loss = dsharp.crossEntropyLoss(output.transpose(1, 2), target)
          loss.reverse()
          optimizer.step()
          print loss
  
  
  // let mutable c = ["e"]
  // for i in 0..5 do
  //     let cc = c |> List.last |> tok.textToTensor |> dsharp.unsqueeze(1) |> rnn.forward |> dsharp.squeeze(1) |> tok.tensorToText
  //     c <- c@[cc]
  
  // print c