<!-- /**
 * @license
 * Copyright 2018 Google LLC. All Rights Reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =============================================================================
 */ -->
<html>

<head>
  <meta charset="utf-8">
  <link rel="stylesheet" href="./tufte.css" />
  <script type="module" src="index.js"></script>

  <style>
    .lang-javascript {
      width: 60%;
      padding-left: 15px;
      font-family: monospace;
    }

    script.show-script {
      display: block;
      max-width: 720px;
      background-color: floralwhite;
      font-family: "Lucida Console", Monaco, "Courier New", Courier, monospace;
      font-size: 12px;
      margin-left: -40px;
      white-space: pre;
      margin-top: 2em;
    }

    code {
      background-color: #eee;
      padding: 2px 4px 2px 4px;
    }
  </style>

  <style>
  </style>
</head>

<body>
  <article>
    <h1>Visualizing training with tfjs-vis</h1>

    <section>
      <p>
        tfjs-vis is a small set of visualization utilities to make it easier to understand what is going on with your
        tfjs models.
        It is designed in a way to work along side regular web apps. This page will use some of the features of tfjs to
        illustrate
        what is going on with a convolutional model that will be trained (in the browser) to recognize handwritten
        digits.
      </p>

      <p>
        tfjs-vis provides 2 main things:
        <ol>
          <li>A place to put visualizations that tries not to interfere with your web page. We call this place a
            <em>visor.</em>
          </li>
          <li>Some built in visualizations that we have found to be useful when working with TensorFlow.js
          </li>
        </ol>
      </p>
    </section>

    <section>
      <h2>The Visor</h2>
      <p>
        Let's take a look at the first. Calling <code>tfvis.visor()</code> will create a visor if it doesn't exist or
        return
        the existing one. Click the button below to show the
        <em>visor.</em>
      </p>
      <button id='show-visor'>Show Visor</button>
      <p>
        Notice the panel that is now displayed on the right. It hovers over your pages content and shouldn't disturb
        the flow of
        your page's DOM Elements. You can see a few controls for showing or hiding the visor, but by default it also
        supports
        the following keyboard shortcuts:
        <ul>
          <li>
            <strong>`</strong> (backtick): Shows or hides the visor</li>
          <li>
            <strong>~</strong> (tilde, shift+backtick): Toggles betweeen the two sizes the visor supports</li>
        </ul>
        The API allows you to disable (unbind) these keyboard shortcuts.
      </p>

      <h3>Surfaces</h3>
      <p>
        To add content to the visor we need a
        <em>surface</em>. We make a surface with the following function call:
        <script type='text' class='show-script'>
          tfvis.visor().surface({name: 'My First Surface', tab: 'Input Data'});
        </script>
      </p>
      <button id='make-first-surface'>Make a surface</button>
      <p>
        To create a surface we must give is a name, we can also optionally specify a tab name that the surface should
        be put on.
        <code>visor().surface()</code> allows us to create a surface if it doesn't exist or fetch it if it does. This
        API
        returns an object that has a pointer to 3 DOM elements:
        <ul>
          <li>
            <strong>container:</strong> The containing DOM element for the surface</li>
          <li>
            <strong>label:</strong> The label element </li>
          <li>
            <strong>drawArea:</strong> A DOM Element where we can render visualizations or other content.
          </li>
        </ul>
      </p>
    </section>

    <section>
      <h2>Our Data</h2>
      <p>
        We will use the MNIST database as our training set, it is comprised of a set of about 60k images of handwritten
        digits, all
        cropped to 28x28 px. Lets take a look at a few examples, we'll use the surface we created earlier.
      </p>
      <button id='load-data'>Load Data</button>
      &nbsp;&nbsp;&nbsp;
      <button id='show-examples' disabled>Show Example Digits</button>

      <p>
        The code to render these examples isn't built into tfjs. But because you have full access to the DOM element
        for each surface,
        you can draw whatever you would like into them. This allows easy integration of custom visualizations into the
        visor.

        <p>
          Here is the code for the "Show Example Digits" button above:
        </p>

        <script class='show-script'>
          async function showExamples() {

            // Get a surface
            const surface =
              tfvis.visor().surface({ name: 'My First Surface', tab: 'Input Data' });
            const drawArea = surface.drawArea;

            // Get the examples
            const examples = data.nextTestBatch(20);
            const numExamples = examples.xs.shape[0];
            for (let i = 0; i < numExamples; i++) {
              const imageTensor = tf.tidy(() => {
                return examples.xs.slice([i, 0], [1, examples.xs.shape[1]]).reshape([
                  28, 28, 1
                ]);
              });

              // Create a canvas element to render each example
              const canvas = document.createElement('canvas');
              canvas.width = 28;
              canvas.height = 28;
              canvas.style = 'margin: 4px;';
              await tf.browser.toPixels(imageTensor, canvas);
              drawArea.appendChild(canvas);

              imageTensor.dispose();
            }
          }

          document.querySelector('#show-examples')
            .addEventListener('click', async (e) => showExamples());
        </script>

    </section>

    <section>
      <h2>Training Our Model</h2>
      <p>
        Our goal is to train a model to recognize similar digits. We have already written a tutorial on how to do so.
        So in this
        article we are going to focus on monitoring that training and also look at how well our model performs.
      </p>

      <p>First let us define a helper function to do our training.</p>

      <script class='show-script'>
        async function train(model, data, fitCallbacks) {
          const BATCH_SIZE = 64;
          const trainDataSize = 500;
          const testDataSize = 100;

          const [trainXs, trainYs] = tf.tidy(() => {
            const d = data.nextTrainBatch(trainDataSize);
            return [
              d.xs.reshape([trainDataSize, 28, 28, 1]),
              d.labels
            ]
          });

          const [testXs, testYs] = tf.tidy(() => {
            const d = data.nextTestBatch(testDataSize);
            return [
              d.xs.reshape([testDataSize, 28, 28, 1]),
              d.labels
            ]
          });

          return model.fit(trainXs, trainYs, {
            batchSize: BATCH_SIZE,
            validationData: [testXs, testYs],
            epochs: 10,
            shuffle: true,
            callbacks: fitCallbacks
          });
        }
      </script>

      <p>
        We can use the <code>show.fitCallbacks</code> method to get functions that will plot the loss after every batch
        and
        epoch.
      </p>

      <button id='start-training-1' disabled>Start Training</button>

      <script class='show-script'>
        async function watchTraining() {
          const metrics = ['loss', 'val_loss', 'acc', 'val_acc'];
          const container = {
            name: 'show.fitCallbacks', tab: 'Training', styles: { height: '1000px' }
          };
          const callbacks = tfvis.show.fitCallbacks(container, metrics);
          return train(model, data, callbacks);
        }

        document.querySelector('#start-training-1')
          .addEventListener('click', () => watchTraining());
      </script>

      <p>
        Another option is to wait for the training to complete and render the loss curve when it is done.
      </p>

      <button id='start-training-2' disabled>Start Training</button>

      <script class='show-script'>
        async function showTrainingHistory() {
          const trainingHistory = await train(model, data);
          tfvis.show.history({ name: 'Training History', tab: 'Training' },
            trainingHistory, ['loss', 'val_loss', 'acc', 'val_acc']);
        }

        document.querySelector('#start-training-2')
          .addEventListener('click', () => showTrainingHistory());
      </script>

      <h3>Customizing training charts.</h3>
      <p>
        The <code>show.fitCallbacks</code> function is designed to help you quickly plot training behaviour
        with reasonable defaults. If you want to customize the rendering of these charts, you can use
        <code>render.linechart</code> function. An example that plots accuracy values at the end
        of every epoch using custom colors and a custom yaxis domain is shown below.
      </p>

      <button id='start-training-3' disabled>Custom Training Charts</button>

      <script class='show-script'>
        // An array to hold training logs
        const epochLogs = [];
        async function customTrainingCharts() {
          const callbacks = {
            onEpochEnd: function(epoch, log) {
              const surface = {
                name: 'Custom Training Charts',
                tab: 'Training',
              };
              const options = {
                xLabel: 'Epoch',
                yLabel: 'Value',
                yAxisDomain: [0, 1],
                seriesColors: ['teal', 'tomato']
              };

              // Prep the data
              epochLogs.push(log);
              const acc = epochLogs.map((log, i) => ({ x: i, y: log.acc }));
              const valAcc = epochLogs.map((log, i) => ({ x: i, y: log.val_acc }));
              const data = {
                values: [acc, valAcc],
                // Custom names for the series
                series: ['Accuracy', 'Validation Accuracy']
              }

              // render the chart
              tfvis.render.linechart(surface, data, options);
            }
          }
          return train(model, data, callbacks);
        }

        document.querySelector('#start-training-3')
          .addEventListener('click', () => customTrainingCharts());
      </script>

    </section>

    <section>
      <h2>Evaluating Our Model</h2>
      <p>
        Now that our model is trained we should evalute its performance. For a classification task like this one we can
        use the `perClassAccuracy`
        and `confusionMatrix` functions. These are demonstrated below.
      </p>

      <p><button id='show-accuracy'>Show per-class accuracy</button></p>
      <p><button id='show-confusion'>Show confusion matrix</button></p>

      <script class='show-script'>
        {
          const classNames = ['Zero', 'One', 'Two', 'Three', 'Four', 'Five',
            'Six', 'Seven', 'Eight', 'Nine'];

          function doPrediction(testDataSize = 500) {
            const testData = data.nextTestBatch(testDataSize);
            const testxs = testData.xs.reshape([testDataSize, 28, 28, 1]);
            const labels = testData.labels.argMax([-1]);
            const preds = model.predict(testxs).argMax([-1]);

            testxs.dispose();
            return [preds, labels];
          }

          async function showAccuracy() {
            const [preds, labels] = doPrediction();
            const classAccuracy = await tfvis.metrics.perClassAccuracy(labels, preds);
            const container = { name: 'Accuracy', tab: 'Evaluation' };
            tfvis.show.perClassAccuracy(container, classAccuracy, classNames);

            labels.dispose();
          }

          async function showConfusion() {
            const [preds, labels] = doPrediction();
            const confusionMatrix = await tfvis.metrics.confusionMatrix(labels, preds);
            const container = { name: 'Confusion Matrix', tab: 'Evaluation' };
            tfvis.render.confusionMatrix(container, {
              values: confusionMatrix,
              tickLabels: classNames,
            });

            labels.dispose();
          }

          document.querySelector('#show-accuracy')
            .addEventListener('click', () => showAccuracy());

          document.querySelector('#show-confusion')
            .addEventListener('click', () => showConfusion());
        }
      </script>


    </section>

  </article>




</body>

</html>
