prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Improve multi-objective optimization tutorial regarding `Study.best_trials`
  ### What is an issue?
  
  ## Background
  
  https://github.com/optuna/optuna/issues/3397 added a note about `study.best_trials` to the [reuse the best trial tutorial](https://optuna.readthedocs.io/en/latest/tutorial/20_recipes/010_reuse_best_trial.html). During the discussion, we found that the [multi-objective optimization tutorial](https://optuna.readthedocs.io/en/latest/tutorial/20_recipes/002_multi_objective.html) does not fetch the trials on the Pareto front (i.e., `Study.best_trials`) while it visualize the Pareto front.
  
  ## Description
  
  I think readers can understand how to use the trials on Pareto front more easily if we mention `Study.best_trials` in the tutorial page. We may have some possible scenarios. For example,
  
  1. Show the Pareto front as can be seen in the current tutorial
  2. Explain how to get the trials on the Pareto front by using `Study.best_trials`
  3. Demonstrate how to get the desired trial from the Pareto front i.e., the trial whose accuracy is highest, and show the parameter values (or objective values)
  
  ![image](https://user-images.githubusercontent.com/3255979/160827518-279a20e7-1c65-4fb7-958c-962e98902864.png)
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `tutorial/20_recipes/002_multi_objective.py`
  Content:
  ```
  1 """
  2 .. _multi_objective:
  3 
  4 Multi-objective Optimization with Optuna
  5 ========================================
  6 
  7 This tutorial showcases Optuna's multi-objective optimization feature by
  8 optimizing the validation accuracy of Fashion MNIST dataset and the FLOPS of the model implemented in PyTorch.
  9 
  10 We use `thop <https://github.com/Lyken17/pytorch-OpCounter>`_ to measure FLOPS.
  11 """
  12 
  13 import thop
  14 import torch
  15 import torch.nn as nn
  16 import torch.nn.functional as F
  17 import torchvision
  18 
  19 import optuna
  20 
  21 
  22 DEVICE = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
  23 DIR = ".."
  24 BATCHSIZE = 128
  25 N_TRAIN_EXAMPLES = BATCHSIZE * 30
  26 N_VALID_EXAMPLES = BATCHSIZE * 10
  27 
  28 
  29 def define_model(trial):
  30     n_layers = trial.suggest_int("n_layers", 1, 3)
  31     layers = []
  32 
  33     in_features = 28 * 28
  34     for i in range(n_layers):
  35         out_features = trial.suggest_int("n_units_l{}".format(i), 4, 128)
  36         layers.append(nn.Linear(in_features, out_features))
  37         layers.append(nn.ReLU())
  38         p = trial.suggest_float("dropout_{}".format(i), 0.2, 0.5)
  39         layers.append(nn.Dropout(p))
  40 
  41         in_features = out_features
  42 
  43     layers.append(nn.Linear(in_features, 10))
  44     layers.append(nn.LogSoftmax(dim=1))
  45 
  46     return nn.Sequential(*layers)
  47 
  48 
  49 # Defines training and evaluation.
  50 def train_model(model, optimizer, train_loader):
  51     model.train()
  52     for batch_idx, (data, target) in enumerate(train_loader):
  53         data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)
  54         optimizer.zero_grad()
  55         F.nll_loss(model(data), target).backward()
  56         optimizer.step()
  57 
  58 
  59 def eval_model(model, valid_loader):
  60     model.eval()
  61     correct = 0
  62     with torch.no_grad():
  63         for batch_idx, (data, target) in enumerate(valid_loader):
  64             data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)
  65             pred = model(data).argmax(dim=1, keepdim=True)
  66             correct += pred.eq(target.view_as(pred)).sum().item()
  67 
  68     accuracy = correct / N_VALID_EXAMPLES
  69 
  70     flops, _ = thop.profile(model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False)
  71     return flops, accuracy
  72 
  73 
  74 ###################################################################################################
  75 # Define multi-objective objective function.
  76 # Objectives are FLOPS and accuracy.
  77 def objective(trial):
  78     train_dataset = torchvision.datasets.FashionMNIST(
  79         DIR, train=True, download=True, transform=torchvision.transforms.ToTensor()
  80     )
  81     train_loader = torch.utils.data.DataLoader(
  82         torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),
  83         batch_size=BATCHSIZE,
  84         shuffle=True,
  85     )
  86 
  87     val_dataset = torchvision.datasets.FashionMNIST(
  88         DIR, train=False, transform=torchvision.transforms.ToTensor()
  89     )
  90     val_loader = torch.utils.data.DataLoader(
  91         torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),
  92         batch_size=BATCHSIZE,
  93         shuffle=True,
  94     )
  95     model = define_model(trial).to(DEVICE)
  96 
  97     optimizer = torch.optim.Adam(
  98         model.parameters(), trial.suggest_float("lr", 1e-5, 1e-1, log=True)
  99     )
  100 
  101     for epoch in range(10):
  102         train_model(model, optimizer, train_loader)
  103     flops, accuracy = eval_model(model, val_loader)
  104     return flops, accuracy
  105 
  106 
  107 ###################################################################################################
  108 # Run multi-objective optimization
  109 # --------------------------------
  110 #
  111 # If your optimization problem is multi-objective,
  112 # Optuna assumes that you will specify the optimization direction for each objective.
  113 # Specifically, in this example, we want to minimize the FLOPS (we want a faster model)
  114 # and maximize the accuracy. So we set ``directions`` to ``["minimize", "maximize"]``.
  115 study = optuna.create_study(directions=["minimize", "maximize"])
  116 study.optimize(objective, n_trials=30, timeout=300)
  117 
  118 print("Number of finished trials: ", len(study.trials))
  119 
  120 
  121 ###################################################################################################
  122 # Check trials on pareto front visually
  123 optuna.visualization.plot_pareto_front(study, target_names=["FLOPS", "accuracy"])
  124 
  125 
  126 ###################################################################################################
  127 # Learn which hyperparameters are affecting the flops most with hyperparameter importance.
  128 optuna.visualization.plot_param_importances(
  129     study, target=lambda t: t.values[0], target_name="flops"
  130 )
  131
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/tutorial/20_recipes/002_multi_objective.py b/tutorial/20_recipes/002_multi_objective.py
  --- a/tutorial/20_recipes/002_multi_objective.py
  +++ b/tutorial/20_recipes/002_multi_objective.py
  @@ -119,10 +119,23 @@
   
   
   ###################################################################################################
  -# Check trials on pareto front visually
  +# Check trials on Pareto front visually.
   optuna.visualization.plot_pareto_front(study, target_names=["FLOPS", "accuracy"])
   
   
  +###################################################################################################
  +# Fetch the list of trials on the Pareto front with :attr:`~optuna.study.Study.best_trials`.
  +#
  +# For example, the following code shows the number of trials on the Pareto front and picks the trial with the highest accuracy.
  +
  +print(f"Number of trials on the Pareto front: {len(study.best_trials)}")
  +
  +trial_with_highest_accuracy = max(study.best_trials, key=lambda t: t.values[1])
  +print(f"Trial with highest accuracy: ")
  +print(f"\tnumber: {trial_with_highest_accuracy.number}")
  +print(f"\tparams: {trial_with_highest_accuracy.params}")
  +print(f"\tvalues: {trial_with_highest_accuracy.values}")
  +
   ###################################################################################################
   # Learn which hyperparameters are affecting the flops most with hyperparameter importance.
   optuna.visualization.plot_param_importances(
  
verification_info: {"golden_diff": "diff --git a/tutorial/20_recipes/002_multi_objective.py b/tutorial/20_recipes/002_multi_objective.py\n--- a/tutorial/20_recipes/002_multi_objective.py\n+++ b/tutorial/20_recipes/002_multi_objective.py\n@@ -119,10 +119,23 @@\n \n \n ###################################################################################################\n-# Check trials on pareto front visually\n+# Check trials on Pareto front visually.\n optuna.visualization.plot_pareto_front(study, target_names=[\"FLOPS\", \"accuracy\"])\n \n \n+###################################################################################################\n+# Fetch the list of trials on the Pareto front with :attr:`~optuna.study.Study.best_trials`.\n+#\n+# For example, the following code shows the number of trials on the Pareto front and picks the trial with the highest accuracy.\n+\n+print(f\"Number of trials on the Pareto front: {len(study.best_trials)}\")\n+\n+trial_with_highest_accuracy = max(study.best_trials, key=lambda t: t.values[1])\n+print(f\"Trial with highest accuracy: \")\n+print(f\"\\tnumber: {trial_with_highest_accuracy.number}\")\n+print(f\"\\tparams: {trial_with_highest_accuracy.params}\")\n+print(f\"\\tvalues: {trial_with_highest_accuracy.values}\")\n+\n ###################################################################################################\n # Learn which hyperparameters are affecting the flops most with hyperparameter importance.\n optuna.visualization.plot_param_importances(\n", "issue": "Improve multi-objective optimization tutorial regarding `Study.best_trials`\n### What is an issue?\n\n## Background\r\n\r\nhttps://github.com/optuna/optuna/issues/3397 added a note about `study.best_trials` to the [reuse the best trial tutorial](https://optuna.readthedocs.io/en/latest/tutorial/20_recipes/010_reuse_best_trial.html). During the discussion, we found that the [multi-objective optimization tutorial](https://optuna.readthedocs.io/en/latest/tutorial/20_recipes/002_multi_objective.html) does not fetch the trials on the Pareto front (i.e., `Study.best_trials`) while it visualize the Pareto front.\r\n\r\n## Description\r\n\r\nI think readers can understand how to use the trials on Pareto front more easily if we mention `Study.best_trials` in the tutorial page. We may have some possible scenarios. For example,\r\n\r\n1. Show the Pareto front as can be seen in the current tutorial\r\n2. Explain how to get the trials on the Pareto front by using `Study.best_trials`\r\n3. Demonstrate how to get the desired trial from the Pareto front i.e., the trial whose accuracy is highest, and show the parameter values (or objective values)\r\n\r\n![image](https://user-images.githubusercontent.com/3255979/160827518-279a20e7-1c65-4fb7-958c-962e98902864.png)\r\n\n", "before_files": [{"content": "\"\"\"\n.. _multi_objective:\n\nMulti-objective Optimization with Optuna\n========================================\n\nThis tutorial showcases Optuna's multi-objective optimization feature by\noptimizing the validation accuracy of Fashion MNIST dataset and the FLOPS of the model implemented in PyTorch.\n\nWe use `thop <https://github.com/Lyken17/pytorch-OpCounter>`_ to measure FLOPS.\n\"\"\"\n\nimport thop\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nimport optuna\n\n\nDEVICE = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\nDIR = \"..\"\nBATCHSIZE = 128\nN_TRAIN_EXAMPLES = BATCHSIZE * 30\nN_VALID_EXAMPLES = BATCHSIZE * 10\n\n\ndef define_model(trial):\n    n_layers = trial.suggest_int(\"n_layers\", 1, 3)\n    layers = []\n\n    in_features = 28 * 28\n    for i in range(n_layers):\n        out_features = trial.suggest_int(\"n_units_l{}\".format(i), 4, 128)\n        layers.append(nn.Linear(in_features, out_features))\n        layers.append(nn.ReLU())\n        p = trial.suggest_float(\"dropout_{}\".format(i), 0.2, 0.5)\n        layers.append(nn.Dropout(p))\n\n        in_features = out_features\n\n    layers.append(nn.Linear(in_features, 10))\n    layers.append(nn.LogSoftmax(dim=1))\n\n    return nn.Sequential(*layers)\n\n\n# Defines training and evaluation.\ndef train_model(model, optimizer, train_loader):\n    model.train()\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n        optimizer.zero_grad()\n        F.nll_loss(model(data), target).backward()\n        optimizer.step()\n\n\ndef eval_model(model, valid_loader):\n    model.eval()\n    correct = 0\n    with torch.no_grad():\n        for batch_idx, (data, target) in enumerate(valid_loader):\n            data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n            pred = model(data).argmax(dim=1, keepdim=True)\n            correct += pred.eq(target.view_as(pred)).sum().item()\n\n    accuracy = correct / N_VALID_EXAMPLES\n\n    flops, _ = thop.profile(model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False)\n    return flops, accuracy\n\n\n###################################################################################################\n# Define multi-objective objective function.\n# Objectives are FLOPS and accuracy.\ndef objective(trial):\n    train_dataset = torchvision.datasets.FashionMNIST(\n        DIR, train=True, download=True, transform=torchvision.transforms.ToTensor()\n    )\n    train_loader = torch.utils.data.DataLoader(\n        torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),\n        batch_size=BATCHSIZE,\n        shuffle=True,\n    )\n\n    val_dataset = torchvision.datasets.FashionMNIST(\n        DIR, train=False, transform=torchvision.transforms.ToTensor()\n    )\n    val_loader = torch.utils.data.DataLoader(\n        torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),\n        batch_size=BATCHSIZE,\n        shuffle=True,\n    )\n    model = define_model(trial).to(DEVICE)\n\n    optimizer = torch.optim.Adam(\n        model.parameters(), trial.suggest_float(\"lr\", 1e-5, 1e-1, log=True)\n    )\n\n    for epoch in range(10):\n        train_model(model, optimizer, train_loader)\n    flops, accuracy = eval_model(model, val_loader)\n    return flops, accuracy\n\n\n###################################################################################################\n# Run multi-objective optimization\n# --------------------------------\n#\n# If your optimization problem is multi-objective,\n# Optuna assumes that you will specify the optimization direction for each objective.\n# Specifically, in this example, we want to minimize the FLOPS (we want a faster model)\n# and maximize the accuracy. So we set ``directions`` to ``[\"minimize\", \"maximize\"]``.\nstudy = optuna.create_study(directions=[\"minimize\", \"maximize\"])\nstudy.optimize(objective, n_trials=30, timeout=300)\n\nprint(\"Number of finished trials: \", len(study.trials))\n\n\n###################################################################################################\n# Check trials on pareto front visually\noptuna.visualization.plot_pareto_front(study, target_names=[\"FLOPS\", \"accuracy\"])\n\n\n###################################################################################################\n# Learn which hyperparameters are affecting the flops most with hyperparameter importance.\noptuna.visualization.plot_param_importances(\n    study, target=lambda t: t.values[0], target_name=\"flops\"\n)\n", "path": "tutorial/20_recipes/002_multi_objective.py"}], "after_files": [{"content": "\"\"\"\n.. _multi_objective:\n\nMulti-objective Optimization with Optuna\n========================================\n\nThis tutorial showcases Optuna's multi-objective optimization feature by\noptimizing the validation accuracy of Fashion MNIST dataset and the FLOPS of the model implemented in PyTorch.\n\nWe use `thop <https://github.com/Lyken17/pytorch-OpCounter>`_ to measure FLOPS.\n\"\"\"\n\nimport thop\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nimport optuna\n\n\nDEVICE = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\nDIR = \"..\"\nBATCHSIZE = 128\nN_TRAIN_EXAMPLES = BATCHSIZE * 30\nN_VALID_EXAMPLES = BATCHSIZE * 10\n\n\ndef define_model(trial):\n    n_layers = trial.suggest_int(\"n_layers\", 1, 3)\n    layers = []\n\n    in_features = 28 * 28\n    for i in range(n_layers):\n        out_features = trial.suggest_int(\"n_units_l{}\".format(i), 4, 128)\n        layers.append(nn.Linear(in_features, out_features))\n        layers.append(nn.ReLU())\n        p = trial.suggest_float(\"dropout_{}\".format(i), 0.2, 0.5)\n        layers.append(nn.Dropout(p))\n\n        in_features = out_features\n\n    layers.append(nn.Linear(in_features, 10))\n    layers.append(nn.LogSoftmax(dim=1))\n\n    return nn.Sequential(*layers)\n\n\n# Defines training and evaluation.\ndef train_model(model, optimizer, train_loader):\n    model.train()\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n        optimizer.zero_grad()\n        F.nll_loss(model(data), target).backward()\n        optimizer.step()\n\n\ndef eval_model(model, valid_loader):\n    model.eval()\n    correct = 0\n    with torch.no_grad():\n        for batch_idx, (data, target) in enumerate(valid_loader):\n            data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n            pred = model(data).argmax(dim=1, keepdim=True)\n            correct += pred.eq(target.view_as(pred)).sum().item()\n\n    accuracy = correct / N_VALID_EXAMPLES\n\n    flops, _ = thop.profile(model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False)\n    return flops, accuracy\n\n\n###################################################################################################\n# Define multi-objective objective function.\n# Objectives are FLOPS and accuracy.\ndef objective(trial):\n    train_dataset = torchvision.datasets.FashionMNIST(\n        DIR, train=True, download=True, transform=torchvision.transforms.ToTensor()\n    )\n    train_loader = torch.utils.data.DataLoader(\n        torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),\n        batch_size=BATCHSIZE,\n        shuffle=True,\n    )\n\n    val_dataset = torchvision.datasets.FashionMNIST(\n        DIR, train=False, transform=torchvision.transforms.ToTensor()\n    )\n    val_loader = torch.utils.data.DataLoader(\n        torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),\n        batch_size=BATCHSIZE,\n        shuffle=True,\n    )\n    model = define_model(trial).to(DEVICE)\n\n    optimizer = torch.optim.Adam(\n        model.parameters(), trial.suggest_float(\"lr\", 1e-5, 1e-1, log=True)\n    )\n\n    for epoch in range(10):\n        train_model(model, optimizer, train_loader)\n    flops, accuracy = eval_model(model, val_loader)\n    return flops, accuracy\n\n\n###################################################################################################\n# Run multi-objective optimization\n# --------------------------------\n#\n# If your optimization problem is multi-objective,\n# Optuna assumes that you will specify the optimization direction for each objective.\n# Specifically, in this example, we want to minimize the FLOPS (we want a faster model)\n# and maximize the accuracy. So we set ``directions`` to ``[\"minimize\", \"maximize\"]``.\nstudy = optuna.create_study(directions=[\"minimize\", \"maximize\"])\nstudy.optimize(objective, n_trials=30, timeout=300)\n\nprint(\"Number of finished trials: \", len(study.trials))\n\n\n###################################################################################################\n# Check trials on Pareto front visually.\noptuna.visualization.plot_pareto_front(study, target_names=[\"FLOPS\", \"accuracy\"])\n\n\n###################################################################################################\n# Fetch the list of trials on the Pareto front with :attr:`~optuna.study.Study.best_trials`.\n#\n# For example, the following code shows the number of trials on the Pareto front and picks the trial with the highest accuracy.\n\nprint(f\"Number of trials on the Pareto front: {len(study.best_trials)}\")\n\ntrial_with_highest_accuracy = max(study.best_trials, key=lambda t: t.values[1])\nprint(f\"Trial with highest accuracy: \")\nprint(f\"\\tnumber: {trial_with_highest_accuracy.number}\")\nprint(f\"\\tparams: {trial_with_highest_accuracy.params}\")\nprint(f\"\\tvalues: {trial_with_highest_accuracy.values}\")\n\n###################################################################################################\n# Learn which hyperparameters are affecting the flops most with hyperparameter importance.\noptuna.visualization.plot_param_importances(\n    study, target=lambda t: t.values[0], target_name=\"flops\"\n)\n", "path": "tutorial/20_recipes/002_multi_objective.py"}]}