prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Plot the results in cifar10 example
  Hi, I ran the experiments (run_2 to run_10 in the cifar10 example) following the steps 3.1-3.6 and would like to plot the results and regenerate the figures you show. However, running this command as advised in step 3.3 to view the results does not work:
    cat ./workspaces/poc_workspace/server/run_2/cross_site_val/cross_site_val.json
  Could you advise please on how to regenerate the results after completing all experiments?
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `examples/cifar10/figs/plot_tensorboard_events.py`
  Content:
  ```
  1 # Copyright (c) 2021, NVIDIA CORPORATION.
  2 #
  3 # Licensed under the Apache License, Version 2.0 (the "License");
  4 # you may not use this file except in compliance with the License.
  5 # You may obtain a copy of the License at
  6 #
  7 #     http://www.apache.org/licenses/LICENSE-2.0
  8 #
  9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 
  15 import glob
  16 import json
  17 import os
  18 
  19 import matplotlib.pyplot as plt
  20 import pandas as pd
  21 import seaborn as sns
  22 import tensorflow as tf
  23 
  24 client_results_root = "./workspaces/secure_workspace/site-1"
  25 server_results_root = "./workspaces/secure_workspace/localhost"
  26 
  27 # 4.1 Central vs. FedAvg
  28 experiments = {
  29     "cifar10_central": {"run": "run_1", "tag": "val_acc_local_model"},
  30     "cifar10_fedavg": {"run": "run_2", "tag": "val_acc_global_model"},
  31     "cifar10_fedavg_he": {"run": "run_9", "tag": "val_acc_global_model"},
  32 }
  33 
  34 # # 4.2 Impact of client data heterogeneity
  35 # experiments = {"cifar10_fedavg (alpha=1.0)": {"run": "run_2", "tag": "val_acc_global_model"},
  36 #                 "cifar10_fedavg (alpha=0.5)": {"run": "run_3", "tag": "val_acc_global_model"},
  37 #                 "cifar10_fedavg (alpha=0.3)": {"run": "run_4", "tag": "val_acc_global_model"},
  38 #                 "cifar10_fedavg (alpha=0.1)": {"run": "run_5", "tag": "val_acc_global_model"}}
  39 #
  40 # # 4.3 FedProx vs. FedOpt vs. SCAFFOLD
  41 # experiments = {"cifar10_fedavg": {"run": "run_5", "tag": "val_acc_global_model"},
  42 #                "cifar10_fedprox": {"run": "run_6", "tag": "val_acc_global_model"},
  43 #                "cifar10_fedopt": {"run": "run_7", "tag": "val_acc_global_model"},
  44 #                "cifar10_scaffold": {"run": "run_8", "tag": "val_acc_global_model"}}
  45 
  46 add_cross_site_val = True
  47 
  48 
  49 def read_eventfile(filepath, tags=["val_acc_global_model"]):
  50     data = {}
  51     for summary in tf.compat.v1.train.summary_iterator(filepath):
  52         for v in summary.summary.value:
  53             if v.tag in tags:
  54                 # print(v.tag, summary.step, v.simple_value)
  55                 if v.tag in data.keys():
  56                     data[v.tag].append([summary.step, v.simple_value])
  57                 else:
  58                     data[v.tag] = [[summary.step, v.simple_value]]
  59     return data
  60 
  61 
  62 def add_eventdata(data, config, filepath, tag="val_acc_global_model"):
  63     event_data = read_eventfile(filepath, tags=[tag])
  64 
  65     assert len(event_data[tag]) > 0, f"No data for key {tag}"
  66     # print(event_data)
  67     for e in event_data[tag]:
  68         # print(e)
  69         data["Config"].append(config)
  70         data["Step"].append(e[0])
  71         data["Accuracy"].append(e[1])
  72     print(f"added {len(event_data[tag])} entries for {tag}")
  73 
  74 
  75 def main():
  76     data = {"Config": [], "Step": [], "Accuracy": []}
  77 
  78     if add_cross_site_val:
  79         xsite_keys = ["SRV_server", "SRV_server_best"]
  80         xsite_data = {"Config": []}
  81         for k in xsite_keys:
  82             xsite_data.update({k: []})
  83     else:
  84         xsite_data = None
  85         xsite_keys = None
  86 
  87     # add event files
  88     for config, exp in experiments.items():
  89         eventfile = glob.glob(os.path.join(client_results_root, exp["run"] + "/**/events.*"), recursive=True)
  90         assert len(eventfile) == 1, "No unique event file found!"
  91         eventfile = eventfile[0]
  92         print("adding", eventfile)
  93         add_eventdata(data, config, eventfile, tag=exp["tag"])
  94 
  95         if add_cross_site_val:
  96             xsite_file = glob.glob(
  97                 os.path.join(server_results_root, exp["run"] + "/**/cross_site_val.json"), recursive=True
  98             )
  99             assert len(xsite_file) == 1, "No unique x-site file found!"
  100             with open(xsite_file[0], "r") as f:
  101                 xsite_results = json.load(f)
  102 
  103             xsite_data["Config"].append(config)
  104             for k in xsite_keys:
  105                 xsite_data[k].append(xsite_results["site-1"][k]["val_accuracy"])
  106 
  107     print("Training TB data:")
  108     print(pd.DataFrame(data))
  109 
  110     if xsite_data:
  111         print("Cross-site val data:")
  112         print(pd.DataFrame(xsite_data))
  113 
  114     sns.lineplot(x="Step", y="Accuracy", hue="Config", data=data)
  115     plt.show()
  116 
  117 
  118 if __name__ == "__main__":
  119     main()
  120
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/examples/cifar10/figs/plot_tensorboard_events.py b/examples/cifar10/figs/plot_tensorboard_events.py
  --- a/examples/cifar10/figs/plot_tensorboard_events.py
  +++ b/examples/cifar10/figs/plot_tensorboard_events.py
  @@ -76,7 +76,7 @@
       data = {"Config": [], "Step": [], "Accuracy": []}
   
       if add_cross_site_val:
  -        xsite_keys = ["SRV_server", "SRV_server_best"]
  +        xsite_keys = ["SRV_FL_global_model.pt", "SRV_best_FL_global_model.pt"]
           xsite_data = {"Config": []}
           for k in xsite_keys:
               xsite_data.update({k: []})
  @@ -94,7 +94,7 @@
   
           if add_cross_site_val:
               xsite_file = glob.glob(
  -                os.path.join(server_results_root, exp["run"] + "/**/cross_site_val.json"), recursive=True
  +                os.path.join(server_results_root, exp["run"] + "/**/cross_val_results.json"), recursive=True
               )
               assert len(xsite_file) == 1, "No unique x-site file found!"
               with open(xsite_file[0], "r") as f:
  
verification_info: {"golden_diff": "diff --git a/examples/cifar10/figs/plot_tensorboard_events.py b/examples/cifar10/figs/plot_tensorboard_events.py\n--- a/examples/cifar10/figs/plot_tensorboard_events.py\n+++ b/examples/cifar10/figs/plot_tensorboard_events.py\n@@ -76,7 +76,7 @@\n     data = {\"Config\": [], \"Step\": [], \"Accuracy\": []}\n \n     if add_cross_site_val:\n-        xsite_keys = [\"SRV_server\", \"SRV_server_best\"]\n+        xsite_keys = [\"SRV_FL_global_model.pt\", \"SRV_best_FL_global_model.pt\"]\n         xsite_data = {\"Config\": []}\n         for k in xsite_keys:\n             xsite_data.update({k: []})\n@@ -94,7 +94,7 @@\n \n         if add_cross_site_val:\n             xsite_file = glob.glob(\n-                os.path.join(server_results_root, exp[\"run\"] + \"/**/cross_site_val.json\"), recursive=True\n+                os.path.join(server_results_root, exp[\"run\"] + \"/**/cross_val_results.json\"), recursive=True\n             )\n             assert len(xsite_file) == 1, \"No unique x-site file found!\"\n             with open(xsite_file[0], \"r\") as f:\n", "issue": "Plot the results in cifar10 example\nHi, I ran the experiments (run_2 to run_10 in the cifar10 example) following the steps 3.1-3.6 and would like to plot the results and regenerate the figures you show. However, running this command as advised in step 3.3 to view the results does not work:\r\n  cat ./workspaces/poc_workspace/server/run_2/cross_site_val/cross_site_val.json\r\nCould you advise please on how to regenerate the results after completing all experiments?\r\n\n", "before_files": [{"content": "# Copyright (c) 2021, NVIDIA CORPORATION.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport json\nimport os\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\n\nclient_results_root = \"./workspaces/secure_workspace/site-1\"\nserver_results_root = \"./workspaces/secure_workspace/localhost\"\n\n# 4.1 Central vs. FedAvg\nexperiments = {\n    \"cifar10_central\": {\"run\": \"run_1\", \"tag\": \"val_acc_local_model\"},\n    \"cifar10_fedavg\": {\"run\": \"run_2\", \"tag\": \"val_acc_global_model\"},\n    \"cifar10_fedavg_he\": {\"run\": \"run_9\", \"tag\": \"val_acc_global_model\"},\n}\n\n# # 4.2 Impact of client data heterogeneity\n# experiments = {\"cifar10_fedavg (alpha=1.0)\": {\"run\": \"run_2\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.5)\": {\"run\": \"run_3\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.3)\": {\"run\": \"run_4\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.1)\": {\"run\": \"run_5\", \"tag\": \"val_acc_global_model\"}}\n#\n# # 4.3 FedProx vs. FedOpt vs. SCAFFOLD\n# experiments = {\"cifar10_fedavg\": {\"run\": \"run_5\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_fedprox\": {\"run\": \"run_6\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_fedopt\": {\"run\": \"run_7\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_scaffold\": {\"run\": \"run_8\", \"tag\": \"val_acc_global_model\"}}\n\nadd_cross_site_val = True\n\n\ndef read_eventfile(filepath, tags=[\"val_acc_global_model\"]):\n    data = {}\n    for summary in tf.compat.v1.train.summary_iterator(filepath):\n        for v in summary.summary.value:\n            if v.tag in tags:\n                # print(v.tag, summary.step, v.simple_value)\n                if v.tag in data.keys():\n                    data[v.tag].append([summary.step, v.simple_value])\n                else:\n                    data[v.tag] = [[summary.step, v.simple_value]]\n    return data\n\n\ndef add_eventdata(data, config, filepath, tag=\"val_acc_global_model\"):\n    event_data = read_eventfile(filepath, tags=[tag])\n\n    assert len(event_data[tag]) > 0, f\"No data for key {tag}\"\n    # print(event_data)\n    for e in event_data[tag]:\n        # print(e)\n        data[\"Config\"].append(config)\n        data[\"Step\"].append(e[0])\n        data[\"Accuracy\"].append(e[1])\n    print(f\"added {len(event_data[tag])} entries for {tag}\")\n\n\ndef main():\n    data = {\"Config\": [], \"Step\": [], \"Accuracy\": []}\n\n    if add_cross_site_val:\n        xsite_keys = [\"SRV_server\", \"SRV_server_best\"]\n        xsite_data = {\"Config\": []}\n        for k in xsite_keys:\n            xsite_data.update({k: []})\n    else:\n        xsite_data = None\n        xsite_keys = None\n\n    # add event files\n    for config, exp in experiments.items():\n        eventfile = glob.glob(os.path.join(client_results_root, exp[\"run\"] + \"/**/events.*\"), recursive=True)\n        assert len(eventfile) == 1, \"No unique event file found!\"\n        eventfile = eventfile[0]\n        print(\"adding\", eventfile)\n        add_eventdata(data, config, eventfile, tag=exp[\"tag\"])\n\n        if add_cross_site_val:\n            xsite_file = glob.glob(\n                os.path.join(server_results_root, exp[\"run\"] + \"/**/cross_site_val.json\"), recursive=True\n            )\n            assert len(xsite_file) == 1, \"No unique x-site file found!\"\n            with open(xsite_file[0], \"r\") as f:\n                xsite_results = json.load(f)\n\n            xsite_data[\"Config\"].append(config)\n            for k in xsite_keys:\n                xsite_data[k].append(xsite_results[\"site-1\"][k][\"val_accuracy\"])\n\n    print(\"Training TB data:\")\n    print(pd.DataFrame(data))\n\n    if xsite_data:\n        print(\"Cross-site val data:\")\n        print(pd.DataFrame(xsite_data))\n\n    sns.lineplot(x=\"Step\", y=\"Accuracy\", hue=\"Config\", data=data)\n    plt.show()\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "examples/cifar10/figs/plot_tensorboard_events.py"}], "after_files": [{"content": "# Copyright (c) 2021, NVIDIA CORPORATION.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport json\nimport os\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\n\nclient_results_root = \"./workspaces/secure_workspace/site-1\"\nserver_results_root = \"./workspaces/secure_workspace/localhost\"\n\n# 4.1 Central vs. FedAvg\nexperiments = {\n    \"cifar10_central\": {\"run\": \"run_1\", \"tag\": \"val_acc_local_model\"},\n    \"cifar10_fedavg\": {\"run\": \"run_2\", \"tag\": \"val_acc_global_model\"},\n    \"cifar10_fedavg_he\": {\"run\": \"run_9\", \"tag\": \"val_acc_global_model\"},\n}\n\n# # 4.2 Impact of client data heterogeneity\n# experiments = {\"cifar10_fedavg (alpha=1.0)\": {\"run\": \"run_2\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.5)\": {\"run\": \"run_3\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.3)\": {\"run\": \"run_4\", \"tag\": \"val_acc_global_model\"},\n#                 \"cifar10_fedavg (alpha=0.1)\": {\"run\": \"run_5\", \"tag\": \"val_acc_global_model\"}}\n#\n# # 4.3 FedProx vs. FedOpt vs. SCAFFOLD\n# experiments = {\"cifar10_fedavg\": {\"run\": \"run_5\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_fedprox\": {\"run\": \"run_6\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_fedopt\": {\"run\": \"run_7\", \"tag\": \"val_acc_global_model\"},\n#                \"cifar10_scaffold\": {\"run\": \"run_8\", \"tag\": \"val_acc_global_model\"}}\n\nadd_cross_site_val = True\n\n\ndef read_eventfile(filepath, tags=[\"val_acc_global_model\"]):\n    data = {}\n    for summary in tf.compat.v1.train.summary_iterator(filepath):\n        for v in summary.summary.value:\n            if v.tag in tags:\n                # print(v.tag, summary.step, v.simple_value)\n                if v.tag in data.keys():\n                    data[v.tag].append([summary.step, v.simple_value])\n                else:\n                    data[v.tag] = [[summary.step, v.simple_value]]\n    return data\n\n\ndef add_eventdata(data, config, filepath, tag=\"val_acc_global_model\"):\n    event_data = read_eventfile(filepath, tags=[tag])\n\n    assert len(event_data[tag]) > 0, f\"No data for key {tag}\"\n    # print(event_data)\n    for e in event_data[tag]:\n        # print(e)\n        data[\"Config\"].append(config)\n        data[\"Step\"].append(e[0])\n        data[\"Accuracy\"].append(e[1])\n    print(f\"added {len(event_data[tag])} entries for {tag}\")\n\n\ndef main():\n    data = {\"Config\": [], \"Step\": [], \"Accuracy\": []}\n\n    if add_cross_site_val:\n        xsite_keys = [\"SRV_FL_global_model.pt\", \"SRV_best_FL_global_model.pt\"]\n        xsite_data = {\"Config\": []}\n        for k in xsite_keys:\n            xsite_data.update({k: []})\n    else:\n        xsite_data = None\n        xsite_keys = None\n\n    # add event files\n    for config, exp in experiments.items():\n        eventfile = glob.glob(os.path.join(client_results_root, exp[\"run\"] + \"/**/events.*\"), recursive=True)\n        assert len(eventfile) == 1, \"No unique event file found!\"\n        eventfile = eventfile[0]\n        print(\"adding\", eventfile)\n        add_eventdata(data, config, eventfile, tag=exp[\"tag\"])\n\n        if add_cross_site_val:\n            xsite_file = glob.glob(\n                os.path.join(server_results_root, exp[\"run\"] + \"/**/cross_val_results.json\"), recursive=True\n            )\n            assert len(xsite_file) == 1, \"No unique x-site file found!\"\n            with open(xsite_file[0], \"r\") as f:\n                xsite_results = json.load(f)\n\n            xsite_data[\"Config\"].append(config)\n            for k in xsite_keys:\n                xsite_data[k].append(xsite_results[\"site-1\"][k][\"val_accuracy\"])\n\n    print(\"Training TB data:\")\n    print(pd.DataFrame(data))\n\n    if xsite_data:\n        print(\"Cross-site val data:\")\n        print(pd.DataFrame(xsite_data))\n\n    sns.lineplot(x=\"Step\", y=\"Accuracy\", hue=\"Config\", data=data)\n    plt.show()\n\n\nif __name__ == \"__main__\":\n    main()\n", "path": "examples/cifar10/figs/plot_tensorboard_events.py"}]}