We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Convert build system to Bazel.
It seems like using Bazel (https://github.com/bazelbuild/bazel) would improve our build system in a lot of ways.

@rshin did an early version of this a long time ago in https://github.com/ray-project/ray-legacy/pull/408. @ericl, @concretevitamin, @pcmoritz have all used Bazel I believe and are proponents. @rsepassi and @dbieber have recently worked on this and have a good sense of what is involved.

Any thoughts about this? Any drawbacks to be aware of?

cc @chuxi @guoyuhong @raulchen

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `python/setup.py`
Content:
```
1 from __future__ import absolute_import
2 from __future__ import division
3 from __future__ import print_function
4 
5 import os
6 import re
7 import shutil
8 import subprocess
9 import sys
10 
11 from setuptools import setup, find_packages, Distribution
12 import setuptools.command.build_ext as _build_ext
13 
14 # Ideally, we could include these files by putting them in a
15 # MANIFEST.in or using the package_data argument to setup, but the
16 # MANIFEST.in gets applied at the very beginning when setup.py runs
17 # before these files have been created, so we have to move the files
18 # manually.
19 
20 # NOTE: The lists below must be kept in sync with ray/CMakeLists.txt.
21 
22 ray_files = [
23     "ray/core/src/ray/thirdparty/redis/src/redis-server",
24     "ray/core/src/ray/gcs/redis_module/libray_redis_module.so",
25     "ray/core/src/plasma/plasma_store_server", "ray/_raylet.so",
26     "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet",
27     "ray/dashboard/dashboard.py", "ray/dashboard/index.html",
28     "ray/dashboard/res/main.css", "ray/dashboard/res/main.js"
29 ]
30 
31 # These are the directories where automatically generated Python flatbuffer
32 # bindings are created.
33 generated_python_directories = [
34     "ray/core/generated", "ray/core/generated/ray",
35     "ray/core/generated/ray/protocol"
36 ]
37 
38 optional_ray_files = []
39 
40 ray_autoscaler_files = [
41     "ray/autoscaler/aws/example-full.yaml",
42     "ray/autoscaler/gcp/example-full.yaml",
43     "ray/autoscaler/local/example-full.yaml",
44 ]
45 
46 if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on":
47     ray_files += [
48         "ray/core/src/credis/build/src/libmember.so",
49         "ray/core/src/credis/build/src/libmaster.so",
50         "ray/core/src/credis/redis/src/redis-server"
51     ]
52 
53 optional_ray_files += ray_autoscaler_files
54 
55 extras = {
56     "rllib": [
57         "pyyaml", "gym[atari]", "opencv-python-headless", "lz4", "scipy"
58     ],
59     "debug": ["psutil", "setproctitle", "py-spy"],
60     "dashboard": ["psutil", "aiohttp"],
61 }
62 
63 
64 class build_ext(_build_ext.build_ext):
65     def run(self):
66         # Note: We are passing in sys.executable so that we use the same
67         # version of Python to build pyarrow inside the build.sh script. Note
68         # that certain flags will not be passed along such as --user or sudo.
69         # TODO(rkn): Fix this.
70         command = ["../build.sh", "-p", sys.executable]
71         if os.getenv("RAY_INSTALL_JAVA") == "1":
72             # Also build binaries for Java if the above env variable exists.
73             command += ["-l", "python,java"]
74         subprocess.check_call(command)
75 
76         # We also need to install pyarrow along with Ray, so make sure that the
77         # relevant non-Python pyarrow files get copied.
78         pyarrow_files = []
79         for (root, dirs, filenames) in os.walk("./ray/pyarrow_files/pyarrow"):
80             for name in filenames:
81                 pyarrow_files.append(os.path.join(root, name))
82 
83         # Make sure the relevant files for modin get copied.
84         modin_files = []
85         for (root, dirs, filenames) in os.walk("./ray/modin"):
86             for name in filenames:
87                 modin_files.append(os.path.join(root, name))
88 
89         files_to_include = ray_files + pyarrow_files + modin_files
90 
91         # Copy over the autogenerated flatbuffer Python bindings.
92         for directory in generated_python_directories:
93             for filename in os.listdir(directory):
94                 if filename[-3:] == ".py":
95                     files_to_include.append(os.path.join(directory, filename))
96 
97         for filename in files_to_include:
98             self.move_file(filename)
99 
100         # Try to copy over the optional files.
101         for filename in optional_ray_files:
102             try:
103                 self.move_file(filename)
104             except Exception:
105                 print("Failed to copy optional file {}. This is ok."
106                       .format(filename))
107 
108     def move_file(self, filename):
109         # TODO(rkn): This feels very brittle. It may not handle all cases. See
110         # https://github.com/apache/arrow/blob/master/python/setup.py for an
111         # example.
112         source = filename
113         destination = os.path.join(self.build_lib, filename)
114         # Create the target directory if it doesn't already exist.
115         parent_directory = os.path.dirname(destination)
116         if not os.path.exists(parent_directory):
117             os.makedirs(parent_directory)
118         if not os.path.exists(destination):
119             print("Copying {} to {}.".format(source, destination))
120             shutil.copy(source, destination)
121 
122 
123 class BinaryDistribution(Distribution):
124     def has_ext_modules(self):
125         return True
126 
127 
128 def find_version(*filepath):
129     # Extract version information from filepath
130     here = os.path.abspath(os.path.dirname(__file__))
131     with open(os.path.join(here, *filepath)) as fp:
132         version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
133                                   fp.read(), re.M)
134         if version_match:
135             return version_match.group(1)
136         raise RuntimeError("Unable to find version string.")
137 
138 
139 requires = [
140     "numpy >= 1.14",
141     "filelock",
142     "funcsigs",
143     "click",
144     "colorama",
145     "pytest",
146     "pyyaml",
147     "redis",
148     # NOTE: Don't upgrade the version of six! Doing so causes installation
149     # problems. See https://github.com/ray-project/ray/issues/4169.
150     "six >= 1.0.0",
151     # The typing module is required by modin.
152     "typing",
153     "flatbuffers",
154     "faulthandler;python_version<'3.3'",
155 ]
156 
157 setup(
158     name="ray",
159     version=find_version("ray", "__init__.py"),
160     author="Ray Team",
161     author_email="ray-dev@googlegroups.com",
162     description=("A system for parallel and distributed Python that unifies "
163                  "the ML ecosystem."),
164     long_description=open("../README.rst").read(),
165     url="https://github.com/ray-project/ray",
166     keywords=("ray distributed parallel machine-learning "
167               "reinforcement-learning deep-learning python"),
168     packages=find_packages(),
169     cmdclass={"build_ext": build_ext},
170     # The BinaryDistribution argument triggers build_ext.
171     distclass=BinaryDistribution,
172     install_requires=requires,
173     setup_requires=["cython >= 0.29"],
174     extras_require=extras,
175     entry_points={
176         "console_scripts": [
177             "ray=ray.scripts.scripts:main",
178             "rllib=ray.rllib.scripts:cli [rllib]", "tune=ray.tune.scripts:cli"
179         ]
180     },
181     include_package_data=True,
182     zip_safe=False,
183     license="Apache 2.0")
184
```

Path: `python/ray/autoscaler/docker.py`
Content:
```
1 from __future__ import absolute_import
2 from __future__ import division
3 from __future__ import print_function
4 
5 import os
6 import logging
7 try:  # py3
8     from shlex import quote
9 except ImportError:  # py2
10     from pipes import quote
11 
12 logger = logging.getLogger(__name__)
13 
14 
15 def dockerize_if_needed(config):
16     if "docker" not in config:
17         return config
18     docker_image = config["docker"].get("image")
19     cname = config["docker"].get("container_name")
20     run_options = config["docker"].get("run_options", [])
21     ssh_user = config["auth"]["ssh_user"]
22     if not docker_image:
23         if cname:
24             logger.warning(
25                 "dockerize_if_needed: "
26                 "Container name given but no Docker image - continuing...")
27         return config
28     else:
29         assert cname, "Must provide container name!"
30     docker_mounts = {dst: dst for dst in config["file_mounts"]}
31 
32     config["setup_commands"] = (
33         docker_start_cmds(ssh_user, docker_image, docker_mounts, cname,
34                           run_options) + with_docker_exec(
35                               config["setup_commands"], container_name=cname))
36 
37     config["head_setup_commands"] = with_docker_exec(
38         config["head_setup_commands"], container_name=cname)
39     config["head_start_ray_commands"] = (
40         docker_autoscaler_setup(cname) + with_docker_exec(
41             config["head_start_ray_commands"], container_name=cname))
42 
43     config["worker_setup_commands"] = with_docker_exec(
44         config["worker_setup_commands"], container_name=cname)
45     config["worker_start_ray_commands"] = with_docker_exec(
46         config["worker_start_ray_commands"],
47         container_name=cname,
48         env_vars=["RAY_HEAD_IP"])
49 
50     return config
51 
52 
53 def with_docker_exec(cmds, container_name, env_vars=None):
54     env_str = ""
55     if env_vars:
56         env_str = " ".join(
57             ["-e {env}=${env}".format(env=env) for env in env_vars])
58     return [
59         "docker exec {} {} /bin/sh -c {} ".format(env_str, container_name,
60                                                   quote(cmd)) for cmd in cmds
61     ]
62 
63 
64 def aptwait_cmd():
65     return ("while sudo fuser"
66             " /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock"
67             " >/dev/null 2>&1; "
68             "do echo 'Waiting for release of dpkg/apt locks'; sleep 5; done")
69 
70 
71 def docker_start_cmds(user, image, mount, cname, user_options):
72     cmds = []
73 
74     # create flags
75     # ports for the redis, object manager, and tune client
76     port_flags = " ".join([
77         "-p {port}:{port}".format(port=port)
78         for port in ["6379", "8076", "4321"]
79     ])
80     mount_flags = " ".join(
81         ["-v {src}:{dest}".format(src=k, dest=v) for k, v in mount.items()])
82 
83     # for click, used in ray cli
84     env_vars = {"LC_ALL": "C.UTF-8", "LANG": "C.UTF-8"}
85     env_flags = " ".join(
86         ["-e {name}={val}".format(name=k, val=v) for k, v in env_vars.items()])
87 
88     user_options_str = " ".join(user_options)
89     # docker run command
90     docker_check = [
91         "docker", "inspect", "-f", "'{{.State.Running}}'", cname, "||"
92     ]
93     docker_run = [
94         "docker", "run", "--rm", "--name {}".format(cname), "-d", "-it",
95         port_flags, mount_flags, env_flags, user_options_str, "--net=host",
96         image, "bash"
97     ]
98     cmds.append(" ".join(docker_check + docker_run))
99     docker_update = [
100         " && ".join(("apt-get -y update", "apt-get -y upgrade",
101                      "apt-get install -y git wget cmake psmisc"))
102     ]
103     cmds.extend(with_docker_exec(docker_update, container_name=cname))
104     return cmds
105 
106 
107 def docker_autoscaler_setup(cname):
108     cmds = []
109     for path in ["~/ray_bootstrap_config.yaml", "~/ray_bootstrap_key.pem"]:
110         # needed because docker doesn't allow relative paths
111         base_path = os.path.basename(path)
112         cmds.append("docker cp {path} {cname}:{dpath}".format(
113             path=path, dpath=base_path, cname=cname))
114         cmds.extend(
115             with_docker_exec(
116                 ["cp {} {}".format("/" + base_path, path)],
117                 container_name=cname))
118     return cmds
119
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/python/ray/autoscaler/docker.py b/python/ray/autoscaler/docker.py
--- a/python/ray/autoscaler/docker.py
+++ b/python/ray/autoscaler/docker.py
@@ -98,7 +98,7 @@
     cmds.append(" ".join(docker_check + docker_run))
     docker_update = [
         " && ".join(("apt-get -y update", "apt-get -y upgrade",
-                     "apt-get install -y git wget cmake psmisc"))
+                     "apt-get install -y git wget psmisc"))
     ]
     cmds.extend(with_docker_exec(docker_update, container_name=cname))
     return cmds
diff --git a/python/setup.py b/python/setup.py
--- a/python/setup.py
+++ b/python/setup.py
@@ -17,7 +17,7 @@
 # before these files have been created, so we have to move the files
 # manually.
 
-# NOTE: The lists below must be kept in sync with ray/CMakeLists.txt.
+# NOTE: The lists below must be kept in sync with ray/BUILD.bazel.
 
 ray_files = [
     "ray/core/src/ray/thirdparty/redis/src/redis-server",

{"golden_diff": "diff --git a/python/ray/autoscaler/docker.py b/python/ray/autoscaler/docker.py\n--- a/python/ray/autoscaler/docker.py\n+++ b/python/ray/autoscaler/docker.py\n@@ -98,7 +98,7 @@\n     cmds.append(\" \".join(docker_check + docker_run))\n     docker_update = [\n         \" && \".join((\"apt-get -y update\", \"apt-get -y upgrade\",\n-                     \"apt-get install -y git wget cmake psmisc\"))\n+                     \"apt-get install -y git wget psmisc\"))\n     ]\n     cmds.extend(with_docker_exec(docker_update, container_name=cname))\n     return cmds\ndiff --git a/python/setup.py b/python/setup.py\n--- a/python/setup.py\n+++ b/python/setup.py\n@@ -17,7 +17,7 @@\n # before these files have been created, so we have to move the files\n # manually.\n \n-# NOTE: The lists below must be kept in sync with ray/CMakeLists.txt.\n+# NOTE: The lists below must be kept in sync with ray/BUILD.bazel.\n \n ray_files = [\n     \"ray/core/src/ray/thirdparty/redis/src/redis-server\",\n", "issue": "Convert build system to Bazel.\nIt seems like using Bazel (https://github.com/bazelbuild/bazel) would improve our build system in a lot of ways.\r\n\r\n@rshin did an early version of this a long time ago in https://github.com/ray-project/ray-legacy/pull/408. @ericl, @concretevitamin, @pcmoritz have all used Bazel I believe and are proponents. @rsepassi and @dbieber have recently worked on this and have a good sense of what is involved.\r\n\r\nAny thoughts about this? Any drawbacks to be aware of?\r\n\r\ncc @chuxi @guoyuhong @raulchen\n", "before_files": [{"content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os\nimport re\nimport shutil\nimport subprocess\nimport sys\n\nfrom setuptools import setup, find_packages, Distribution\nimport setuptools.command.build_ext as _build_ext\n\n# Ideally, we could include these files by putting them in a\n# MANIFEST.in or using the package_data argument to setup, but the\n# MANIFEST.in gets applied at the very beginning when setup.py runs\n# before these files have been created, so we have to move the files\n# manually.\n\n# NOTE: The lists below must be kept in sync with ray/CMakeLists.txt.\n\nray_files = [\n    \"ray/core/src/ray/thirdparty/redis/src/redis-server\",\n    \"ray/core/src/ray/gcs/redis_module/libray_redis_module.so\",\n    \"ray/core/src/plasma/plasma_store_server\", \"ray/_raylet.so\",\n    \"ray/core/src/ray/raylet/raylet_monitor\", \"ray/core/src/ray/raylet/raylet\",\n    \"ray/dashboard/dashboard.py\", \"ray/dashboard/index.html\",\n    \"ray/dashboard/res/main.css\", \"ray/dashboard/res/main.js\"\n]\n\n# These are the directories where automatically generated Python flatbuffer\n# bindings are created.\ngenerated_python_directories = [\n    \"ray/core/generated\", \"ray/core/generated/ray\",\n    \"ray/core/generated/ray/protocol\"\n]\n\noptional_ray_files = []\n\nray_autoscaler_files = [\n    \"ray/autoscaler/aws/example-full.yaml\",\n    \"ray/autoscaler/gcp/example-full.yaml\",\n    \"ray/autoscaler/local/example-full.yaml\",\n]\n\nif \"RAY_USE_NEW_GCS\" in os.environ and os.environ[\"RAY_USE_NEW_GCS\"] == \"on\":\n    ray_files += [\n        \"ray/core/src/credis/build/src/libmember.so\",\n        \"ray/core/src/credis/build/src/libmaster.so\",\n        \"ray/core/src/credis/redis/src/redis-server\"\n    ]\n\noptional_ray_files += ray_autoscaler_files\n\nextras = {\n    \"rllib\": [\n        \"pyyaml\", \"gym[atari]\", \"opencv-python-headless\", \"lz4\", \"scipy\"\n    ],\n    \"debug\": [\"psutil\", \"setproctitle\", \"py-spy\"],\n    \"dashboard\": [\"psutil\", \"aiohttp\"],\n}\n\n\nclass build_ext(_build_ext.build_ext):\n    def run(self):\n        # Note: We are passing in sys.executable so that we use the same\n        # version of Python to build pyarrow inside the build.sh script. Note\n        # that certain flags will not be passed along such as --user or sudo.\n        # TODO(rkn): Fix this.\n        command = [\"../build.sh\", \"-p\", sys.executable]\n        if os.getenv(\"RAY_INSTALL_JAVA\") == \"1\":\n            # Also build binaries for Java if the above env variable exists.\n            command += [\"-l\", \"python,java\"]\n        subprocess.check_call(command)\n\n        # We also need to install pyarrow along with Ray, so make sure that the\n        # relevant non-Python pyarrow files get copied.\n        pyarrow_files = []\n        for (root, dirs, filenames) in os.walk(\"./ray/pyarrow_files/pyarrow\"):\n            for name in filenames:\n                pyarrow_files.append(os.path.join(root, name))\n\n        # Make sure the relevant files for modin get copied.\n        modin_files = []\n        for (root, dirs, filenames) in os.walk(\"./ray/modin\"):\n            for name in filenames:\n                modin_files.append(os.path.join(root, name))\n\n        files_to_include = ray_files + pyarrow_files + modin_files\n\n        # Copy over the autogenerated flatbuffer Python bindings.\n        for directory in generated_python_directories:\n            for filename in os.listdir(directory):\n                if filename[-3:] == \".py\":\n                    files_to_include.append(os.path.join(directory, filename))\n\n        for filename in files_to_include:\n            self.move_file(filename)\n\n        # Try to copy over the optional files.\n        for filename in optional_ray_files:\n            try:\n                self.move_file(filename)\n            except Exception:\n                print(\"Failed to copy optional file {}. This is ok.\"\n                      .format(filename))\n\n    def move_file(self, filename):\n        # TODO(rkn): This feels very brittle. It may not handle all cases. See\n        # https://github.com/apache/arrow/blob/master/python/setup.py for an\n        # example.\n        source = filename\n        destination = os.path.join(self.build_lib, filename)\n        # Create the target directory if it doesn't already exist.\n        parent_directory = os.path.dirname(destination)\n        if not os.path.exists(parent_directory):\n            os.makedirs(parent_directory)\n        if not os.path.exists(destination):\n            print(\"Copying {} to {}.\".format(source, destination))\n            shutil.copy(source, destination)\n\n\nclass BinaryDistribution(Distribution):\n    def has_ext_modules(self):\n        return True\n\n\ndef find_version(*filepath):\n    # Extract version information from filepath\n    here = os.path.abspath(os.path.dirname(__file__))\n    with open(os.path.join(here, *filepath)) as fp:\n        version_match = re.search(r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\",\n                                  fp.read(), re.M)\n        if version_match:\n            return version_match.group(1)\n        raise RuntimeError(\"Unable to find version string.\")\n\n\nrequires = [\n    \"numpy >= 1.14\",\n    \"filelock\",\n    \"funcsigs\",\n    \"click\",\n    \"colorama\",\n    \"pytest\",\n    \"pyyaml\",\n    \"redis\",\n    # NOTE: Don't upgrade the version of six! Doing so causes installation\n    # problems. See https://github.com/ray-project/ray/issues/4169.\n    \"six >= 1.0.0\",\n    # The typing module is required by modin.\n    \"typing\",\n    \"flatbuffers\",\n    \"faulthandler;python_version<'3.3'\",\n]\n\nsetup(\n    name=\"ray\",\n    version=find_version(\"ray\", \"__init__.py\"),\n    author=\"Ray Team\",\n    author_email=\"ray-dev@googlegroups.com\",\n    description=(\"A system for parallel and distributed Python that unifies \"\n                 \"the ML ecosystem.\"),\n    long_description=open(\"../README.rst\").read(),\n    url=\"https://github.com/ray-project/ray\",\n    keywords=(\"ray distributed parallel machine-learning \"\n              \"reinforcement-learning deep-learning python\"),\n    packages=find_packages(),\n    cmdclass={\"build_ext\": build_ext},\n    # The BinaryDistribution argument triggers build_ext.\n    distclass=BinaryDistribution,\n    install_requires=requires,\n    setup_requires=[\"cython >= 0.29\"],\n    extras_require=extras,\n    entry_points={\n        \"console_scripts\": [\n            \"ray=ray.scripts.scripts:main\",\n            \"rllib=ray.rllib.scripts:cli [rllib]\", \"tune=ray.tune.scripts:cli\"\n        ]\n    },\n    include_package_data=True,\n    zip_safe=False,\n    license=\"Apache 2.0\")\n", "path": "python/setup.py"}, {"content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os\nimport logging\ntry:  # py3\n    from shlex import quote\nexcept ImportError:  # py2\n    from pipes import quote\n\nlogger = logging.getLogger(__name__)\n\n\ndef dockerize_if_needed(config):\n    if \"docker\" not in config:\n        return config\n    docker_image = config[\"docker\"].get(\"image\")\n    cname = config[\"docker\"].get(\"container_name\")\n    run_options = config[\"docker\"].get(\"run_options\", [])\n    ssh_user = config[\"auth\"][\"ssh_user\"]\n    if not docker_image:\n        if cname:\n            logger.warning(\n                \"dockerize_if_needed: \"\n                \"Container name given but no Docker image - continuing...\")\n        return config\n    else:\n        assert cname, \"Must provide container name!\"\n    docker_mounts = {dst: dst for dst in config[\"file_mounts\"]}\n\n    config[\"setup_commands\"] = (\n        docker_start_cmds(ssh_user, docker_image, docker_mounts, cname,\n                          run_options) + with_docker_exec(\n                              config[\"setup_commands\"], container_name=cname))\n\n    config[\"head_setup_commands\"] = with_docker_exec(\n        config[\"head_setup_commands\"], container_name=cname)\n    config[\"head_start_ray_commands\"] = (\n        docker_autoscaler_setup(cname) + with_docker_exec(\n            config[\"head_start_ray_commands\"], container_name=cname))\n\n    config[\"worker_setup_commands\"] = with_docker_exec(\n        config[\"worker_setup_commands\"], container_name=cname)\n    config[\"worker_start_ray_commands\"] = with_docker_exec(\n        config[\"worker_start_ray_commands\"],\n        container_name=cname,\n        env_vars=[\"RAY_HEAD_IP\"])\n\n    return config\n\n\ndef with_docker_exec(cmds, container_name, env_vars=None):\n    env_str = \"\"\n    if env_vars:\n        env_str = \" \".join(\n            [\"-e {env}=${env}\".format(env=env) for env in env_vars])\n    return [\n        \"docker exec {} {} /bin/sh -c {} \".format(env_str, container_name,\n                                                  quote(cmd)) for cmd in cmds\n    ]\n\n\ndef aptwait_cmd():\n    return (\"while sudo fuser\"\n            \" /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock\"\n            \" >/dev/null 2>&1; \"\n            \"do echo 'Waiting for release of dpkg/apt locks'; sleep 5; done\")\n\n\ndef docker_start_cmds(user, image, mount, cname, user_options):\n    cmds = []\n\n    # create flags\n    # ports for the redis, object manager, and tune client\n    port_flags = \" \".join([\n        \"-p {port}:{port}\".format(port=port)\n        for port in [\"6379\", \"8076\", \"4321\"]\n    ])\n    mount_flags = \" \".join(\n        [\"-v {src}:{dest}\".format(src=k, dest=v) for k, v in mount.items()])\n\n    # for click, used in ray cli\n    env_vars = {\"LC_ALL\": \"C.UTF-8\", \"LANG\": \"C.UTF-8\"}\n    env_flags = \" \".join(\n        [\"-e {name}={val}\".format(name=k, val=v) for k, v in env_vars.items()])\n\n    user_options_str = \" \".join(user_options)\n    # docker run command\n    docker_check = [\n        \"docker\", \"inspect\", \"-f\", \"'{{.State.Running}}'\", cname, \"||\"\n    ]\n    docker_run = [\n        \"docker\", \"run\", \"--rm\", \"--name {}\".format(cname), \"-d\", \"-it\",\n        port_flags, mount_flags, env_flags, user_options_str, \"--net=host\",\n        image, \"bash\"\n    ]\n    cmds.append(\" \".join(docker_check + docker_run))\n    docker_update = [\n        \" && \".join((\"apt-get -y update\", \"apt-get -y upgrade\",\n                     \"apt-get install -y git wget cmake psmisc\"))\n    ]\n    cmds.extend(with_docker_exec(docker_update, container_name=cname))\n    return cmds\n\n\ndef docker_autoscaler_setup(cname):\n    cmds = []\n    for path in [\"~/ray_bootstrap_config.yaml\", \"~/ray_bootstrap_key.pem\"]:\n        # needed because docker doesn't allow relative paths\n        base_path = os.path.basename(path)\n        cmds.append(\"docker cp {path} {cname}:{dpath}\".format(\n            path=path, dpath=base_path, cname=cname))\n        cmds.extend(\n            with_docker_exec(\n                [\"cp {} {}\".format(\"/\" + base_path, path)],\n                container_name=cname))\n    return cmds\n", "path": "python/ray/autoscaler/docker.py"}], "after_files": [{"content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os\nimport re\nimport shutil\nimport subprocess\nimport sys\n\nfrom setuptools import setup, find_packages, Distribution\nimport setuptools.command.build_ext as _build_ext\n\n# Ideally, we could include these files by putting them in a\n# MANIFEST.in or using the package_data argument to setup, but the\n# MANIFEST.in gets applied at the very beginning when setup.py runs\n# before these files have been created, so we have to move the files\n# manually.\n\n# NOTE: The lists below must be kept in sync with ray/BUILD.bazel.\n\nray_files = [\n    \"ray/core/src/ray/thirdparty/redis/src/redis-server\",\n    \"ray/core/src/ray/gcs/redis_module/libray_redis_module.so\",\n    \"ray/core/src/plasma/plasma_store_server\", \"ray/_raylet.so\",\n    \"ray/core/src/ray/raylet/raylet_monitor\", \"ray/core/src/ray/raylet/raylet\",\n    \"ray/dashboard/dashboard.py\", \"ray/dashboard/index.html\",\n    \"ray/dashboard/res/main.css\", \"ray/dashboard/res/main.js\"\n]\n\n# These are the directories where automatically generated Python flatbuffer\n# bindings are created.\ngenerated_python_directories = [\n    \"ray/core/generated\", \"ray/core/generated/ray\",\n    \"ray/core/generated/ray/protocol\"\n]\n\noptional_ray_files = []\n\nray_autoscaler_files = [\n    \"ray/autoscaler/aws/example-full.yaml\",\n    \"ray/autoscaler/gcp/example-full.yaml\",\n    \"ray/autoscaler/local/example-full.yaml\",\n]\n\nif \"RAY_USE_NEW_GCS\" in os.environ and os.environ[\"RAY_USE_NEW_GCS\"] == \"on\":\n    ray_files += [\n        \"ray/core/src/credis/build/src/libmember.so\",\n        \"ray/core/src/credis/build/src/libmaster.so\",\n        \"ray/core/src/credis/redis/src/redis-server\"\n    ]\n\noptional_ray_files += ray_autoscaler_files\n\nextras = {\n    \"rllib\": [\n        \"pyyaml\", \"gym[atari]\", \"opencv-python-headless\", \"lz4\", \"scipy\"\n    ],\n    \"debug\": [\"psutil\", \"setproctitle\", \"py-spy\"],\n    \"dashboard\": [\"psutil\", \"aiohttp\"],\n}\n\n\nclass build_ext(_build_ext.build_ext):\n    def run(self):\n        # Note: We are passing in sys.executable so that we use the same\n        # version of Python to build pyarrow inside the build.sh script. Note\n        # that certain flags will not be passed along such as --user or sudo.\n        # TODO(rkn): Fix this.\n        command = [\"../build.sh\", \"-p\", sys.executable]\n        if os.getenv(\"RAY_INSTALL_JAVA\") == \"1\":\n            # Also build binaries for Java if the above env variable exists.\n            command += [\"-l\", \"python,java\"]\n        subprocess.check_call(command)\n\n        # We also need to install pyarrow along with Ray, so make sure that the\n        # relevant non-Python pyarrow files get copied.\n        pyarrow_files = []\n        for (root, dirs, filenames) in os.walk(\"./ray/pyarrow_files/pyarrow\"):\n            for name in filenames:\n                pyarrow_files.append(os.path.join(root, name))\n\n        # Make sure the relevant files for modin get copied.\n        modin_files = []\n        for (root, dirs, filenames) in os.walk(\"./ray/modin\"):\n            for name in filenames:\n                modin_files.append(os.path.join(root, name))\n\n        files_to_include = ray_files + pyarrow_files + modin_files\n\n        # Copy over the autogenerated flatbuffer Python bindings.\n        for directory in generated_python_directories:\n            for filename in os.listdir(directory):\n                if filename[-3:] == \".py\":\n                    files_to_include.append(os.path.join(directory, filename))\n\n        for filename in files_to_include:\n            self.move_file(filename)\n\n        # Try to copy over the optional files.\n        for filename in optional_ray_files:\n            try:\n                self.move_file(filename)\n            except Exception:\n                print(\"Failed to copy optional file {}. This is ok.\"\n                      .format(filename))\n\n    def move_file(self, filename):\n        # TODO(rkn): This feels very brittle. It may not handle all cases. See\n        # https://github.com/apache/arrow/blob/master/python/setup.py for an\n        # example.\n        source = filename\n        destination = os.path.join(self.build_lib, filename)\n        # Create the target directory if it doesn't already exist.\n        parent_directory = os.path.dirname(destination)\n        if not os.path.exists(parent_directory):\n            os.makedirs(parent_directory)\n        if not os.path.exists(destination):\n            print(\"Copying {} to {}.\".format(source, destination))\n            shutil.copy(source, destination)\n\n\nclass BinaryDistribution(Distribution):\n    def has_ext_modules(self):\n        return True\n\n\ndef find_version(*filepath):\n    # Extract version information from filepath\n    here = os.path.abspath(os.path.dirname(__file__))\n    with open(os.path.join(here, *filepath)) as fp:\n        version_match = re.search(r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\",\n                                  fp.read(), re.M)\n        if version_match:\n            return version_match.group(1)\n        raise RuntimeError(\"Unable to find version string.\")\n\n\nrequires = [\n    \"numpy >= 1.14\",\n    \"filelock\",\n    \"funcsigs\",\n    \"click\",\n    \"colorama\",\n    \"pytest\",\n    \"pyyaml\",\n    \"redis\",\n    # NOTE: Don't upgrade the version of six! Doing so causes installation\n    # problems. See https://github.com/ray-project/ray/issues/4169.\n    \"six >= 1.0.0\",\n    # The typing module is required by modin.\n    \"typing\",\n    \"flatbuffers\",\n    \"faulthandler;python_version<'3.3'\",\n]\n\nsetup(\n    name=\"ray\",\n    version=find_version(\"ray\", \"__init__.py\"),\n    author=\"Ray Team\",\n    author_email=\"ray-dev@googlegroups.com\",\n    description=(\"A system for parallel and distributed Python that unifies \"\n                 \"the ML ecosystem.\"),\n    long_description=open(\"../README.rst\").read(),\n    url=\"https://github.com/ray-project/ray\",\n    keywords=(\"ray distributed parallel machine-learning \"\n              \"reinforcement-learning deep-learning python\"),\n    packages=find_packages(),\n    cmdclass={\"build_ext\": build_ext},\n    # The BinaryDistribution argument triggers build_ext.\n    distclass=BinaryDistribution,\n    install_requires=requires,\n    setup_requires=[\"cython >= 0.29\"],\n    extras_require=extras,\n    entry_points={\n        \"console_scripts\": [\n            \"ray=ray.scripts.scripts:main\",\n            \"rllib=ray.rllib.scripts:cli [rllib]\", \"tune=ray.tune.scripts:cli\"\n        ]\n    },\n    include_package_data=True,\n    zip_safe=False,\n    license=\"Apache 2.0\")\n", "path": "python/setup.py"}, {"content": "from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os\nimport logging\ntry:  # py3\n    from shlex import quote\nexcept ImportError:  # py2\n    from pipes import quote\n\nlogger = logging.getLogger(__name__)\n\n\ndef dockerize_if_needed(config):\n    if \"docker\" not in config:\n        return config\n    docker_image = config[\"docker\"].get(\"image\")\n    cname = config[\"docker\"].get(\"container_name\")\n    run_options = config[\"docker\"].get(\"run_options\", [])\n    ssh_user = config[\"auth\"][\"ssh_user\"]\n    if not docker_image:\n        if cname:\n            logger.warning(\n                \"dockerize_if_needed: \"\n                \"Container name given but no Docker image - continuing...\")\n        return config\n    else:\n        assert cname, \"Must provide container name!\"\n    docker_mounts = {dst: dst for dst in config[\"file_mounts\"]}\n\n    config[\"setup_commands\"] = (\n        docker_start_cmds(ssh_user, docker_image, docker_mounts, cname,\n                          run_options) + with_docker_exec(\n                              config[\"setup_commands\"], container_name=cname))\n\n    config[\"head_setup_commands\"] = with_docker_exec(\n        config[\"head_setup_commands\"], container_name=cname)\n    config[\"head_start_ray_commands\"] = (\n        docker_autoscaler_setup(cname) + with_docker_exec(\n            config[\"head_start_ray_commands\"], container_name=cname))\n\n    config[\"worker_setup_commands\"] = with_docker_exec(\n        config[\"worker_setup_commands\"], container_name=cname)\n    config[\"worker_start_ray_commands\"] = with_docker_exec(\n        config[\"worker_start_ray_commands\"],\n        container_name=cname,\n        env_vars=[\"RAY_HEAD_IP\"])\n\n    return config\n\n\ndef with_docker_exec(cmds, container_name, env_vars=None):\n    env_str = \"\"\n    if env_vars:\n        env_str = \" \".join(\n            [\"-e {env}=${env}\".format(env=env) for env in env_vars])\n    return [\n        \"docker exec {} {} /bin/sh -c {} \".format(env_str, container_name,\n                                                  quote(cmd)) for cmd in cmds\n    ]\n\n\ndef aptwait_cmd():\n    return (\"while sudo fuser\"\n            \" /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock\"\n            \" >/dev/null 2>&1; \"\n            \"do echo 'Waiting for release of dpkg/apt locks'; sleep 5; done\")\n\n\ndef docker_start_cmds(user, image, mount, cname, user_options):\n    cmds = []\n\n    # create flags\n    # ports for the redis, object manager, and tune client\n    port_flags = \" \".join([\n        \"-p {port}:{port}\".format(port=port)\n        for port in [\"6379\", \"8076\", \"4321\"]\n    ])\n    mount_flags = \" \".join(\n        [\"-v {src}:{dest}\".format(src=k, dest=v) for k, v in mount.items()])\n\n    # for click, used in ray cli\n    env_vars = {\"LC_ALL\": \"C.UTF-8\", \"LANG\": \"C.UTF-8\"}\n    env_flags = \" \".join(\n        [\"-e {name}={val}\".format(name=k, val=v) for k, v in env_vars.items()])\n\n    user_options_str = \" \".join(user_options)\n    # docker run command\n    docker_check = [\n        \"docker\", \"inspect\", \"-f\", \"'{{.State.Running}}'\", cname, \"||\"\n    ]\n    docker_run = [\n        \"docker\", \"run\", \"--rm\", \"--name {}\".format(cname), \"-d\", \"-it\",\n        port_flags, mount_flags, env_flags, user_options_str, \"--net=host\",\n        image, \"bash\"\n    ]\n    cmds.append(\" \".join(docker_check + docker_run))\n    docker_update = [\n        \" && \".join((\"apt-get -y update\", \"apt-get -y upgrade\",\n                     \"apt-get install -y git wget psmisc\"))\n    ]\n    cmds.extend(with_docker_exec(docker_update, container_name=cname))\n    return cmds\n\n\ndef docker_autoscaler_setup(cname):\n    cmds = []\n    for path in [\"~/ray_bootstrap_config.yaml\", \"~/ray_bootstrap_key.pem\"]:\n        # needed because docker doesn't allow relative paths\n        base_path = os.path.basename(path)\n        cmds.append(\"docker cp {path} {cname}:{dpath}\".format(\n            path=path, dpath=base_path, cname=cname))\n        cmds.extend(\n            with_docker_exec(\n                [\"cp {} {}\".format(\"/\" + base_path, path)],\n                container_name=cname))\n    return cmds\n", "path": "python/ray/autoscaler/docker.py"}]}