<NME> test_template.py
<BEF> # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re

from unittest.mock import Mock

from trove.common import template
from trove.datastore.models import DatastoreVersion
from trove.tests.unittests import trove_testtools
from trove.tests.unittests.util import util


    def setUp(self):
        super(TemplateTest, self).setUp()
        util.init_db()
        self.env = template.get_env()
        self.template = self.env.get_template("mysql.config.template")
        self.flavor_dict = {'ram': 1024}

        self.server_id = "180b5ed1-3e57-4459-b7a3-2aeee4ac012a"

    def tearDown(self):
        super(TemplateTest, self).tearDown()

    def _find_in_template(self, contents, teststr):
        found_group = None
        for line in contents.split('\n'):
            m = re.search('^%s.*' % teststr, line)
            if m:
                found_group = m.group(0)
        return found_group

    def validate_template(self, contents, teststr, test_flavor, server_id):
        # expected innodb_buffer_pool_size = {{ (150 * flavor_multiplier }}M
        flavor_multiplier = test_flavor['ram'] // 512
        found_group = self._find_in_template(contents, teststr)
        if not found_group:
            raise Exception("Could not find text in template")
        # Check that the last group has been rendered
        memsize = found_group.split(" ")[2]
        self.assertEqual("%sM" % (150 * flavor_multiplier), memsize)
        location = "/etc/mysql/my.cnf"
        config = template.SingleInstanceConfigTemplate('mysql',
                                                       self.flavor_dict)
        config.render()
        self.assertEqual(location, config.config_location)
        self.validate_template(config.config_contents, "query_cache_size",
                               self.flavor_dict)
        self.validate_template(rendered,
                               "innodb_buffer_pool_size",
                               self.flavor_dict,
                               self.server_id)

    def test_single_instance_config_rendering(self):
        datastore = Mock(spec=DatastoreVersion)
        datastore.datastore_name = 'MySql'
        datastore.name = 'mysql-5.7'
        datastore.manager = 'mysql'
        config = template.SingleInstanceConfigTemplate(datastore,
                                                       self.flavor_dict,
                                                       self.server_id)
        self.validate_template(config.render(), "innodb_buffer_pool_size",
                               self.flavor_dict, self.server_id)

    def test_renderer_discovers_special_config(self):
        """Finds our special config file for the version 'mysql-test'."""
        datastore = Mock(spec=DatastoreVersion)
        datastore.datastore_name = 'mysql'
        datastore.name = 'mysql-test'
        datastore.manager = 'mysql'
        config = template.SingleInstanceConfigTemplate(datastore,
                                                       self.flavor_dict,
                                                       self.server_id)
        self.validate_template(config.render(), "hyper",
                               {'ram': 0}, self.server_id)

    def test_replica_source_config_rendering(self):
        datastore = Mock(spec=DatastoreVersion)
        datastore.datastore_name = 'MySql'
        datastore.name = 'mysql-5.7'
        datastore.manager = 'mysql'
        config = template.ReplicaSourceConfigTemplate(datastore,
                                                      self.flavor_dict,
                                                      self.server_id)
        self.assertTrue(self._find_in_template(config.render(), "log_bin"))

    def test_replica_config_rendering(self):
        datastore = Mock(spec=DatastoreVersion)
        datastore.datastore_name = 'MySql'
        datastore.name = 'mysql-5.7'
        datastore.manager = 'mysql'
        config = template.ReplicaConfigTemplate(datastore,
                                                self.flavor_dict,
                                                self.server_id)
        self.assertTrue(self._find_in_template(config.render(), "relay_log"))

<MSG> Added docs and made template filename variable

Changed how jinja ENV is created
Added docstrings to functions in template.py
Renamed _location_types private attribute of
 SingleInstanceConfigTemplate to _config_paths
SingleInstanceConfigTemplate constructor now uses filename based on
 service type
SingleInstanceConfigTemplate render method now returns the result
Tests updated to use new style of render method
mysql.config.template duplicated to percona.config.template

implements blueprint redis-servicetype

Change-Id: Id9317b560236d5d4967b888d0dc3a4135f1822dc

<DFF> @@ -22,7 +22,7 @@ class TemplateTest(testtools.TestCase):
     def setUp(self):
         super(TemplateTest, self).setUp()
         util.init_db()
-        self.env = template.get_env()
+        self.env = template.ENV
         self.template = self.env.get_template("mysql.config.template")
         self.flavor_dict = {'ram': 1024}
 
@@ -51,7 +51,6 @@ class TemplateTest(testtools.TestCase):
         location = "/etc/mysql/my.cnf"
         config = template.SingleInstanceConfigTemplate('mysql',
                                                        self.flavor_dict)
-        config.render()
         self.assertEqual(location, config.config_location)
-        self.validate_template(config.config_contents, "query_cache_size",
+        self.validate_template(config.render(), "query_cache_size",
                                self.flavor_dict)

{'commit': 'diff --git a/trove/common/template.py b/trove/common/template.py\nindex dafc626b..df5dc4e6 100644\n--- a/trove/common/template.py\n+++ b/trove/common/template.py\n@@ -12,35 +12,40 @@\n #    License for the specific language governing permissions and limitations\n #    under the License.\n \n-_ENV = None\n-\n import jinja2\n \n-\n-def get_env():\n-    global _ENV\n-    if not _ENV:\n-        _ENV = create_env()\n-    return _ENV\n+ENV = jinja2.Environment(loader=jinja2.ChoiceLoader([\n+    jinja2.FileSystemLoader("/etc/trove/templates"),\n+    jinja2.PackageLoader("trove", "templates")\n+]))\n \n \n-def create_env():\n-    loader = jinja2.ChoiceLoader([\n-        jinja2.FileSystemLoader("/etc/trove/templates"),\n-        jinja2.PackageLoader("trove", "templates")\n-    ])\n-    return jinja2.Environment(loader=loader)\n+class SingleInstanceConfigTemplate(object):\n+    """ This class selects a single configuration file by database type for\n+    rendering on the guest """\n+    _config_paths = {\'mysql\': \'/etc/mysql/my.cnf\',\n+                     \'percona\': \'/etc/mysql/my.cnf\'}\n \n+    def __init__(self, service_type, flavor_dict):\n+        """ Constructor\n \n-class SingleInstanceConfigTemplate(object):\n-    _location_types = {\'mysql\': \'/etc/mysql/my.cnf\',\n-                       \'percona\': \'/etc/mysql/my.cnf\'}\n+        :param service_type: The database type.\n+        :type name: str.\n+        :param flavor_dict: dict containing flavor details for use in jinja.\n+        :type flavor_dict: dict.\n \n-    def __init__(self, location_type, flavor_dict):\n-        self.config_location = self._location_types[location_type]\n+        """\n+        self.config_location = self._config_paths[service_type]\n         self.flavor_dict = flavor_dict\n-        self.template = get_env().get_template("mysql.config.template")\n+        template_filename = "%s.config.template" % service_type\n+        self.template = ENV.get_template(template_filename)\n \n     def render(self):\n+        """ Renders the jinja template\n+\n+        :returns: str -- The rendered configuration file\n+\n+        """\n         self.config_contents = self.template.render(\n             flavor=self.flavor_dict)\n+        return self.config_contents\ndiff --git a/trove/templates/percona.config.template b/trove/templates/percona.config.template\nnew file mode 100644\nindex 00000000..1bd6aae7\n--- /dev/null\n+++ b/trove/templates/percona.config.template\n@@ -0,0 +1,58 @@\n+[client]\n+port = 3306\n+socket = /var/run/mysqld/mysqld.sock\n+\n+[mysqld_safe]\n+socket = /var/run/mysqld/mysqld.sock\n+nice = 0\n+\n+[mysqld]\n+user = mysql\n+socket = /var/run/mysqld/mysqld.sock\n+port = 3306\n+basedir = /usr\n+datadir = /var/lib/mysql\n+####tmpdir = /tmp\n+tmpdir = /var/tmp\n+pid_file = /var/run/mysqld/mysqld.pid\n+skip-external-locking\n+key_buffer_size = {{ 50 * flavor[\'ram\']//512 }}M\n+max_allowed_packet = {{ 1 * flavor[\'ram\']//512 }}M\n+thread_stack = 192K\n+thread_cache_size = {{ 4 * flavor[\'ram\']//512 }}\n+myisam-recover = BACKUP\n+query_cache_type = 1\n+query_cache_limit = 1M\n+query_cache_size = {{ 8 * flavor[\'ram\']//512 }}M\n+log_error = /var/log/mysql/mysqld.log\n+innodb_data_file_path = ibdata1:10M:autoextend\n+innodb_buffer_pool_size = {{ 150 * flavor[\'ram\']//512 }}M\n+innodb_file_per_table = 1\n+innodb_log_files_in_group = 2\n+innodb_log_file_size=50M\n+innodb_log_buffer_size=25M\n+connect_timeout = 15\n+wait_timeout = 120\n+join_buffer_size = 1M\n+read_buffer_size = 512K\n+read_rnd_buffer_size = 512K\n+sort_buffer_size = 1M\n+tmp_table_size = {{ 16 * flavor[\'ram\']//512 }}M\n+max_heap_table_size = {{ 16 * flavor[\'ram\']//512 }}M\n+table_open_cache = {{ 256 * flavor[\'ram\']//512 }}\n+table_definition_cache = {{ 256 * flavor[\'ram\']//512 }}\n+open_files_limit = {{ 512 * flavor[\'ram\']//512 }}\n+max_user_connections = {{ 100 * flavor[\'ram\']//512 }}\n+max_connections = {{ 100 * flavor[\'ram\']//512 }}\n+default_storage_engine = innodb\n+local-infile = 0\n+\n+[mysqldump]\n+quick\n+quote-names\n+max_allowed_packet = 16M\n+\n+[isamchk]\n+key_buffer = 16M\n+\n+!includedir /etc/mysql/conf.d/\ndiff --git a/trove/tests/api/instances_resize.py b/trove/tests/api/instances_resize.py\nindex c6c16f1e..b2410d7d 100644\n--- a/trove/tests/api/instances_resize.py\n+++ b/trove/tests/api/instances_resize.py\n@@ -117,9 +117,8 @@ class ResizeTests(ResizeTestBase):\n     def _start_mysql(self):\n         config = template.SingleInstanceConfigTemplate(\n             "mysql", NEW_FLAVOR.__dict__)\n-        config.render()\n         self.instance.guest.start_db_with_conf_changes(config.config_location,\n-                                                       config.config_contents)\n+                                                       config.render())\n \n     def test_guest_wont_stop_mysql(self):\n         self.guest.stop_db(do_not_start_on_reboot=True)\\\ndiff --git a/trove/tests/unittests/common/test_template.py b/trove/tests/unittests/common/test_template.py\nindex ad4a5977..f6e4e6ca 100644\n--- a/trove/tests/unittests/common/test_template.py\n+++ b/trove/tests/unittests/common/test_template.py\n@@ -22,7 +22,7 @@ class TemplateTest(testtools.TestCase):\n     def setUp(self):\n         super(TemplateTest, self).setUp()\n         util.init_db()\n-        self.env = template.get_env()\n+        self.env = template.ENV\n         self.template = self.env.get_template("mysql.config.template")\n         self.flavor_dict = {\'ram\': 1024}\n \n@@ -51,7 +51,6 @@ class TemplateTest(testtools.TestCase):\n         location = "/etc/mysql/my.cnf"\n         config = template.SingleInstanceConfigTemplate(\'mysql\',\n                                                        self.flavor_dict)\n-        config.render()\n         self.assertEqual(location, config.config_location)\n-        self.validate_template(config.config_contents, "query_cache_size",\n+        self.validate_template(config.render(), "query_cache_size",\n                                self.flavor_dict)\n', 'commit_subject': 'Added docs and made template filename variable', 'repo_name': 'openstack/trove', 'committer': {'name': 'Michael Basnight', 'email': '180923fa42bdd6f2d234322b27c91f18387134d1@gmail.com', 'time_sec': '1375292127', 'tz_offset': '-420', 'date': {'seconds': '1375292127'}}, 'commit_tree': '22f4a39bcc9cdcfc72e49c27f555559ff7e4df20', 'commit_message': 'Added docs and made template filename variable\n\nChanged how jinja ENV is created\nAdded docstrings to functions in template.py\nRenamed _location_types private attribute of\n SingleInstanceConfigTemplate to _config_paths\nSingleInstanceConfigTemplate constructor now uses filename based on\n service type\nSingleInstanceConfigTemplate render method now returns the result\nTests updated to use new style of render method\nmysql.config.template duplicated to percona.config.template\n\nimplements blueprint redis-servicetype\n\nChange-Id: Id9317b560236d5d4967b888d0dc3a4135f1822dc\n', 'language_name': 'Shell', 'license': 'apache-2.0', 'hunks': '@@ -22,7 +22,7 @@ class TemplateTest(testtools.TestCase):\n     def setUp(self):\n         super(TemplateTest, self).setUp()\n         util.init_db()\n-        self.env = template.get_env()\n+        self.env = template.ENV\n         self.template = self.env.get_template("mysql.config.template")\n         self.flavor_dict = {\'ram\': 1024}\n \n@@ -51,7 +51,6 @@ class TemplateTest(testtools.TestCase):\n         location = "/etc/mysql/my.cnf"\n         config = template.SingleInstanceConfigTemplate(\'mysql\',\n                                                        self.flavor_dict)\n-        config.render()\n         self.assertEqual(location, config.config_location)\n-        self.validate_template(config.config_contents, "query_cache_size",\n+        self.validate_template(config.render(), "query_cache_size",\n                                self.flavor_dict)\n', 'addition_count': 2, 'deletion_count': 3, 'src_file': 'a/trove/tests/unittests/common/test_template.py', 'tgt_file': 'b/trove/tests/unittests/common/test_template.py', 'file_extension': '.py', 'before_file': '# Licensed under the Apache License, Version 2.0 (the "License");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an "AS IS" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport re\n\nfrom unittest.mock import Mock\n\nfrom trove.common import template\nfrom trove.datastore.models import DatastoreVersion\nfrom trove.tests.unittests import trove_testtools\nfrom trove.tests.unittests.util import util\n\n\n    def setUp(self):\n        super(TemplateTest, self).setUp()\n        util.init_db()\n        self.env = template.get_env()\n        self.template = self.env.get_template("mysql.config.template")\n        self.flavor_dict = {\'ram\': 1024}\n\n        self.server_id = "180b5ed1-3e57-4459-b7a3-2aeee4ac012a"\n\n    def tearDown(self):\n        super(TemplateTest, self).tearDown()\n\n    def _find_in_template(self, contents, teststr):\n        found_group = None\n        for line in contents.split(\'\\n\'):\n            m = re.search(\'^%s.*\' % teststr, line)\n            if m:\n                found_group = m.group(0)\n        return found_group\n\n    def validate_template(self, contents, teststr, test_flavor, server_id):\n        # expected innodb_buffer_pool_size = {{ (150 * flavor_multiplier }}M\n        flavor_multiplier = test_flavor[\'ram\'] // 512\n        found_group = self._find_in_template(contents, teststr)\n        if not found_group:\n            raise Exception("Could not find text in template")\n        # Check that the last group has been rendered\n        memsize = found_group.split(" ")[2]\n        self.assertEqual("%sM" % (150 * flavor_multiplier), memsize)\n        location = "/etc/mysql/my.cnf"\n        config = template.SingleInstanceConfigTemplate(\'mysql\',\n                                                       self.flavor_dict)\n        config.render()\n        self.assertEqual(location, config.config_location)\n        self.validate_template(config.config_contents, "query_cache_size",\n                               self.flavor_dict)\n        self.validate_template(rendered,\n                               "innodb_buffer_pool_size",\n                               self.flavor_dict,\n                               self.server_id)\n\n    def test_single_instance_config_rendering(self):\n        datastore = Mock(spec=DatastoreVersion)\n        datastore.datastore_name = \'MySql\'\n        datastore.name = \'mysql-5.7\'\n        datastore.manager = \'mysql\'\n        config = template.SingleInstanceConfigTemplate(datastore,\n                                                       self.flavor_dict,\n                                                       self.server_id)\n        self.validate_template(config.render(), "innodb_buffer_pool_size",\n                               self.flavor_dict, self.server_id)\n\n    def test_renderer_discovers_special_config(self):\n        """Finds our special config file for the version \'mysql-test\'."""\n        datastore = Mock(spec=DatastoreVersion)\n        datastore.datastore_name = \'mysql\'\n        datastore.name = \'mysql-test\'\n        datastore.manager = \'mysql\'\n        config = template.SingleInstanceConfigTemplate(datastore,\n                                                       self.flavor_dict,\n                                                       self.server_id)\n        self.validate_template(config.render(), "hyper",\n                               {\'ram\': 0}, self.server_id)\n\n    def test_replica_source_config_rendering(self):\n        datastore = Mock(spec=DatastoreVersion)\n        datastore.datastore_name = \'MySql\'\n        datastore.name = \'mysql-5.7\'\n        datastore.manager = \'mysql\'\n        config = template.ReplicaSourceConfigTemplate(datastore,\n                                                      self.flavor_dict,\n                                                      self.server_id)\n        self.assertTrue(self._find_in_template(config.render(), "log_bin"))\n\n    def test_replica_config_rendering(self):\n        datastore = Mock(spec=DatastoreVersion)\n        datastore.datastore_name = \'MySql\'\n        datastore.name = \'mysql-5.7\'\n        datastore.manager = \'mysql\'\n        config = template.ReplicaConfigTemplate(datastore,\n                                                self.flavor_dict,\n                                                self.server_id)\n        self.assertTrue(self._find_in_template(config.render(), "relay_log"))\n', 'message': 'Added docs and made template filename variable\n\nChanged how jinja ENV is created\nAdded docstrings to functions in template.py\nRenamed _location_types private attribute of\n SingleInstanceConfigTemplate to _config_paths\nSingleInstanceConfigTemplate constructor now uses filename based on\n service type\nSingleInstanceConfigTemplate render method now returns the result\nTests updated to use new style of render method\nmysql.config.template duplicated to percona.config.template\n\nimplements blueprint redis-servicetype\n\nChange-Id: Id9317b560236d5d4967b888d0dc3a4135f1822dc\n', 'author_name': 'Michael Basnight', 'lang': 'py'}