### Function versioning API: get

```Support 2 types of queries:

- Get basic function version information

TODO:

functional test(in the subsequent patch)

Change-Id: I1bcb7c13a7b65432e1d6714d7539077ab82ee831
Story: #2001829
changes/68/562268/3 Lingxian Kong 3 years ago
parent
commit
e8e8084982
8 changed files with 129 additions and 10 deletions
1. 1
qinling/api/controllers/v1/function.py
2. 40
qinling/api/controllers/v1/function_version.py
3. 4
qinling/db/api.py
4. 13
qinling/db/sqlalchemy/api.py
5. 3
qinling/storage/base.py
6. 36
qinling/storage/file_system.py
7. 9
qinling/tests/unit/api/controllers/v1/test_function_version.py
8. 33
qinling/tests/unit/storage/test_file_system.py

#### 1 qinling/api/controllers/v1/function.py View File

 `@ -129,7 +129,6 @@ class FunctionsController(rest.RestController):` ` pecan.response.headers['Content-Disposition'] = (` ` 'attachment; filename="%s"' % os.path.basename(func_db.name)` ` )` ` LOG.info("Downloaded function %s", id)` ``` ``` ` @rest_utils.wrap_pecan_controller_exception` ` @pecan.expose('json')`

#### 40 qinling/api/controllers/v1/function_version.py View File

 `@ -12,10 +12,15 @@` `# See the License for the specific language governing permissions and` `# limitations under the License.` ``` ``` `import collections` ``` ``` `from oslo_config import cfg` `from oslo_log import log as logging` `from oslo_utils import strutils` `import pecan` `from pecan import rest` `import tenacity` `from webob.static import FileIter` `import wsmeext.pecan as wsme_pecan` ``` ``` `from qinling.api import access_control as acl` `@ -137,3 +142,38 @@ class FunctionVersionsController(rest.RestController):` ` for v in db_versions]` ``` ``` ` return resources.FunctionVersions(function_versions=versions)` ``` ``` ` @rest_utils.wrap_pecan_controller_exception` ` @pecan.expose()` ` def get(self, function_id, version):` ` ctx = context.get_ctx()` ` acl.enforce('function_version:get', ctx)` ``` ``` ` download = strutils.bool_from_string(` ` pecan.request.GET.get('download', False)` ` )` ` version = int(version)` ``` ``` ` version_db = db_api.get_function_version(function_id, version)` ``` ``` ` if not download:` ` LOG.info("Getting version %s for function %s.", version,` ` function_id)` ` pecan.override_template('json')` ` return resources.FunctionVersion.from_dict(` ` version_db.to_dict()).to_dict()` ``` ``` ` LOG.info("Downloading version %s for function %s.", version,` ` function_id)` ``` ``` ` f = self.storage_provider.retrieve(ctx.projectid, function_id,` ` None, version=version)` ``` ``` ` if isinstance(f, collections.Iterable):` ` pecan.response.app_iter = f` ` else:` ` pecan.response.app_iter = FileIter(f)` ` pecan.response.headers['Content-Type'] = 'application/zip'` ` pecan.response.headers['Content-Disposition'] = (` ` 'attachment; filename="%s_%s"' % (function_id, version)` ` )`

#### 4 qinling/db/api.py View File

 `@ -206,3 +206,7 @@ def delete_webhooks(**kwargs):` `def increase_function_version(function_id, old_version, **kwargs):` ` """This function is meant to be invoked within locking section."""` ` return IMPL.increase_function_version(function_id, old_version, **kwargs)` ``` ``` ``` ``` `def get_function_version(function_id, version):` ` return IMPL.get_function_version(function_id, version)`

#### 13 qinling/db/sqlalchemy/api.py View File

 `@ -514,3 +514,16 @@ def increase_function_version(function_id, old_version, session=None,` ` )` ``` ``` ` return version` ``` ``` ``` ``` `@db_base.session_aware()` `def get_function_version(function_id, version, session=None):` ` version_db = _secure_query(models.FunctionVersion).filter_by(` ` function_id=function_id, version_number=version).first()` ` if not version_db:` ` raise exc.DBEntityNotFoundError(` ` "FunctionVersion not found [function_id=%s, version_number=%s]" %` ` (function_id, version)` ` )` ``` ``` ` return version_db`

#### 3 qinling/storage/base.py View File

 `@ -40,12 +40,13 @@ class PackageStorage(object):` ` raise NotImplementedError` ``` ``` ` @abc.abstractmethod` ` def retrieve(self, project_id, function, md5sum):` ` def retrieve(self, project_id, function, md5sum, version=None):` ` """Get function package data.` ``` ``` ` :param project_id: Project ID.` ` :param function: Function ID.` ` :param md5sum: The function MD5.` ` :param version: Optional. The function version number.` ` :return: File descriptor that needs to close outside.` ` """` ` raise NotImplementedError`

#### 36 qinling/storage/file_system.py View File

 `@ -74,23 +74,42 @@ class FileSystemStorage(base.PackageStorage):` ` os.rename(new_func_zip, func_zip)` ` return md5_actual` ``` ``` ` def retrieve(self, project_id, function, md5sum):` ` def retrieve(self, project_id, function, md5sum, version=0):` ` """Get function package data.` ``` ``` ` If version is not 0, return the package data of that specific function` ` version.` ``` ``` ` :param project_id: Project ID.` ` :param function: Function ID.` ` :param md5sum: The function MD5.` ` :param version: Optional. The function version number.` ` :return: File descriptor that needs to close outside.` ` """` ` LOG.debug(` ` 'Getting package data, function: %s, md5sum: %s, project: %s',` ` function, md5sum, project_id` ` 'Getting package data, function: %s, version: %s, md5sum: %s, '` ` 'project: %s',` ` function, md5sum, version, project_id` ` )` ``` ``` ` func_zip = os.path.join(` ` self.base_path,` ` PACKAGE_PATH_TEMPLATE % (project_id, function, md5sum)` ` )` ` if version != 0:` ` project_dir = os.path.join(self.base_path, project_id)` ` for filename in os.listdir(project_dir):` ` root, ext = os.path.splitext(filename)` ` if (root.startswith("%s_%d" % (function, version))` ` and ext == '.zip'):` ` func_zip = os.path.join(project_dir, filename)` ` break` ` else:` ` raise exc.StorageNotFoundException(` ` 'Package of version %d function %s for project %s not '` ` 'found.' % (version, function, project_id)` ` )` ` else:` ` func_zip = os.path.join(` ` self.base_path,` ` PACKAGE_PATH_TEMPLATE % (project_id, function, md5sum)` ` )` ``` ``` ` if not os.path.exists(func_zip):` ` raise exc.StorageNotFoundException(` `@ -99,7 +118,8 @@ class FileSystemStorage(base.PackageStorage):` ` )` ``` ``` ` f = open(func_zip, 'rb')` ` LOG.debug('Found package data for function %s', function)` ` LOG.debug('Found package data for function %s version %d', function,` ` version)` ``` ``` ` return f` ``` ```

#### 9 qinling/tests/unit/api/controllers/v1/test_function_version.py View File

 `@ -108,3 +108,12 @@ class TestFunctionVersionController(base.APITest):` ` actual = self._assert_single_item(resp.json['function_versions'],` ` version_number=1)` ` self.assertEqual("version 1", actual.get('description'))` ``` ``` ` def test_get(self):` ` db_api.increase_function_version(self.func_id, 0,` ` description="version 1")` ``` ``` ` resp = self.app.get('/v1/functions/%s/versions/1' % self.func_id)` ``` ``` ` self.assertEqual(200, resp.status_int)` ` self.assertEqual("version 1", resp.json.get('description'))`

#### 33 qinling/tests/unit/storage/test_file_system.py View File

 `@ -150,6 +150,39 @@ class TestFileSystemStorage(base.BaseTest):` ` )` ` exists_mock.assert_called_once_with(package_path)` ``` ``` ` @mock.patch('qinling.storage.file_system.open')` ` @mock.patch('os.path.exists')` ` @mock.patch('os.listdir')` ` def test_retrieve_version(self, mock_list, mock_exist, mock_open):` ` function = "fake_function_id"` ` version = 1` ` md5 = "md5"` ` mock_list.return_value = ["%s_%s_%s.zip" % (function, version, md5)]` ` mock_exist.return_value = True` ``` ``` ` self.storage.retrieve(self.project_id, function, None,` ` version=version)` ``` ``` ` version_zip = os.path.join(FAKE_STORAGE_PATH, self.project_id,` ` "%s_%s_%s.zip" % (function, version, md5))` ``` ``` ` mock_exist.assert_called_once_with(version_zip)` ``` ``` ` @mock.patch('os.listdir')` ` def test_retrieve_version_not_found(self, mock_list):` ` function = "fake_function_id"` ` version = 1` ` mock_list.return_value = [""]` ``` ``` ` self.assertRaises(` ` exc.StorageNotFoundException,` ` self.storage.retrieve,` ` function,` ` self.project_id,` ` None,` ` version=version` ` )` ``` ``` ` @mock.patch('os.path.exists')` ` @mock.patch('os.remove')` ` def test_delete(self, remove_mock, exists_mock):`
{"ft_lang_label":"__label__en","ft_lang_prob":0.5993938,"math_prob":0.487203,"size":855,"snap":"2021-31-2021-39","text_gpt3_token_len":231,"char_repetition_ratio":0.15628672,"word_repetition_ratio":0.0,"special_character_ratio":0.27017543,"punctuation_ratio":0.10769231,"nsfw_num_words":0,"has_unicode_error":false,"math_prob_llama3":0.95542145,"pos_list":[0,1,2],"im_url_duplicate_count":[null,null,null],"WARC_HEADER":"{\"WARC-Type\":\"response\",\"WARC-Date\":\"2021-08-03T15:33:55Z\",\"WARC-Record-ID\":\"<urn:uuid:7cd66ce7-f7b4-4923-b906-116895da53d9>\",\"Content-Length\":\"193757\",\"Content-Type\":\"application/http; msgtype=response\",\"WARC-Warcinfo-ID\":\"<urn:uuid:688378f7-dcde-48c5-8d1e-32a87b546686>\",\"WARC-Concurrent-To\":\"<urn:uuid:ee7424f4-ce81-4a14-95b0-ccfd013eddd4>\",\"WARC-IP-Address\":\"38.108.68.124\",\"WARC-Target-URI\":\"https://opendev.org/openstack/qinling/commit/e8e8084982afe8493cbe04f1a515943315cc5dd8\",\"WARC-Payload-Digest\":\"sha1:JWNSM3Z63KTXBISZEAFDQDPN6ENM5Z76\",\"WARC-Block-Digest\":\"sha1:YBGF5KZVFWHMGVLTICNCM7XZGKGYXAM7\",\"WARC-Identified-Payload-Type\":\"text/html\",\"warc_filename\":\"/cc_download/warc_2021/CC-MAIN-2021-31/CC-MAIN-2021-31_segments_1627046154459.22_warc_CC-MAIN-20210803124251-20210803154251-00620.warc.gz\"}"}