diff options
author | tmartins <thigm85@gmail.com> | 2020-07-02 12:43:29 +0200 |
---|---|---|
committer | tmartins <thigm85@gmail.com> | 2020-07-02 12:43:29 +0200 |
commit | 08db10b5950570373dd13851bebedae0f76f95dc (patch) | |
tree | 48f2d05d2bd51fcca4249739ad872ec2e40079d6 /python | |
parent | 2800e149a2cdcb3ec90ed6725ccbddc9ee5bdd4a (diff) |
add application package serialization code
Diffstat (limited to 'python')
-rw-r--r-- | python/vespa/vespa/package.py | 78 | ||||
-rw-r--r-- | python/vespa/vespa/test_package.py | 81 |
2 files changed, 147 insertions, 12 deletions
diff --git a/python/vespa/vespa/package.py b/python/vespa/vespa/package.py index 195557da338..b234ff1c077 100644 --- a/python/vespa/vespa/package.py +++ b/python/vespa/vespa/package.py @@ -1,5 +1,7 @@ +from time import sleep from typing import List, Mapping, Optional +from jinja2 import Environment, PackageLoader, select_autoescape import docker from vespa.json_serialization import ToJson, FromJson @@ -11,7 +13,7 @@ class Field(ToJson, FromJson["Field"]): name: str, type: str, indexing: Optional[List[str]] = None, - index: Optional[List[str]] = None, + index: Optional[str] = None, ) -> None: """ Object representing a Vespa document field. @@ -26,6 +28,11 @@ class Field(ToJson, FromJson["Field"]): self.indexing = indexing self.index = index + @property + def indexing_to_text(self) -> Optional[str]: + if self.indexing is not None: + return " | ".join(self.indexing) + @staticmethod def from_dict(mapping: Mapping) -> "Field": return Field( @@ -107,6 +114,11 @@ class FieldSet(ToJson, FromJson["FieldSet"]): self.name = name self.fields = fields + @property + def fields_to_text(self): + if self.fields is not None: + return ", ".join(self.fields) + @staticmethod def from_dict(mapping: Mapping) -> "FieldSet": return FieldSet(name=mapping["name"], fields=mapping["fields"]) @@ -231,8 +243,8 @@ class Schema(ToJson, FromJson["Schema"]): return "{0}\n{1}".format(self.__class__.__name__, str(self.to_dict)) -class ApplicationPackage(object): - def __init__(self, name: str, disk_folder: str) -> None: +class ApplicationPackage(ToJson, FromJson["ApplicationPackage"]): + def __init__(self, name: str, schema: Optional[Schema] = None) -> None: """ Vespa Application Package. @@ -240,13 +252,14 @@ class ApplicationPackage(object): :param disk_folder: Absolute path of the folder containing the application files. """ self.name = name - self.disk_folder = disk_folder + self.schema = schema self.container = None - def run_vespa_engine_container(self, container_memory: str = "4G"): + def run_vespa_engine_container(self, disk_folder: str, container_memory: str): """ Run a vespa container. + :param disk_folder: Folder containing the application files. :param container_memory: Memory limit of the container :return: """ @@ -262,7 +275,7 @@ class ApplicationPackage(object): name=self.name, hostname=self.name, privileged=True, - volumes={self.disk_folder: {"bind": "/app", "mode": "rw"}}, + volumes={disk_folder: {"bind": "/app", "mode": "rw"}}, ports={8080: 8080, 19112: 19112}, ) @@ -282,8 +295,59 @@ class ApplicationPackage(object): == "HTTP/1.1 200 OK" ) - def deploy_locally(self): + def deploy_locally(self, disk_folder, container_memory: str = "4G"): + + # todo: use `with tempfile.TemporaryDirectory() as dirpath:` to create a temp dir and write app files there. + self.run_vespa_engine_container( + disk_folder=disk_folder, container_memory=container_memory + ) + + while not self.check_configuration_server(): + print("Waiting for configuration server.") + sleep(5) + deployment = self.container.exec_run( "bash -c '/opt/vespa/bin/vespa-deploy prepare /app/application && /opt/vespa/bin/vespa-deploy activate'" ) return deployment.output.decode("utf-8").split("\n") + + @property + def schema_to_text(self): + env = Environment( + loader=PackageLoader("vespa", "templates"), + autoescape=select_autoescape( + disabled_extensions=("txt",), default_for_string=True, default=True, + ), + ) + env.trim_blocks = True + env.lstrip_blocks = True + schema_template = env.get_template("schema.txt") + return schema_template.render( + schema_name=self.schema.name, + document_name=self.schema.name, + fields=self.schema.document.fields, + fieldsets=self.schema.fieldsets, + rank_profiles=self.schema.rank_profiles, + ) + + @staticmethod + def from_dict(mapping: Mapping) -> "ApplicationPackage": + schema = mapping.get("schema", None) + if schema is not None: + schema = FromJson.map(schema) + return ApplicationPackage(name=mapping["name"], schema=schema) + + @property + def to_dict(self) -> Mapping: + map = {"name": self.name} + if self.schema is not None: + map.update({"schema": self.schema.to_envelope}) + return map + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return self.name == other.name and self.schema == other.schema + + def __repr__(self): + return "{0}\n{1}".format(self.__class__.__name__, str(self.to_dict)) diff --git a/python/vespa/vespa/test_package.py b/python/vespa/vespa/test_package.py index 5bca3e08c2d..917d3470324 100644 --- a/python/vespa/vespa/test_package.py +++ b/python/vespa/vespa/test_package.py @@ -1,6 +1,13 @@ import unittest -from vespa.package import Field, Document, FieldSet, RankProfile, Schema +from vespa.package import ( + Field, + Document, + FieldSet, + RankProfile, + Schema, + ApplicationPackage, +) class TestField(unittest.TestCase): @@ -11,25 +18,26 @@ class TestField(unittest.TestCase): self.assertEqual(field.to_dict, {"name": "test_name", "type": "string"}) self.assertEqual(field, Field(name="test_name", type="string")) self.assertEqual(field, Field.from_dict(field.to_dict)) + self.assertIsNone(field.indexing_to_text) def test_field_name_type_indexing_index(self): field = Field( name="body", type="string", indexing=["index", "summary"], - index=["enable-bm25"], + index="enable-bm25", ) self.assertEqual(field.name, "body") self.assertEqual(field.type, "string") self.assertEqual(field.indexing, ["index", "summary"]) - self.assertEqual(field.index, ["enable-bm25"]) + self.assertEqual(field.index, "enable-bm25") self.assertEqual( field.to_dict, { "name": "body", "type": "string", "indexing": ["index", "summary"], - "index": ["enable-bm25"], + "index": "enable-bm25", }, ) self.assertEqual( @@ -38,10 +46,11 @@ class TestField(unittest.TestCase): name="body", type="string", indexing=["index", "summary"], - index=["enable-bm25"], + index="enable-bm25", ), ) self.assertEqual(field, Field.from_dict(field.to_dict)) + self.assertEqual(field.indexing_to_text, "index | summary") class TestDocument(unittest.TestCase): @@ -80,6 +89,7 @@ class TestFieldSet(unittest.TestCase): self.assertEqual(field_set.name, "default") self.assertEqual(field_set.fields, ["title", "body"]) self.assertEqual(field_set, FieldSet.from_dict(field_set.to_dict)) + self.assertEqual(field_set.fields_to_text, "title, body") class TestRankProfile(unittest.TestCase): @@ -117,3 +127,64 @@ class TestSchema(unittest.TestCase): "default": RankProfile(name="default", first_phase="NativeRank(title)"), }, ) + + +class TestApplicationPackage(unittest.TestCase): + def setUp(self) -> None: + test_schema = Schema( + name="msmarco", + document=Document( + fields=[ + Field(name="id", type="string", indexing=["attribute", "summary"]), + Field( + name="title", + type="string", + indexing=["index", "summary"], + index="enable-bm25", + ), + Field( + name="body", + type="string", + indexing=["index", "summary"], + index="enable-bm25", + ), + ] + ), + fieldsets=[FieldSet(name="default", fields=["title", "body"])], + rank_profiles=[ + RankProfile(name="default", first_phase="nativeRank(title, body)") + ], + ) + self.app_package = ApplicationPackage(name="test_app", schema=test_schema) + + def test_application_package(self): + self.assertEqual( + self.app_package, ApplicationPackage.from_dict(self.app_package.to_dict) + ) + + def test_schema_to_text(self): + expected_result = "schema msmarco {\n" \ + " document msmarco {\n" \ + " field id type string {\n" \ + " indexing: attribute | summary\n" \ + " }\n" \ + " field title type string {\n" \ + " indexing: index | summary\n" \ + " index: enable-bm25\n" \ + " }\n" \ + " field body type string {\n" \ + " indexing: index | summary\n" \ + " index: enable-bm25\n" \ + " }\n" \ + " }\n" \ + " fieldset default {\n" \ + " fields: title, body\n" \ + " }\n" \ + " rank-profile default {\n" \ + " first-phase {\n" \ + " expression: nativeRank(title, body)\n" \ + " }\n" \ + " }\n" \ + "}" + self.assertEqual(self.app_package.schema_to_text, expected_result) + |