diff --git a/aether-kernel/aether/kernel/api/utils.py b/aether-kernel/aether/kernel/api/utils.py index b6882e921..b01d10344 100644 --- a/aether-kernel/aether/kernel/api/utils.py +++ b/aether-kernel/aether/kernel/api/utils.py @@ -21,11 +21,11 @@ import re import string import uuid -from io import BytesIO -import fastavro as avro import jsonpath_ng from jsonpath_ng import parse +from spavro.schema import parse as parse_schema +from spavro.io import validate from django.utils.safestring import mark_safe from pygments import highlight @@ -533,13 +533,12 @@ def merge_objects(source, target, direction): def validate_entity_payload(project_Schema, payload): - # Use fastavro to validate payload against the linked schema + # Use spavro to validate payload against the linked schema try: - # fastavro is primarily for (de)serialization. To test the schema compliance, - # we can just try to serialze the data using the schema. - with BytesIO() as file_obj: - avro.writer(file_obj, project_Schema.schema.definition, [payload]) - # if we didn't get an exception, we're ok! - return + avro_schema = parse_schema(json.dumps(project_Schema.schema.definition, indent=2)) + valid = validate(avro_schema, payload) + if not valid: + raise TypeError('Record did not conform to registered schema.') + return True except Exception as err: raise err diff --git a/aether-kernel/conf/pip/primary-requirements.txt b/aether-kernel/conf/pip/primary-requirements.txt index e58916892..a0f93fc35 100644 --- a/aether-kernel/conf/pip/primary-requirements.txt +++ b/aether-kernel/conf/pip/primary-requirements.txt @@ -8,7 +8,6 @@ # # ################################################################################ -avro-python3 django<2 django-cas-ng django-cors-headers @@ -21,7 +20,6 @@ djangorestframework djangorestframework-filters drf-dynamic-fields drf-openapi -fastavro jsonpath_ng jsonschema psycopg2-binary @@ -29,6 +27,7 @@ pygments python-dateutil raven requests +spavro uwsgi # Aether common module diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt index 0b4ada5dd..4e566275c 100644 --- a/aether-kernel/conf/pip/requirements.txt +++ b/aether-kernel/conf/pip/requirements.txt @@ -10,15 +10,13 @@ ################################################################################ aether.common==0.0.0 -avro-python3==1.8.2 -certifi==2018.1.18 +certifi==2018.4.16 chardet==3.0.4 click==6.7 coreapi==2.3.3 coreschema==0.0.4 coverage==4.5.1 -Cython==0.28.1 -decorator==4.2.1 +decorator==4.3.0 Django==1.11.12 django-cas-ng==3.5.9 django-cors-headers==2.2.0 @@ -34,7 +32,6 @@ djangorestframework-csv==2.1.0 djangorestframework-filters==0.10.2 drf-dynamic-fields==0.3.0 drf-openapi==1.3.0 -fastavro==0.17.10 flake8==3.5.0 flake8-quotes==1.0.0 idna==2.6 @@ -46,7 +43,7 @@ MarkupSafe==1.0 mccabe==0.6.1 mock==2.0.0 openapi-codec==1.3.2 -pbr==4.0.1 +pbr==4.0.2 ply==3.11 psycopg2-binary==2.7.4 pycodestyle==2.3.1 @@ -54,11 +51,12 @@ pyflakes==1.6.0 Pygments==2.2.0 python-cas==1.2.0 python-dateutil==2.7.2 -pytz==2018.3 -raven==6.6.0 +pytz==2018.4 +raven==6.7.0 requests==2.18.4 -simplejson==3.13.2 +simplejson==3.14.0 six==1.11.0 +spavro==1.1.10 sqlparse==0.2.4 unicodecsv==0.14.1 uritemplate==3.0.0 diff --git a/aether-mock-data/test.py b/aether-mock-data/test.py index ea0a86c19..1422d70aa 100644 --- a/aether-mock-data/test.py +++ b/aether-mock-data/test.py @@ -43,8 +43,8 @@ def show_mockfn(): def main(): - person = "http://demo.eha.org/Person" - location = "http://demo.eha.org/GeoLocation" + person = "org.eha.demo.Person" + location = "org.eha.demo.GeoLocation" manager = MockingManager() manager.types[location].override_property( diff --git a/aether-producer/Dockerfile b/aether-producer/Dockerfile index 30467d0d7..1dd4616db 100644 --- a/aether-producer/Dockerfile +++ b/aether-producer/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2.7 +FROM python:3.6 ################################################################################ ## setup container diff --git a/aether-producer/conf/docker/setup.sh b/aether-producer/conf/docker/setup.sh index 02e466a57..41e41e00b 100755 --- a/aether-producer/conf/docker/setup.sh +++ b/aether-producer/conf/docker/setup.sh @@ -28,19 +28,6 @@ set -Eeuox pipefail # Do not buffer stdout so we see log output immediatly export PYTHONUNBUFFERED=true - -################################################################################ -# install packages -################################################################################ - -# Add postgres apt repo to get more recent postgres versions -echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' > /etc/apt/sources.list.d/pgdg.list -wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - -apt-get update -qq -cat /tmp/apt-packages.txt | xargs apt-get -qq --yes --force-yes install - - ################################################################################ # last steps and cleaning ################################################################################ diff --git a/aether-producer/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl b/aether-producer/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl index 4bf510949..dde971d93 100644 Binary files a/aether-producer/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl and b/aether-producer/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl differ diff --git a/aether-producer/conf/pip/primary-requirements.txt b/aether-producer/conf/pip/primary-requirements.txt index d0562fb61..9c652450b 100644 --- a/aether-producer/conf/pip/primary-requirements.txt +++ b/aether-producer/conf/pip/primary-requirements.txt @@ -9,7 +9,6 @@ ################################################################################ aether.client -avro kafka psycopg2-binary requests diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt index a281f5be3..7b38badac 100644 --- a/aether-producer/conf/pip/requirements.txt +++ b/aether-producer/conf/pip/requirements.txt @@ -8,31 +8,23 @@ # then run 'docker-compose run producer pip_freeze' # # # ################################################################################ - aether.client==0.0.0 -avro==1.8.2 -kafka==1.3.5 -psycopg2-binary==2.7.4 -requests==2.18.4 - - -# test libraries +avro-python3==1.8.2 +certifi==2018.4.16 +chardet==3.0.4 coverage==4.5.1 flake8==3.5.0 -flake8-quotes==0.14.0 -mock==2.0.0 - - -## The following requirements were added by pip freeze: -certifi==2018.1.18 -chardet==3.0.4 -configparser==3.5.0 -enum34==1.1.6 -funcsigs==1.0.2 +flake8-quotes==1.0.0 idna==2.6 +kafka==1.3.5 mccabe==0.6.1 -pbr==3.1.1 +mock==2.0.0 +pbr==4.0.2 +psycopg2==2.7.4 +psycopg2-binary==2.7.4 pycodestyle==2.3.1 pyflakes==1.6.0 +requests==2.18.4 six==1.11.0 +spavro==1.1.15 urllib3==1.22 diff --git a/aether-producer/producer/Pipfile b/aether-producer/producer/Pipfile index 6479107ad..6af287cec 100644 --- a/aether-producer/producer/Pipfile +++ b/aether-producer/producer/Pipfile @@ -4,9 +4,9 @@ url = "https://pypi.python.org/simple" [packages] kafka = "*" -py3-psycopg2 = "*" +psycopg2-binary = "*" requests = "*" -avro-python3 = "*" +spavro = "*" [dev-packages] "-e /aether-utils/aether-client" = "*" diff --git a/aether-producer/producer/aether_producer.py b/aether-producer/producer/aether_producer.py index 317e1d968..f4bd60591 100644 --- a/aether-producer/producer/aether_producer.py +++ b/aether-producer/producer/aether_producer.py @@ -16,18 +16,21 @@ # specific language governing permissions and limitations # under the License. -import json -import psycopg2 -import avro.schema -import io import ast +import io +import json import os +import psycopg2 import signal +import avro.schema import sys -from avro.io import DatumWriter from avro.datafile import DataFileWriter -from kafka import KafkaProducer +from avro.io import DatumWriter +from avro.io import Validate + + +from kafka import KafkaProducer, KafkaConsumer from aether.client import KernelClient from psycopg2.extras import DictCursor from time import sleep as Sleep @@ -52,28 +55,28 @@ def load(self, path): self.offset_path = "%s/%s" % (FILE_PATH, self.offset_file) -''' -SETTINGS = load_settings(SETTINGS_FILE) - -def init_settings(SETTINGS): - global - KAFKA_SERVER = SETTINGS.get('kafka_server') - jdbc_connection_string = SETTINGS.get('jdbc_connection_string') - jdbc_user = SETTINGS.get('jdbc_user') - SLEEP_TIME = SETTINGS.get('sleep_time') # seconds between looking for changes - kernel_url = SETTINGS.get("kernel_url") - kernel_credentials = SETTINGS.get('kernel_credentials') - postgres_connection_info = SETTINGS.get('postgres_connection_info') -''' - -def connect(): - try: - kernel = KernelClient(url=_settings.kernel_url, **_settings.kernel_credentials) - return kernel - except Exception as e: - kernel = None - print ("Error initializing connection to Aether: %s" % e) - raise e +def connect(_settings, retry=3): + kernel = None + consumer = None + for x in range(retry): + try: + kernel = KernelClient(url=_settings.kernel_url, **_settings.kernel_credentials) + break + except Exception as e: + Sleep(_settings.start_delay) + if not kernel: + print("No connection to AetherKernel") + sys.exit(1) + for x in range(retry): + try: + consumer = KafkaConsumer(bootstrap_servers=_settings.kafka_server) + break + except Exception as err: + Sleep(_settings.start_delay) + if not consumer: + print("No connection to Kafka") + sys.exit(1) + return kernel def set_offset_value(key, value): @@ -103,47 +106,63 @@ def get_offset(key): return None -def count_since(offset=None): +def count_since(offset=None, topic=None): if not offset: offset = "" with psycopg2.connect(**_settings.postgres_connection_info) as conn: cursor = conn.cursor(cursor_factory=DictCursor) + # We'd originally used a 'count(CASE WHEN e.modified >' + # That broke mysteriously and borked everything so we're using a less optimal call. + # Benchmarks show it good enough for now but probably should be fixed # TODO count_str = ''' SELECT - count(CASE WHEN e.modified > '%s' THEN 1 END) as new_rows - FROM kernel_entity e; + e.id, + e.modified, + ps.name as project_schema_name, + ps.id as project_schema_id, + s.name as schema_name, + s.id as schema_id + FROM kernel_entity e + inner join kernel_projectschema ps on e.projectschema_id = ps.id + inner join kernel_schema s on ps.schema_id = s.id + WHERE e.modified > '%s' ''' % (offset) + if topic: + count_str += '''AND ps.name = '%s' + ORDER BY e.modified ASC;''' % topic + else: + count_str += '''ORDER BY e.modified ASC;''' cursor.execute(count_str); - for row in cursor: - return row.get("new_rows") + return sum([1 for row in cursor]) -def get_entities(offset = None): +def get_entities(offset = None, max_size=1000): # TODO implement batch pull by topic in Stream if not offset: offset = "" - conn = psycopg2.connect(**_settings.postgres_connection_info) - cursor = conn.cursor(cursor_factory=DictCursor) - query_str = ''' - SELECT - e.id, - e.revision, - e.payload, - e.modified, - e.status, - ps.id as project_schema_id, - ps.name as project_schema_name, - s.name as schema_name, - s.id as schema_id, - s.revision as schema_revision - from kernel_entity e - inner join kernel_projectschema ps on e.projectschema_id = ps.id - inner join kernel_schema s on ps.schema_id = s.id - WHERE e.modified > '%s' - ORDER BY e.modified ASC; - ''' % (offset) - cursor.execute(query_str) - for row in cursor: - yield {key : row[key] for key in row.keys()} + with psycopg2.connect(**_settings.postgres_connection_info) as conn: + cursor = conn.cursor(cursor_factory=DictCursor) + query_str = ''' + SELECT + e.id, + e.revision, + e.payload, + e.modified, + e.status, + ps.id as project_schema_id, + ps.name as project_schema_name, + s.name as schema_name, + s.id as schema_id, + s.revision as schema_revision + FROM kernel_entity e + inner join kernel_projectschema ps on e.projectschema_id = ps.id + inner join kernel_schema s on ps.schema_id = s.id + WHERE e.modified > '%s' + ORDER BY e.modified ASC + LIMIT %d; + ''' % (offset, max_size) + cursor.execute(query_str) + for row in cursor: + yield {key : row[key] for key in row.keys()} class KafkaStream(object): @@ -159,6 +178,10 @@ def __init__(self, topic, kernel): def send(self, row): msg = row.get("payload") offset = row.get("modified") + bytes_writer = io.BytesIO() + valid = Validate(self.schema, msg) + if not valid: + raise ValueError("message doesn't adhere to schema \n%s\n%s" % (json.dumps(self.schema), json.dumps(msg))) try: bytes_writer = io.BytesIO() writer = DataFileWriter(bytes_writer, DatumWriter(), self.schema, codec='deflate') @@ -185,7 +208,7 @@ def get_avro(self): #Gets avro schema used for encoding messages #TODO Fix issue with json coming from API Client being single quoted definition = ast.literal_eval(str(self.kernel.Resource.Schema.get(self.topic).definition)) - self.schema = avro.schema.parse(json.dumps(definition)) + self.schema = avro.schema.Parse(json.dumps(definition)) def stop(self): @@ -205,7 +228,6 @@ def __init__(self, kernel): self.start() - def start(self): self.kernel.refresh() topics = self.kernel.Resource.Schema @@ -230,23 +252,14 @@ def stop(self): self.streams = {} print ("manager stopped") - def kill(self): + def kill(self, *args, **kwargs): self.killed = True def main_loop(test=False): global _settings _settings = Settings(test) manager = None - kernel = None - for x in range(3): - try: - kernel = connect() - break - except Exception as err: - Sleep(_settings.start_delay) - if not kernel: - sys.exit(1) - print("Producer Connected to Aether.") + kernel = connect(_settings, retry=3) try: while True: offset = get_offset("entities") @@ -262,17 +275,17 @@ def main_loop(test=False): break manager = None else: - print("Sleeping for %s" % (_settings.sleep_time)) + # print("Sleeping for %s" % (_settings.sleep_time)) Sleep(_settings.sleep_time) except KeyboardInterrupt as ek: print ("Caught Keyboard interrupt") if manager: print ("Trying to kill manager") manager.stop() - except Exception as e: - print(e) + finally: if manager: manager.stop() + if __name__ == "__main__": main_loop() diff --git a/aether-utils/aether-client/dist/aether.client-0.0.0-py2.py3-none-any.whl b/aether-utils/aether-client/dist/aether.client-0.0.0-py2.py3-none-any.whl index 4bf510949..dde971d93 100644 Binary files a/aether-utils/aether-client/dist/aether.client-0.0.0-py2.py3-none-any.whl and b/aether-utils/aether-client/dist/aether.client-0.0.0-py2.py3-none-any.whl differ diff --git a/aether-utils/aether-mock-data/Pipfile b/aether-utils/aether-mock-data/Pipfile index 762ac726a..fe330fec9 100644 --- a/aether-utils/aether-mock-data/Pipfile +++ b/aether-utils/aether-mock-data/Pipfile @@ -6,7 +6,8 @@ verify_ssl = true [packages] -aether = {path = "./../aether-client", editable = true} +"aether.client" = {path = "./../aether-client", editable = true} +"aether.mocker" = {path = ".", editable = true} [dev-packages] diff --git a/aether-utils/aether-mock-data/aether/mocker.py b/aether-utils/aether-mock-data/aether/mocker.py index 6c1ebdb74..6c4f23dc4 100644 --- a/aether-utils/aether-mock-data/aether/mocker.py +++ b/aether-utils/aether-mock-data/aether/mocker.py @@ -437,6 +437,7 @@ def load(self): # loads schemas and project schemas from aether client for schema in self.client.Resource.Schema: name = schema.get("name") + print("Load %s" % name) _id = schema.get('id') definition = schema.get('definition') full_name = [obj.get("name") for obj in definition if obj.get( diff --git a/aether-utils/aether-mock-data/setup.py b/aether-utils/aether-mock-data/setup.py index f37fcfc22..7c8d5fb75 100644 --- a/aether-utils/aether-mock-data/setup.py +++ b/aether-utils/aether-mock-data/setup.py @@ -30,7 +30,7 @@ name='aether.mocker', version='0.0.0', install_requires=[ - "aether-client" + "aether.client" ], packages=find_packages(), namespace_packages=['aether'] diff --git a/aether-utils/aether-mock-data/test.py b/aether-utils/aether-mock-data/test.py index ea0a86c19..0cc2c5109 100644 --- a/aether-utils/aether-mock-data/test.py +++ b/aether-utils/aether-mock-data/test.py @@ -17,7 +17,7 @@ # under the License. import json -from mocker import MockFn, DataMocker, Generic, MockingManager +from aether.mocker import MockFn, DataMocker, Generic, MockingManager def pprint(obj): @@ -43,8 +43,8 @@ def show_mockfn(): def main(): - person = "http://demo.eha.org/Person" - location = "http://demo.eha.org/GeoLocation" + person = "org.eha.demo.Person" + location = "org.eha.demo.GeoLocation" manager = MockingManager() manager.types[location].override_property( diff --git a/aether-utils/aether-saladbar/.gitignore b/aether-utils/aether-saladbar/.gitignore index fea5fed16..8aab741fb 100644 --- a/aether-utils/aether-saladbar/.gitignore +++ b/aether-utils/aether-saladbar/.gitignore @@ -1,8 +1,9 @@ build/ dist/ aether_saladbar.egg-info/ -saladbar/schemas/* -saladbar/test-schemas/* -!saladbar/test-schemas/.gitignore -!saladbar/schemas/.gitignore +aether/saladbar/schemas/* +aether/saladbar/test-schemas/* +!aether/saladbar/library/*/*/json +!aether/saladbar/test-schemas/.gitignore +!aether/saladbar/schemas/.gitignore Pipfile.lock diff --git a/aether-utils/aether-saladbar/Dockerfile b/aether-utils/aether-saladbar/Dockerfile index 43b10fea7..e3e744730 100644 --- a/aether-utils/aether-saladbar/Dockerfile +++ b/aether-utils/aether-saladbar/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2.7 +FROM python:3.6 ################################################################################ ## install app diff --git a/aether-utils/aether-saladbar/MANIFEST.in b/aether-utils/aether-saladbar/MANIFEST.in index c9cb0beff..0126caf55 100644 --- a/aether-utils/aether-saladbar/MANIFEST.in +++ b/aether-utils/aether-saladbar/MANIFEST.in @@ -1,2 +1,2 @@ -recursive-include saladbar * +recursive-include aether * diff --git a/aether-utils/aether-saladbar/saladbar/saladbar/__init__.py b/aether-utils/aether-saladbar/aether/__init__.py similarity index 82% rename from aether-utils/aether-saladbar/saladbar/saladbar/__init__.py rename to aether-utils/aether-saladbar/aether/__init__.py index a13bf21c8..665f441ee 100644 --- a/aether-utils/aether-saladbar/saladbar/saladbar/__init__.py +++ b/aether-utils/aether-saladbar/aether/__init__.py @@ -15,3 +15,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + + +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + __path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/aether-utils/aether-saladbar/saladbar/Pipfile b/aether-utils/aether-saladbar/aether/saladbar/Pipfile similarity index 55% rename from aether-utils/aether-saladbar/saladbar/Pipfile rename to aether-utils/aether-saladbar/aether/saladbar/Pipfile index e08ac6ead..1e3052ae7 100644 --- a/aether-utils/aether-saladbar/saladbar/Pipfile +++ b/aether-utils/aether-saladbar/aether/saladbar/Pipfile @@ -8,7 +8,9 @@ verify_ssl = true requests = "*" schema-salad = "*" -aether = {path = "./../../aether-client", editable = true} +"aether.saladbar" = {path = "../..", editable = true} +"aether.client" = {path = "../../../aether-client", editable = true} + [dev-packages] diff --git a/aether-utils/aether-saladbar/saladbar/README.txt b/aether-utils/aether-saladbar/aether/saladbar/README.txt similarity index 100% rename from aether-utils/aether-saladbar/saladbar/README.txt rename to aether-utils/aether-saladbar/aether/saladbar/README.txt diff --git a/aether-utils/aether-saladbar/saladbar/__init__.py b/aether-utils/aether-saladbar/aether/saladbar/__init__.py similarity index 100% rename from aether-utils/aether-saladbar/saladbar/__init__.py rename to aether-utils/aether-saladbar/aether/saladbar/__init__.py diff --git a/aether-utils/aether-saladbar/saladbar/conf/_settings.json b/aether-utils/aether-saladbar/aether/saladbar/conf/_settings.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/conf/_settings.json rename to aether-utils/aether-saladbar/aether/saladbar/conf/_settings.json diff --git a/aether-utils/aether-saladbar/saladbar/conf/settings.json b/aether-utils/aether-saladbar/aether/saladbar/conf/settings.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/conf/settings.json rename to aether-utils/aether-saladbar/aether/saladbar/conf/settings.json diff --git a/aether-utils/aether-saladbar/saladbar/conf/test_settings.json b/aether-utils/aether-saladbar/aether/saladbar/conf/test_settings.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/conf/test_settings.json rename to aether-utils/aether-saladbar/aether/saladbar/conf/test_settings.json diff --git a/aether-utils/aether-saladbar/saladbar/conf/types.json b/aether-utils/aether-saladbar/aether/saladbar/conf/types.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/conf/types.json rename to aether-utils/aether-saladbar/aether/saladbar/conf/types.json diff --git a/aether-utils/aether-saladbar/saladbar/example/extends.json b/aether-utils/aether-saladbar/aether/saladbar/example/extends.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/example/extends.json rename to aether-utils/aether-saladbar/aether/saladbar/example/extends.json diff --git a/aether-utils/aether-saladbar/aether/saladbar/library/__init__.py b/aether-utils/aether-saladbar/aether/saladbar/library/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aether-utils/aether-saladbar/saladbar/library/library_utils.py b/aether-utils/aether-saladbar/aether/saladbar/library/library_utils.py similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/library_utils.py rename to aether-utils/aether-saladbar/aether/saladbar/library/library_utils.py diff --git a/aether-utils/aether-saladbar/aether/saladbar/library/src/.gitignore b/aether-utils/aether-saladbar/aether/saladbar/library/src/.gitignore new file mode 100644 index 000000000..b6b93549c --- /dev/null +++ b/aether-utils/aether-saladbar/aether/saladbar/library/src/.gitignore @@ -0,0 +1,2 @@ +!eha.org/* +!schema.org/* diff --git a/aether-utils/aether-saladbar/saladbar/library/src/eha.org/demo/library.json b/aether-utils/aether-saladbar/aether/saladbar/library/src/eha.org/demo/library.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/src/eha.org/demo/library.json rename to aether-utils/aether-saladbar/aether/saladbar/library/src/eha.org/demo/library.json diff --git a/aether-utils/aether-saladbar/saladbar/library/src/eha.org/demo/properties.csv b/aether-utils/aether-saladbar/aether/saladbar/library/src/eha.org/demo/properties.csv similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/src/eha.org/demo/properties.csv rename to aether-utils/aether-saladbar/aether/saladbar/library/src/eha.org/demo/properties.csv diff --git a/aether-utils/aether-saladbar/saladbar/library/src/eha.org/demo/scrape.json b/aether-utils/aether-saladbar/aether/saladbar/library/src/eha.org/demo/scrape.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/src/eha.org/demo/scrape.json rename to aether-utils/aether-saladbar/aether/saladbar/library/src/eha.org/demo/scrape.json diff --git a/aether-utils/aether-saladbar/saladbar/library/src/eha.org/demo/types.csv b/aether-utils/aether-saladbar/aether/saladbar/library/src/eha.org/demo/types.csv similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/src/eha.org/demo/types.csv rename to aether-utils/aether-saladbar/aether/saladbar/library/src/eha.org/demo/types.csv diff --git a/aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/all-layers-enumvalues.csv b/aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/all-layers-enumvalues.csv similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/all-layers-enumvalues.csv rename to aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/all-layers-enumvalues.csv diff --git a/aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/all-layers-properties.csv b/aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/all-layers-properties.csv similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/all-layers-properties.csv rename to aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/all-layers-properties.csv diff --git a/aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/all-layers-types.csv b/aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/all-layers-types.csv similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/all-layers-types.csv rename to aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/all-layers-types.csv diff --git a/aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/library.json b/aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/library.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/library.json rename to aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/library.json diff --git a/aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/scrape.json b/aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/scrape.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/src/schema.org/3.3/scrape.json rename to aether-utils/aether-saladbar/aether/saladbar/library/src/schema.org/3.3/scrape.json diff --git a/aether-utils/aether-saladbar/saladbar/library/__init__.py b/aether-utils/aether-saladbar/aether/saladbar/saladbar/__init__.py similarity index 100% rename from aether-utils/aether-saladbar/saladbar/library/__init__.py rename to aether-utils/aether-saladbar/aether/saladbar/saladbar/__init__.py diff --git a/aether-utils/aether-saladbar/aether/saladbar/saladbar/parsers/__init__.py b/aether-utils/aether-saladbar/aether/saladbar/saladbar/parsers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aether-utils/aether-saladbar/saladbar/saladbar/parsers/schemadotorg.py b/aether-utils/aether-saladbar/aether/saladbar/saladbar/parsers/schemadotorg.py similarity index 100% rename from aether-utils/aether-saladbar/saladbar/saladbar/parsers/schemadotorg.py rename to aether-utils/aether-saladbar/aether/saladbar/saladbar/parsers/schemadotorg.py diff --git a/aether-utils/aether-saladbar/saladbar/saladbar/salad_handler.py b/aether-utils/aether-saladbar/aether/saladbar/saladbar/salad_handler.py similarity index 56% rename from aether-utils/aether-saladbar/saladbar/saladbar/salad_handler.py rename to aether-utils/aether-saladbar/aether/saladbar/saladbar/salad_handler.py index 51a745fa0..9ab5598fa 100644 --- a/aether-utils/aether-saladbar/saladbar/saladbar/salad_handler.py +++ b/aether-utils/aether-saladbar/aether/saladbar/saladbar/salad_handler.py @@ -27,6 +27,71 @@ def pprint(obj): print(json.dumps(obj, indent=2)) +################################################################################## +# Utility functions to make namespaces Avro Compliant +################################################################################## + +def replace_url(test): + if not isinstance(test, str): + return test + flag = "http://" + if not flag in test: + return test + test = test.replace(flag, "") # cut out http:// + parts = test.split("/") + if not parts: + # just namespace + try: + parts = test.split(".") + return ".".join([i for i in parts[::-1]]) + except Exception: + return test + namespace = parts[0] + path = None + if parts[-1] != namespace: + path = ".".join(parts[1:]) + # reverse namespace + parts = namespace.split(".") + if len(parts) > 1: + if parts[0] not in ["org", "com", "net", "gov"]: + namespace = ".".join([i for i in parts[::-1]]) + return ".".join([namespace,path]) + + + +#we recurse until the object is just replaced strings or ignored items +def re_namespace(obj): + if isinstance(obj, str): + return replace_url(obj) + if isinstance(obj, list): + return [re_namespace(i) for i in obj] + if isinstance(obj, dict): + return {re_namespace(k): re_namespace(v) for k, v in obj.items()} + return obj + + +def check_namespace(fn): + def wrapper(*args, **kwargs): + out = fn(*args, **kwargs) + return re_namespace(out) + return wrapper + + +# for some reason the schema salad lib is adding an empty name field to array objects. +# It's useless so we to delete them. +def handle_empty_fields(obj): + if isinstance(obj, str): + return + if isinstance(obj, list): + return [handle_empty_fields(i) for i in obj] + if isinstance(obj, dict): + for k in list(obj.keys()): + v = obj[k] + if not v: + del obj[k] + else: + handle_empty_fields(v) + class SaladHandler(object): @@ -83,6 +148,7 @@ def get_avro(self, depends=None): if not depends: return avsc_names, avsc_obj avsc_dict = {i.get('name'): i for i in avsc_obj} + handle_empty_fields(avsc_dict) pprint([i for i in avsc_dict.keys()]) out = {} deps = depends.keys() @@ -93,8 +159,26 @@ def get_avro(self, depends=None): # unpack nested values all_props = [i for j in reqs.get( 'properties').values() for i in j] - avro_item = [ + items = [ avsc_dict.get(i) for i in all_props if i in avsc_dict.keys()] - avro_item.append(avsc_dict.get(name)) + # split names and namespaces + prime = avsc_dict.get(name) + del prime['extends'] + prime['aetherBaseSchema'] = True + # del avro_item['extends'] + items.append(prime) + avro_item = [] + for i in items: + i = re_namespace(i) + parts = i.get('name').split(".") + print(parts) + i['label'] = parts[-1] + i['name'] = ".".join(parts) + i['namespace'] = ".".join(parts[:-1]) + if i.get('aetherBaseSchema') is True: + name = i.get('label') + avro_item.append(i) + + out[name] = avro_item return out diff --git a/aether-utils/aether-saladbar/saladbar/test-schemas/.gitignore b/aether-utils/aether-saladbar/aether/saladbar/test-schemas/.gitignore similarity index 100% rename from aether-utils/aether-saladbar/saladbar/test-schemas/.gitignore rename to aether-utils/aether-saladbar/aether/saladbar/test-schemas/.gitignore diff --git a/aether-utils/aether-saladbar/saladbar/wizard.py b/aether-utils/aether-saladbar/aether/saladbar/wizard.py similarity index 89% rename from aether-utils/aether-saladbar/saladbar/wizard.py rename to aether-utils/aether-saladbar/aether/saladbar/wizard.py index 7cb0f5610..bb26ab4c3 100755 --- a/aether-utils/aether-saladbar/saladbar/wizard.py +++ b/aether-utils/aether-saladbar/aether/saladbar/wizard.py @@ -26,9 +26,9 @@ from aether.client import KernelClient -from library import library_utils -import saladbar.parsers as Parsers -from saladbar import salad_handler as salad +import aether.saladbar.library.library_utils as library_utils +import aether.saladbar.saladbar.parsers as Parsers +import aether.saladbar.saladbar.salad_handler as salad HERE = (os.path.dirname(os.path.realpath(__file__))) SETTINGS = "%s/conf/settings.json" % HERE @@ -38,6 +38,10 @@ SCHEMAS = "%s/schemas/" % HERE TEST_SCHEMAS = "%s/test-schemas/" % HERE +# alais raw_input for python3 +if sys.version_info >= (3, 0): + def raw_input(prompt): + return input(prompt) def pprint(obj): print(json.dumps(obj, indent=2)) @@ -105,6 +109,7 @@ def make_base_salad_doc(imports, namespaces, project=None, base_type=None, schem graph = [{"$import": i} for i in imports] graph.append(base_doc) doc['$graph'] = graph + pprint(graph) project_file = "%s%s.json" % (schema_path, project) with open(project_file, "w") as f: json.dump(doc, f, indent=2) @@ -212,10 +217,13 @@ def test_setup(): make_base_salad_doc(imports, namespaces, project, base_type, schema_path=TEST_SCHEMAS) project_file = "%s%s.json" % (TEST_SCHEMAS, project) salad_handler = salad.SaladHandler(project_file) + salad_doc = salad_handler.schema_doc + salad_file = "%s%s.salad" % (TEST_SCHEMAS, project) + with open(salad_file, "w") as f: + json.dump(salad_doc, f, indent=2) avsc_dict = salad_handler.get_avro(all_depends) for k, v in avsc_dict.items(): - # Fix this .org nonsense... - filename = "%s%s.avsc" % (TEST_SCHEMAS, k.split(".org/")[1]) + filename = "%s%s.avsc" % (TEST_SCHEMAS, k.split(".")[-1]) with open(filename, "w") as f: json.dump(v, f, indent=2) kernel_url = settings.get("kernel_url") @@ -251,21 +259,27 @@ def main(): make_base_salad_doc(imports, namespaces, project, base_type) project_file = "%s%s.json" % (SCHEMAS, project) salad_handler = salad.SaladHandler(project_file) + salad_doc = salad_handler.schema_doc + salad_file = "%s%s.salad" % (SCHEMAS, project) + with open(salad_file, "w") as f: + json.dump(salad_doc, f, indent=2) avsc_dict = salad_handler.get_avro(all_depends) + pprint(avsc_dict) for k, v in avsc_dict.items(): # Fix this .org nonsense... - filename = "%s%s.avsc" % (SCHEMAS, k.split(".org/")[1]) + filename = "%s%s.avsc" % (SCHEMAS, k) with open(filename, "w") as f: json.dump(v, f, indent=2) - kernel_url = settings.get("kernel_url") - kernel_user = settings.get("kernel_user") - kernel_pw = settings.get("kernel_pw") - kernel_credentials = {"username": kernel_user, "password": kernel_pw} - client = KernelClient(url=kernel_url, **kernel_credentials) ok = ask( "Setup of project titled: %s complete. Register generated schemas with Aether?" % (project)) + if ok: + kernel_url = settings.get("kernel_url") + kernel_user = settings.get("kernel_user") + kernel_pw = settings.get("kernel_pw") + kernel_credentials = {"username": kernel_user, "password": kernel_pw} + client = KernelClient(url=kernel_url, **kernel_credentials) register_project(client, project) register_schemas(client, project) prompt_schema_clean( diff --git a/aether-utils/aether-saladbar/saladbar/wizard_resources/base_model.json b/aether-utils/aether-saladbar/aether/saladbar/wizard_resources/base_model.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/wizard_resources/base_model.json rename to aether-utils/aether-saladbar/aether/saladbar/wizard_resources/base_model.json diff --git a/aether-utils/aether-saladbar/saladbar/wizard_resources/base_salad.json b/aether-utils/aether-saladbar/aether/saladbar/wizard_resources/base_salad.json similarity index 100% rename from aether-utils/aether-saladbar/saladbar/wizard_resources/base_salad.json rename to aether-utils/aether-saladbar/aether/saladbar/wizard_resources/base_salad.json diff --git a/aether-utils/aether-saladbar/entrypoint.sh b/aether-utils/aether-saladbar/entrypoint.sh index d5ed707e5..b546150ee 100755 --- a/aether-utils/aether-saladbar/entrypoint.sh +++ b/aether-utils/aether-saladbar/entrypoint.sh @@ -106,11 +106,11 @@ case "$1" in ;; start ) - ./saladbar/wizard.py + ./aether/saladbar/wizard.py ;; start_test ) - ./saladbar/wizard.py test + ./aether/saladbar/wizard.py test ;; help) diff --git a/aether-utils/aether-saladbar/saladbar/library/src/.gitignore b/aether-utils/aether-saladbar/saladbar/library/src/.gitignore deleted file mode 100644 index c7f39f4aa..000000000 --- a/aether-utils/aether-saladbar/saladbar/library/src/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!demo/* diff --git a/aether-utils/aether-saladbar/saladbar/saladbar/parsers/__init__.py b/aether-utils/aether-saladbar/saladbar/saladbar/parsers/__init__.py deleted file mode 100644 index a13bf21c8..000000000 --- a/aether-utils/aether-saladbar/saladbar/saladbar/parsers/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (C) 2018 by eHealth Africa : http://www.eHealthAfrica.org -# -# See the NOTICE file distributed with this work for additional information -# regarding copyright ownership. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on anx -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/aether-utils/aether-saladbar/setup.py b/aether-utils/aether-saladbar/setup.py index 3ec185d65..3add48614 100644 --- a/aether-utils/aether-saladbar/setup.py +++ b/aether-utils/aether-saladbar/setup.py @@ -20,18 +20,16 @@ from setuptools import setup, find_packages import sys -if sys.version_info > (3,0): - sys.exit('saladbar requires Python 2.x for schema_salad_tool') - # allow setup.py to be run from any path os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) setup( - name='aether-saladbar', + name='aether.saladbar', version='0.0.0', install_requires=[ "schema-salad" ], include_package_data=True, packages=find_packages(), + namespace_packages=['aether'] ) diff --git a/docker-compose.yml b/docker-compose.yml index b60303944..42e841108 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,10 @@ version: "2.1" networks: - internal: - driver: bridge + internal: # alias as internal + external: + name: aether_internal # if this isn't explicitly set it inherits from the containing folder name (aether or not) + services: diff --git a/scripts/build_aether_utils_and_distribute.sh b/scripts/build_aether_utils_and_distribute.sh index 9d3ca97e1..5df5c8c18 100755 --- a/scripts/build_aether_utils_and_distribute.sh +++ b/scripts/build_aether_utils_and_distribute.sh @@ -68,7 +68,7 @@ done $DC_CLIENT build saladbar $DC_CLIENT run saladbar build -PCK_FILE=aether_saladbar-0.0.0-py2.py3-none-any.whl +PCK_FILE=aether.saladbar-0.0.0-py2.py3-none-any.whl # distribute within the containers containers=( test-aether-integration ) diff --git a/test-aether-integration-module/Dockerfile b/test-aether-integration-module/Dockerfile index 7e0e6ad7a..fe5378d53 100644 --- a/test-aether-integration-module/Dockerfile +++ b/test-aether-integration-module/Dockerfile @@ -4,12 +4,6 @@ FROM python:3.6 ## setup container ################################################################################ -#install pip2 -RUN wget https://bootstrap.pypa.io/get-pip.py -O - | python2 - -RUN pip2 --version -RUN pip3 --version - ################################################################################ ## install app ## copy files one by one and split commands to use docker cache @@ -19,8 +13,7 @@ WORKDIR /code COPY ./conf/pip /code/conf/pip -RUN pip3 install -f /code/conf/pip/dependencies -r /code/conf/pip/requirements.py3.txt -RUN pip2 install -f /code/conf/pip/dependencies -r /code/conf/pip/requirements.py2.txt +RUN pip3 install -f /code/conf/pip/dependencies -r /code/conf/pip/requirements.txt COPY ./ /code diff --git a/test-aether-integration-module/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl b/test-aether-integration-module/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl index 4bf510949..dde971d93 100644 Binary files a/test-aether-integration-module/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl and b/test-aether-integration-module/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl differ diff --git a/test-aether-integration-module/conf/pip/dependencies/aether.mocker-0.0.0-py2.py3-none-any.whl b/test-aether-integration-module/conf/pip/dependencies/aether.mocker-0.0.0-py2.py3-none-any.whl index cdd8238d5..333c05367 100644 Binary files a/test-aether-integration-module/conf/pip/dependencies/aether.mocker-0.0.0-py2.py3-none-any.whl and b/test-aether-integration-module/conf/pip/dependencies/aether.mocker-0.0.0-py2.py3-none-any.whl differ diff --git a/test-aether-integration-module/conf/pip/dependencies/aether_saladbar-0.0.0-py2.py3-none-any.whl b/test-aether-integration-module/conf/pip/dependencies/aether.saladbar-0.0.0-py2.py3-none-any.whl similarity index 83% rename from test-aether-integration-module/conf/pip/dependencies/aether_saladbar-0.0.0-py2.py3-none-any.whl rename to test-aether-integration-module/conf/pip/dependencies/aether.saladbar-0.0.0-py2.py3-none-any.whl index fd68bf520..6faa5aff1 100644 Binary files a/test-aether-integration-module/conf/pip/dependencies/aether_saladbar-0.0.0-py2.py3-none-any.whl and b/test-aether-integration-module/conf/pip/dependencies/aether.saladbar-0.0.0-py2.py3-none-any.whl differ diff --git a/test-aether-integration-module/conf/pip/primary-requirements.py2.txt b/test-aether-integration-module/conf/pip/primary-requirements.py2.txt deleted file mode 100644 index 38fde6717..000000000 --- a/test-aether-integration-module/conf/pip/primary-requirements.py2.txt +++ /dev/null @@ -1,19 +0,0 @@ -################################################################################################################# -# # -# Primary requirements # -# # -# CHANGE ME !!! # -# # -# Update me and then run 'docker-compose -f docker-compose-test.yml run integration-test pip_freeze' # -# # -################################################################################################################# - -aether.client -aether_saladbar -coverage -flake8 -mock -pytest -pytest-runner -requests - diff --git a/test-aether-integration-module/conf/pip/primary-requirements.py3.txt b/test-aether-integration-module/conf/pip/primary-requirements.txt similarity index 98% rename from test-aether-integration-module/conf/pip/primary-requirements.py3.txt rename to test-aether-integration-module/conf/pip/primary-requirements.txt index 67b19a9e9..10e1e501d 100644 --- a/test-aether-integration-module/conf/pip/primary-requirements.py3.txt +++ b/test-aether-integration-module/conf/pip/primary-requirements.txt @@ -11,7 +11,7 @@ aether.client aether.mocker -avro-python3 +aether.saladbar coverage flake8 kafka diff --git a/test-aether-integration-module/conf/pip/requirements.py3.txt b/test-aether-integration-module/conf/pip/requirements.py3.txt deleted file mode 100644 index a5c4d4bbd..000000000 --- a/test-aether-integration-module/conf/pip/requirements.py3.txt +++ /dev/null @@ -1,38 +0,0 @@ -######################################################################################### -# # -# Frozen requirements # -# # -# DO NOT CHANGE !!! # -# # -# To update this first update 'primary-requirements.txt' # -# then run 'docker-compose -f docker-compose-test.yml run integration-test pip_freeze' # -# # -######################################################################################### -aether.client==0.0.0 -aether.mocker==0.0.0 -attrs==17.4.0 -avro-python3==1.8.2 -certifi==2018.1.18 -chardet==3.0.4 -configobj==5.0.6 -configparser==3.5.0 -coverage==4.5.1 -enum34==1.1.6 -flake8==3.5.0 -funcsigs==1.0.2 -idna==2.6 -kafka==1.3.5 -mccabe==0.6.1 -mock==2.0.0 -more-itertools==4.1.0 -pbr==4.0.1 -pluggy==0.6.0 -py==1.5.3 -pycodestyle==2.3.1 -pyflakes==1.6.0 -pytest==3.5.0 -pytest-runner==4.2 -requests==2.18.4 -six==1.11.0 -spavro==1.1.10 -urllib3==1.22 diff --git a/test-aether-integration-module/conf/pip/requirements.py2.txt b/test-aether-integration-module/conf/pip/requirements.txt similarity index 90% rename from test-aether-integration-module/conf/pip/requirements.py2.txt rename to test-aether-integration-module/conf/pip/requirements.txt index 79adb9b48..cf74cc244 100644 --- a/test-aether-integration-module/conf/pip/requirements.py2.txt +++ b/test-aether-integration-module/conf/pip/requirements.txt @@ -8,13 +8,13 @@ # then run 'docker-compose -f docker-compose-test.yml run integration-test pip_freeze' # # # ######################################################################################### -aether-saladbar==0.0.0 aether.client==0.0.0 +aether.mocker==0.0.0 +aether.saladbar==0.0.0 attrs==17.4.0 -avro==1.8.1 -bzr==2.7.0.dev1 +avro-cwl==1.8.4 CacheControl==0.11.7 -certifi==2018.1.18 +certifi==2018.4.16 chardet==3.0.4 configobj==5.0.6 configparser==3.5.0 @@ -22,28 +22,29 @@ coverage==4.5.1 enum34==1.1.6 flake8==3.5.0 funcsigs==1.0.2 +future==0.16.0 idna==2.6 isodate==0.6.0 +kafka==1.3.5 lockfile==0.12.2 mccabe==0.6.1 -mercurial==3.1.2 mistune==0.7.4 mock==2.0.0 more-itertools==4.1.0 -pbr==4.0.1 +pbr==4.0.2 pluggy==0.6.0 py==1.5.3 pycodestyle==2.3.1 pyflakes==1.6.0 pyparsing==2.2.0 -pytest==3.5.0 +pytest==3.5.1 pytest-runner==4.2 rdflib==4.2.2 rdflib-jsonld==0.4.0 requests==2.18.4 -ruamel.ordereddict==0.4.13 ruamel.yaml==0.14.12 schema-salad==2.7.20180322174040 six==1.11.0 +spavro==1.1.10 typing==3.6.4 urllib3==1.22 diff --git a/test-aether-integration-module/entrypoint.sh b/test-aether-integration-module/entrypoint.sh index 54fa4c90f..08fb35cd0 100755 --- a/test-aether-integration-module/entrypoint.sh +++ b/test-aether-integration-module/entrypoint.sh @@ -18,8 +18,7 @@ # specific language governing permissions and limitations # under the License. # -set -e - +set -Eeuo pipefail # Define help message show_help() { @@ -28,19 +27,11 @@ show_help() { ---------------------------------------------------------------------------- bash : run bash eval : eval shell command - manage : invoke django manage.py commands pip_freeze : freeze pip dependencies and write to requirements.txt - setupproddb : create/migrate database for production - setuplocaldb : create/migrate database for development (creates superuser and token) - test : run tests test_lint : run flake8 tests - test_coverage : run tests with coverage output - - start : start webserver behind nginx - start_dev : start webserver for development """ } @@ -49,11 +40,6 @@ test_flake8() { } test_coverage() { - # Python2 Tests !!Must run first as we rely on the schema generation - python2 setup.py -q test "${@:1}" - - cat /code/conf/extras/good_job.txt - # Python3 Tests python3 setup.py -q test "${@:1}" @@ -72,22 +58,13 @@ case "$1" in eval "${@:2}" ;; - manage ) - ./manage.py "${@:2}" - ;; - pip_freeze ) - rm -rf /tmp/env - pip2 install -f ./conf/pip/dependencies -r ./conf/pip/primary-requirements.py2.txt --upgrade - - cat /code/conf/pip/requirements_header.txt | tee conf/pip/requirements.py2.txt - pip2 freeze --local | grep -v appdir | tee -a conf/pip/requirements.py2.txt rm -rf /tmp/env - pip3 install -f ./conf/pip/dependencies -r ./conf/pip/primary-requirements.py3.txt --upgrade + pip3 install -f ./conf/pip/dependencies -r ./conf/pip/primary-requirements.txt --upgrade - cat /code/conf/pip/requirements_header.txt | tee conf/pip/requirements.py3.txt - pip3 freeze --local | grep -v appdir | tee -a conf/pip/requirements.py3.txt + cat /code/conf/pip/requirements_header.txt | tee conf/pip/requirements.txt + pip3 freeze --local | grep -v appdir | tee -a conf/pip/requirements.txt ;; setuplocaldb ) @@ -108,10 +85,6 @@ case "$1" in test_flake8 ;; - test_coverage) - test_coverage "${@:2}" - ;; - help) show_help ;; diff --git a/test-aether-integration-module/test/__init__.py b/test-aether-integration-module/test/__init__.py index a6434661c..6d9b467b4 100644 --- a/test-aether-integration-module/test/__init__.py +++ b/test-aether-integration-module/test/__init__.py @@ -16,11 +16,14 @@ # specific language governing permissions and limitations # under the License. -from aether.client import KernelClient import pytest -import sys from time import sleep +from aether.client import KernelClient +import aether.saladbar.wizard as wizard + +from .consumer import get_consumer, read + KERNEL_URL = "http://kernel-test:9000/v1" kernel_credentials = { @@ -28,15 +31,12 @@ "password": "adminadmin", } -kernel_retry = 3 -kernel_retry_time = 5 +kernel_retry = 15 +kernel_retry_time = 1 SEED_ENTITIES = 10 SEED_TYPE = "Person" -py2 = pytest.mark.skipif(sys.version_info >= (3, 0), reason="Test only required for python2") -py3 = pytest.mark.skipif(sys.version_info <= (3, 0), reason="Test only required for python3") - @pytest.fixture(scope="session") def aether_client(): @@ -54,11 +54,11 @@ def aether_client(): @pytest.fixture(scope="session") def schema_registration(): - from saladbar import wizard # we have to import this locally so it only gets into @py2 scopes try: wizard.test_setup() return True except Exception as err: + raise(err) print("Schema registration failed with: %s" % err) return False @@ -83,7 +83,7 @@ def existing_entities(aether_client, existing_projectschemas): entities = {} for ps in existing_projectschemas: name = ps.get("name") - endpoint = aether_client.Entity.get(name) + endpoint = aether_client.Entity.get(name, strict=False) entities[name] = [i for i in endpoint] return entities @@ -93,8 +93,8 @@ def generate_entities(aether_client, existing_schemas, existing_projectschemas): entities = [] manager = None from aether.mocker import MockingManager, MockFn, Generic - person = "http://demo.eha.org/Person" - location = "http://demo.eha.org/GeoLocation" + person = "org.eha.demo.Person" + location = "org.eha.demo.GeoLocation" try: manager = MockingManager(kernel_url=KERNEL_URL) manager.types[location].override_property( @@ -117,7 +117,6 @@ def generate_entities(aether_client, existing_schemas, existing_projectschemas): @pytest.fixture(scope="function") def read_people(): - from .consumer import get_consumer, read consumer = get_consumer(SEED_TYPE) messages = read(consumer, start="FIRST", verbose=True, timeout_ms=500) consumer.close() # leaving consumers open can slow down zookeeper, try to stay tidy diff --git a/test-aether-integration-module/test/test-py2_schema_generation.py b/test-aether-integration-module/test/test-py2_schema_generation.py deleted file mode 100644 index bab0e0c65..000000000 --- a/test-aether-integration-module/test/test-py2_schema_generation.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) 2018 by eHealth Africa : http://www.eHealthAfrica.org -# -# See the NOTICE file distributed with this work for additional information -# regarding copyright ownership. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on anx -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from . import * - - -@py2 -def test_1_register_schemas(schema_registration): - assert(schema_registration) # Try to use the test mode of the wizard - - -@py2 -def test_2_check_schemas(existing_schemas): - assert(len(existing_schemas) > 0) diff --git a/test-aether-integration-module/test/test-py3_entity_generation.py b/test-aether-integration-module/test/test-py3_entity_generation.py deleted file mode 100644 index 2ad1ac56e..000000000 --- a/test-aether-integration-module/test/test-py3_entity_generation.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) 2018 by eHealth Africa : http://www.eHealthAfrica.org -# -# See the NOTICE file distributed with this work for additional information -# regarding copyright ownership. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on anx -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from . import * - - -@py3 -def test_1_generate_entities(generate_entities): - assert(len(generate_entities) == SEED_ENTITIES) - - -@py3 -def test_2_check_updated_count(existing_entities, generate_entities): - assert(len(existing_entities.get(SEED_TYPE)) >= SEED_ENTITIES) diff --git a/test-aether-integration-module/test/test-py3_kafka_consistency.py b/test-aether-integration-module/test/test-py3_kafka_consistency.py deleted file mode 100644 index b66f9c085..000000000 --- a/test-aether-integration-module/test/test-py3_kafka_consistency.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (C) 2018 by eHealth Africa : http://www.eHealthAfrica.org -# -# See the NOTICE file distributed with this work for additional information -# regarding copyright ownership. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on anx -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from . import * - - -@py3 -def test_1_check_stream_entities(read_people, existing_entities): - kernel_messages = [msg.get("payload").get("id") for msg in existing_entities.get(SEED_TYPE)] - kafka_messages = [msg.get("id") for msg in read_people] - assert(len(kernel_messages) == len(kafka_messages)) - for _id in kernel_messages: - assert(_id in kafka_messages) diff --git a/test-aether-integration-module/test/test_integration.py b/test-aether-integration-module/test/test_integration.py new file mode 100644 index 000000000..95b919871 --- /dev/null +++ b/test-aether-integration-module/test/test_integration.py @@ -0,0 +1,30 @@ +from . import * + + +#################################################################################### +# Create Schemas -> Entities -> Produce -> Check in Kafka for result +#################################################################################### + + +def test_1_register_schemas(schema_registration): + assert(schema_registration) # Try to use the test mode of the wizard + + +def test_2_check_schemas(existing_schemas): + assert(len(existing_schemas) > 0) + + +def test_3_generate_entities(generate_entities): + assert(len(generate_entities) == SEED_ENTITIES) + + +def test_4_check_updated_count(existing_entities, generate_entities): + assert(len(existing_entities.get(SEED_TYPE)) >= SEED_ENTITIES) + + +def test_5_check_stream_entities(read_people, existing_entities): + kernel_messages = [msg.get("payload").get("id") for msg in existing_entities.get(SEED_TYPE)] + kafka_messages = [msg.get("id") for msg in read_people] + assert(len(kernel_messages) == len(kafka_messages)) + for _id in kernel_messages: + assert(_id in kafka_messages)