Fix/schema issues (#146)

* fix: made wizard compatible with python3 * fix: salad-bar avro naming convention change working for schema gen and for data mocking * fixed python3 build processes. Integrations tests ok up to publisher/consumer * fix: ordering matters in unioned avro schemas (main type last). Producer working * fix: updated kernel to use spavro for validation * fix: aether producer properly handles new schema format after update to kernel * chore: remove meaningless error handling * fix: got rid of sneaky fastavro dependency
eHealthAfrica · May 4, 2018 · 31bbe1a · 31bbe1a
1 parent 58f1ea8
commit 31bbe1a
Show file tree

Hide file tree

Showing 68 changed files with 319 additions and 386 deletions.
diff --git a/aether-kernel/aether/kernel/api/utils.py b/aether-kernel/aether/kernel/api/utils.py
@@ -21,11 +21,11 @@
 import re
 import string
 import uuid
-from io import BytesIO
 
-import fastavro as avro
 import jsonpath_ng
 from jsonpath_ng import parse
+from spavro.schema import parse as parse_schema
+from spavro.io import validate
 
 from django.utils.safestring import mark_safe
 from pygments import highlight
@@ -533,13 +533,12 @@ def merge_objects(source, target, direction):
 
 
 def validate_entity_payload(project_Schema, payload):
-    # Use fastavro to validate payload against the linked schema
+    # Use spavro to validate payload against the linked schema
     try:
-        # fastavro is primarily for (de)serialization. To test the schema compliance,
-        # we can just try to serialze the data using the schema.
-        with BytesIO() as file_obj:
-            avro.writer(file_obj, project_Schema.schema.definition, [payload])
-            # if we didn't get an exception, we're ok!
-        return
+        avro_schema = parse_schema(json.dumps(project_Schema.schema.definition, indent=2))
+        valid = validate(avro_schema, payload)
+        if not valid:
+            raise TypeError('Record did not conform to registered schema.')
+        return True
     except Exception as err:
         raise err
diff --git a/aether-kernel/conf/pip/primary-requirements.txt b/aether-kernel/conf/pip/primary-requirements.txt
@@ -8,7 +8,6 @@
 #                                                                              #
 ################################################################################
 
-avro-python3
 django<2
 django-cas-ng
 django-cors-headers
@@ -21,14 +20,14 @@ djangorestframework
 djangorestframework-filters
 drf-dynamic-fields
 drf-openapi
-fastavro
 jsonpath_ng
 jsonschema
 psycopg2-binary
 pygments
 python-dateutil
 raven
 requests
+spavro
 uwsgi
 
 # Aether common module

diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt
@@ -10,15 +10,13 @@
 ################################################################################
 
 aether.common==0.0.0
-avro-python3==1.8.2
-certifi==2018.1.18
+certifi==2018.4.16
 chardet==3.0.4
 click==6.7
 coreapi==2.3.3
 coreschema==0.0.4
 coverage==4.5.1
-Cython==0.28.1
-decorator==4.2.1
+decorator==4.3.0
 Django==1.11.12
 django-cas-ng==3.5.9
 django-cors-headers==2.2.0
@@ -34,7 +32,6 @@ djangorestframework-csv==2.1.0
 djangorestframework-filters==0.10.2
 drf-dynamic-fields==0.3.0
 drf-openapi==1.3.0
-fastavro==0.17.10
 flake8==3.5.0
 flake8-quotes==1.0.0
 idna==2.6
@@ -46,19 +43,20 @@ MarkupSafe==1.0
 mccabe==0.6.1
 mock==2.0.0
 openapi-codec==1.3.2
-pbr==4.0.1
+pbr==4.0.2
 ply==3.11
 psycopg2-binary==2.7.4
 pycodestyle==2.3.1
 pyflakes==1.6.0
 Pygments==2.2.0
 python-cas==1.2.0
 python-dateutil==2.7.2
-pytz==2018.3
-raven==6.6.0
+pytz==2018.4
+raven==6.7.0
 requests==2.18.4
-simplejson==3.13.2
+simplejson==3.14.0
 six==1.11.0
+spavro==1.1.10
 sqlparse==0.2.4
 unicodecsv==0.14.1
 uritemplate==3.0.0

diff --git a/aether-mock-data/test.py b/aether-mock-data/test.py
@@ -43,8 +43,8 @@ def show_mockfn():
 
 def main():
 
-    person = "http://demo.eha.org/Person"
-    location = "http://demo.eha.org/GeoLocation"
+    person = "org.eha.demo.Person"
+    location = "org.eha.demo.GeoLocation"
 
     manager = MockingManager()
     manager.types[location].override_property(

diff --git a/aether-producer/Dockerfile b/aether-producer/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:2.7
+FROM python:3.6
 
 ################################################################################
 ## setup container

diff --git a/aether-producer/conf/docker/setup.sh b/aether-producer/conf/docker/setup.sh
@@ -28,19 +28,6 @@ set -Eeuox pipefail
 # Do not buffer stdout so we see log output immediatly
 export PYTHONUNBUFFERED=true
 
-
-################################################################################
-# install packages
-################################################################################
-
-# Add postgres apt repo to get more recent postgres versions
-echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' > /etc/apt/sources.list.d/pgdg.list
-wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-
-apt-get update -qq
-cat /tmp/apt-packages.txt | xargs apt-get -qq --yes --force-yes install
-
-
 ################################################################################
 # last steps and cleaning
 ################################################################################

diff --git a/aether-producer/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl b/aether-producer/conf/pip/dependencies/aether.client-0.0.0-py2.py3-none-any.whl
diff --git a/aether-producer/conf/pip/primary-requirements.txt b/aether-producer/conf/pip/primary-requirements.txt
@@ -9,7 +9,6 @@
 ################################################################################
 
 aether.client
-avro
 kafka
 psycopg2-binary
 requests

diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt
@@ -8,31 +8,23 @@
 # then run 'docker-compose run producer pip_freeze'                            #
 #                                                                              #
 ################################################################################
-
 aether.client==0.0.0
-avro==1.8.2
-kafka==1.3.5
-psycopg2-binary==2.7.4
-requests==2.18.4
-
-
-# test libraries
+avro-python3==1.8.2
+certifi==2018.4.16
+chardet==3.0.4
 coverage==4.5.1
 flake8==3.5.0
-flake8-quotes==0.14.0
-mock==2.0.0
-
-
-## The following requirements were added by pip freeze:
-certifi==2018.1.18
-chardet==3.0.4
-configparser==3.5.0
-enum34==1.1.6
-funcsigs==1.0.2
+flake8-quotes==1.0.0
 idna==2.6
+kafka==1.3.5
 mccabe==0.6.1
-pbr==3.1.1
+mock==2.0.0
+pbr==4.0.2
+psycopg2==2.7.4
+psycopg2-binary==2.7.4
 pycodestyle==2.3.1
 pyflakes==1.6.0
+requests==2.18.4
 six==1.11.0
+spavro==1.1.15
 urllib3==1.22
diff --git a/aether-producer/producer/Pipfile b/aether-producer/producer/Pipfile
@@ -4,9 +4,9 @@ url = "https://pypi.python.org/simple"
 
 [packages]
 kafka = "*"
-py3-psycopg2 = "*"
+psycopg2-binary = "*"
 requests = "*"
-avro-python3 = "*"
+spavro = "*"
 
 [dev-packages]
 "-e /aether-utils/aether-client" = "*"