Merge branch 'release/1.29.0'

Vagrantfile

@@ -68,6 +68,9 @@ Vagrant.configure("2") do |config|
     worker.vm.network "private_network", ip: ENV.fetch("MMW_WORKER_IP", "33.33.34.20")
 
     worker.vm.synced_folder "src/mmw", "/opt/app"
+    # Facilitates the sharing of Django media root directories across virtual machines.
+    worker.vm.synced_folder ".vagrant/machines/app/virtualbox/media", "/tmp/media",
+      create: true
 
     # Path to RWD data (ex. /media/passport/rwd-nhd)
     worker.vm.synced_folder ENV.fetch("RWD_DATA", "/tmp"), "/opt/rwd-data"
@@ -105,6 +108,9 @@ Vagrant.configure("2") do |config|
     app.vm.network "private_network", ip: ENV.fetch("MMW_APP_IP", "33.33.34.10")
 
     app.vm.synced_folder "src/mmw", "/opt/app"
+    # Facilitates the sharing of Django media root directories across virtual machines.
+    app.vm.synced_folder ".vagrant/machines/app/virtualbox/media", "/var/www/mmw/media",
+      create: true, mount_options: ["dmode=777"]
 
     # Django via Nginx/Gunicorn
     app.vm.network "forwarded_port", {

deployment/ansible/roles/model-my-watershed.celery-worker/defaults/main.yml

@@ -5,8 +5,8 @@ app_deploy_branch: "master"
 app_config:
   DJANGO_TEST_DB_NAME: "{{ django_test_database }}"
   DJANGO_SETTINGS_MODULE: "{{ django_settings_module }}"
-  DJANGO_STATIC_ROOT: "/tmp"
-  DJANGO_MEDIA_ROOT: "/tmp"
+  DJANGO_STATIC_ROOT: "/tmp/static"
+  DJANGO_MEDIA_ROOT: "/tmp/media"
   DJANGO_POSTGIS_VERSION: "{{ app_postgis_version }}"
   DJANGO_SECRET_KEY: "{{ app_secret_key }}"
 

src/mmw/apps/export/tasks.py

@@ -14,8 +14,7 @@
 from django.utils.timezone import now
 from django.contrib.gis.geos import GEOSGeometry
 
-from apps.core.models import Job
-from apps.modeling.models import Project
+from apps.modeling.models import Project, Scenario
 from apps.modeling.tasks import to_gms_file
 
 from hydroshare import HydroShareService
@@ -40,20 +39,7 @@ def update_resource(user_id, project_id, params):
         raise RuntimeError('HydroShare could not find requested resource')
 
     # Update files
-    files = params.get('files', [])
-    for md in params.get('mapshed_data', []):
-        muuid = md.get('uuid')
-        if muuid:
-            try:
-                job = Job.objects.get(uuid=muuid)
-                mdata = json.loads(job.result)
-                files.append({
-                    'name': md.get('name'),
-                    'contents': to_gms_file(mdata).read(),
-                })
-            except (Job.DoesNotExist, ValueError):
-                # Either the job wasn't found, or its content isn't JSON
-                pass
+    files = _hs_gather_client_files(params)
 
     hs.add_files(hsresource.resource, files)
 
@@ -83,7 +69,7 @@ def create_resource(user_id, project_id, params):
     )
 
     # Files sent from the client
-    files = params.get('files', [])
+    files = _hs_gather_client_files(params)
 
     # AoI GeoJSON
     aoi_geojson = GEOSGeometry(project.area_of_interest).geojson
@@ -92,21 +78,6 @@ def create_resource(user_id, project_id, params):
         'contents': aoi_geojson,
     })
 
-    # MapShed Data
-    for md in params.get('mapshed_data', []):
-        muuid = md.get('uuid')
-        if muuid:
-            try:
-                job = Job.objects.get(uuid=muuid)
-                mdata = json.loads(job.result)
-                files.append({
-                    'name': md.get('name'),
-                    'contents': to_gms_file(mdata).read(),
-                })
-            except (Job.DoesNotExist, ValueError):
-                # Either the job wasn't found, or its content isn't JSON
-                pass
-
     # Add all files
     hs.add_files(resource, files)
 
@@ -145,7 +116,7 @@ def create_resource(user_id, project_id, params):
         'hs_file_type': 'GeoFeature',
     })
 
-    # Link HydroShareResrouce to Project and save
+    # Link HydroShareResource to Project and save
     hsresource = HydroShareResource.objects.create(
         project=project,
         resource=resource,
@@ -164,6 +135,29 @@ def create_resource(user_id, project_id, params):
     return serializer.data
 
 
+def _hs_gather_client_files(params):
+    # Files sent from the client
+    files = params.get('files', [])
+
+    # MapShed Data
+    for md in params.get('mapshed_data', []):
+        mdata = md.get('data')
+        files.append({
+            'name': md.get('name'),
+            'contents': to_gms_file(mdata).read() if mdata else None,
+        })
+
+    # Weather Data
+    for wd in params.get('weather_data', []):
+        s = Scenario.objects.get(id=wd.get('id'))
+        files.append({
+            'name': wd.get('name'),
+            'contents': s.weather_custom.read(),
+        })
+
+    return files
+
+
 @shared_task
 def padep_worksheet(results):
     """

src/mmw/apps/modeling/calcs.py

@@ -3,9 +3,11 @@
 from __future__ import unicode_literals
 from __future__ import absolute_import
 
+import csv
 import json
 
 from copy import deepcopy
+from datetime import datetime, timedelta
 
 from django.conf import settings
 from django.db import connection
@@ -15,7 +17,129 @@
 from apps.modeling.mapshed.calcs import area_calculations
 
 
+NODATA = -999.0
 HECTARES_PER_SQM = 0.0001
+DATE_FORMAT = '%m/%d/%Y'
+MAX_ERRORS = 10  # Number of maximum errors reported while parsing weather data
+
+
+def get_weather_modifications(csv_file):
+    """
+    Given a CSV file like:
+
+    DATE,PRCP,TAVG
+    01/01/2001,0,-6
+    01/02/2001,0,-7
+    01/03/2001,0,-8
+
+    where PRCP is in CM and TAVG is in C, converts it into a JSON
+    modifications object that can override gis_data. Also parses
+    for errors.
+
+    Returns a tuple where the first item is the output and the second is
+    a list of errors.
+    """
+    rows = list(csv.reader(csv_file))
+    errs = []
+
+    def err(msg, line=None):
+        text = 'Line {}: {}'.format(line, msg) if line else msg
+        errs.append(text)
+
+    if rows[0] != ['DATE', 'PRCP', 'TAVG']:
+        err('Missing or incorrect header. Expected "DATE,PRCP,TAVG", got {}'
+            .format(','.join(rows[0]), 1))
+
+    if len(rows) < 1097:
+        err('Need at least 3 years of contiguous data.')
+
+    if len(rows) > 10958:
+        err('Need at most 30 years of contiguous data.')
+
+    if errs:
+        return None, errs
+
+    try:
+        begyear = datetime.strptime(rows[1][0], DATE_FORMAT).year
+    except ValueError as ve:
+        err(ve.message, 2)
+        return None, errs
+
+    try:
+        endyear = datetime.strptime(rows[-1][0], DATE_FORMAT).year
+    except ValueError as ve:
+        err(ve.message, len(rows))
+        return None, errs
+
+    year_range = endyear - begyear + 1
+
+    if year_range < 3 or year_range > 30:
+        err('Invalid year range {} between beginning year {}'
+            ' and end year {}. Year range must be between 3 and 30.'
+            .format(year_range, begyear, endyear))
+
+    if errs:
+        return None, errs
+
+    # Initialize precipitation and temperature dicts. Same shape as in
+    # apps.modeling.mapshed.calcs.weather_data. The extra days for months
+    # with fewer than 31 will remain NODATA.
+    prcps = [[[NODATA] * 31 for m in range(12)] for y in range(year_range)]
+    tavgs = [[[NODATA] * 31 for m in range(12)] for y in range(year_range)]
+
+    previous_d = None
+
+    for row in enumerate(rows[1:]):
+        # Only report so many errors before abandoning parsing
+        if len(errs) >= MAX_ERRORS:
+            err('Maximum error limit reached.')
+            return None, errs
+
+        idx, (date, prcp, tavg) = row
+
+        try:
+            d = datetime.strptime(date, DATE_FORMAT)
+
+            # For every row after the first, check to see that the date
+            # is the next one in the sequence.
+            if idx > 0:
+                if d == previous_d:
+                    raise ValueError('Duplicate date: {}'.format(date))
+
+                expected_d = previous_d + timedelta(days=1)
+                if d != expected_d:
+                    raise ValueError('Incorrect date sequence:'
+                                     ' Expected {}, got {}.'.format(
+                                         datetime.strftime(expected_d,
+                                                           DATE_FORMAT),
+                                         datetime.strftime(d, DATE_FORMAT)))
+
+            yidx = d.year - begyear
+            midx = d.month - 1
+            didx = d.day - 1
+
+            prcps[yidx][midx][didx] = float(prcp)
+            tavgs[yidx][midx][didx] = float(tavg)
+
+            if (prcps[yidx][midx][didx]) < 0:
+                raise ValueError('Precipitation cannot be less than 0.')
+
+        except Exception as e:
+            # Record error with line. idx + 2 because idx starts at 0 while
+            # line numbers start at 1, and we need to account for the header.
+            err(e.message, idx + 2)
+
+        previous_d = d
+
+    mods = {
+        'WxYrBeg': begyear,
+        'WxYrEnd': endyear,
+        'WxYrs': year_range,
+        'Prec': prcps,
+        'Temp': tavgs,
+    }
+
+    return mods, errs
 
 
 def split_into_huc12s(code, id):

src/mmw/apps/modeling/migrations/0030_custom_weather_dataset.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-24 19:52
+from __future__ import unicode_literals
+
+import apps.modeling.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('modeling', '0029_project_on_delete'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='custom_weather_dataset',
+            field=models.FileField(help_text='Reference path of the uploaded file.', null=True, upload_to=apps.modeling.models.project_filename),
+        ),
+        migrations.AddField(
+            model_name='project',
+            name='uses_custom_weather',
+            field=models.BooleanField(default=False, help_text='Whether or not this project currently uses a custom weather dataset. If true, requires a non-NULL value for custom_weather_dataset.'),
+        ),
+    ]

src/mmw/apps/modeling/migrations/0031_scenario_add_weather_fields.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-04-21 02:19
+from __future__ import unicode_literals
+
+import apps.modeling.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('modeling', '0030_custom_weather_dataset'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scenario',
+            name='weather_type',
+            field=models.CharField(choices=[('DEFAULT', 'DEFAULT'), ('SIMULATION', 'SIMULATION'), ('CUSTOM', 'CUSTOM')], default='DEFAULT', help_text='The source of weather data for this scenario. Only applies to GWLF-E scenarios.', max_length=255),
+        ),
+        migrations.AddField(
+            model_name='scenario',
+            name='weather_simulation',
+            field=models.CharField(help_text='Identifier of the weather simulation to use.', max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='scenario',
+            name='weather_custom',
+            field=models.FileField(help_text='Reference path of the custom weather file.', null=True, upload_to=apps.modeling.models.scenario_filename),
+        ),
+    ]

src/mmw/apps/modeling/migrations/0032_project_remove_weather_fields.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-04-21 04:42
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('modeling', '0031_scenario_add_weather_fields'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='project',
+            name='custom_weather_dataset',
+        ),
+        migrations.RemoveField(
+            model_name='project',
+            name='uses_custom_weather',
+        ),
+    ]