Add git conflict handling to migration diff output

.docker-compose.vital-strategies-theme.yaml

@@ -23,9 +23,9 @@ services:
         EXTRA_PACKAGES: cron
         EXTRA_FILESYSTEM: "./overrides/vital-strategies/filesystem/"
         POST_INSTALL: |
-          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b dev/python3-upgrade &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b cc6c8e6f19f59e6842d370bf7ac87d94e37a2831 &&\
           install_standard_ckan_extension_github -r ckan/ckanext-geoview && \
-          #install_standard_ckan_extension_github -r datopian/ckanext-sentry -b 2.10 && \
+          install_standard_ckan_extension_github -r datopian/ckanext-sentry -b 2.10 && \
           install_standard_ckan_extension_github -r datopian/ckanext-gtm && \
           install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l en -f && \
@@ -47,7 +47,7 @@ services:
         CKAN_BRANCH: ckan-2.10.4
         POST_INSTALL: |
           install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 &&\
-          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b dev/python3-upgrade &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b cc6c8e6f19f59e6842d370bf7ac87d94e37a2831 &&\
           install_standard_ckan_extension_github -r ckan/ckanext-geoview
     environment:
       - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme-

ckan/Dockerfile

@@ -101,7 +101,7 @@ RUN ckan-pip install --upgrade pip &&\
 
 RUN ckan-pip install supervisor && \
     ckan-pip install uwsgi==2.0.19.1 && \
-    ckan-pip install gevent==21.12.0
+    ckan-pip install gevent==24.2.1
 
 RUN apt-get update && apt-get install -y \
     uwsgi-plugins-all \

ckan/post_install_functions.sh

@@ -30,11 +30,15 @@ install_standard_ckan_extension_github() {
     # Check if the branch exists by examining the output directly
     BRANCH_EXISTS=$(git ls-remote --heads ${GITHUB_URL}/${REPO_NAME}.git ${BRANCH})
 
+    if [ ! -n "$BRANCH_EXISTS" ]; then
+      # It might be a commit instead of a branch
+      BRANCH_EXISTS=$(git ls-remote --heads ${GITHUB_URL}/${REPO_NAME}.git | grep -o $BRANCH)
+    fi
+
     if [ -z "$BRANCH_EXISTS" ]; then
       echo "#### BRANCH EXISTS: $BRANCH_EXISTS ####"
 
       if [ "$BRANCH" = "master" ]; then
-        # Attempt to switch to 'main' if 'master' does not exist
         BRANCH_EXISTS=$(git ls-remote --heads ${GITHUB_URL}/${REPO_NAME}.git main)
         if [ -n "$BRANCH_EXISTS" ]; then
           echo "Branch 'master' not found, switching to 'main'."
@@ -44,7 +48,6 @@ install_standard_ckan_extension_github() {
           exit 1
         fi
       else
-        # If a specific branch was provided and does not exist, print an error and exit
         echo "Branch '$BRANCH' not found. Please check the branch name."
         exit 1
       fi
@@ -62,14 +65,40 @@ install_standard_ckan_extension_github() {
       done &&\
       ckan-pip install --no-use-pep517 --index-url ${PIP_INDEX_URL} -e ${TMPDIR}
     else
-      TEMPFILE=`mktemp`
-      for REQUIREMENTS_FILE_NAME in requirements pip-requirements
-      do
-        if wget -O $TEMPFILE https://raw.githubusercontent.com/${REPO_NAME}/$BRANCH/$REQUIREMENTS_FILE_NAME.txt
-          then ckan-pip install --index-url ${PIP_INDEX_URL} -r $TEMPFILE && break;
+      # Remove poetry files: ckan-cloud-docker currently has issues with poetry dependencies
+      if [ "${REPO_NAME}" = "datopian/ckanext-sentry" ]; then
+        TMPDIR=${CKAN_VENV}/src/${EGG}
+        git clone -b $BRANCH ${GITHUB_URL}/${REPO_NAME}.git ${TMPDIR}
+
+        CURRENT_DIR=$(pwd)
+
+        cd ${TMPDIR}
+
+        if [ -f "poetry.lock" ] && [ -f "pyproject.toml" ]; then
+          rm -f "poetry.lock" "pyproject.toml"
         fi
-      done &&\
-      ckan-pip install --no-use-pep517 --index-url ${PIP_INDEX_URL} -e git+${GITHUB_URL}/${REPO_NAME}.git@$BRANCH#egg=${EGG}
+
+        for REQUIREMENTS_FILE_NAME in requirements pip-requirements
+        do
+          if [ -f ${TMPDIR}/$REQUIREMENTS_FILE_NAME.txt ]; then
+            ckan-pip install --index-url ${PIP_INDEX_URL} -r ${TMPDIR}/$REQUIREMENTS_FILE_NAME.txt && break;
+          fi
+        done
+
+        ckan-pip install --no-use-pep517 --index-url ${PIP_INDEX_URL} -e ${TMPDIR}
+
+        cd ${CURRENT_DIR}
+
+      else
+        TEMPFILE=`mktemp`
+        for REQUIREMENTS_FILE_NAME in requirements pip-requirements
+        do
+          if wget -O $TEMPFILE https://raw.githubusercontent.com/${REPO_NAME}/$BRANCH/$REQUIREMENTS_FILE_NAME.txt
+            then ckan-pip install --index-url ${PIP_INDEX_URL} -r $TEMPFILE && break;
+          fi
+        done &&\
+        ckan-pip install --no-use-pep517 --index-url ${PIP_INDEX_URL} -e git+${GITHUB_URL}/${REPO_NAME}.git@$BRANCH#egg=${EGG}
+      fi
     fi
 }
 

ckan/templater.sh

@@ -79,5 +79,20 @@ for var in $vars; do
     value=$(echo "$value" | sed 's/\//\\\//g');
     replaces="-e 's/{{$var}}/${value}/g' $replaces"
 done
+
+if [[ ! -z "$(var_value GTM_ID)" ]]; then
+    replaces="-e '/^ckan.plugins =/ s/$/ gtm/' $replaces"
+fi
+
+if [[ ! -z "$(var_value SENTRY_DSN)" ]]; then
+    replaces="-e '/^ckan.plugins =/ s/$/ sentry/' $replaces"
+fi
+
+if [[ ! -z "$(var_value "AWS_ACCESS_KEY_ID")" ]] && [[ ! -z "$(var_value "AWS_SECRET_ACCESS_KEY")" ]] && [[ "$CURRENT_PLUGINS" != *"s3filestore"* ]]; then
+    replaces="-e '/^ckan.plugins =/ s/$/ s3filestore/' $replaces"
+fi
+
+replaces="$replaces -e '/^ckan.plugins =/ s/querytool//g' -e '/^ckan.plugins =/ s/$/ querytool/'"
+
 escaped_template_path=$(echo $template | sed 's/ /\\ /g')
 eval sed $replaces "$escaped_template_path"

docker-compose.yaml

@@ -3,11 +3,15 @@ services:
     image: traefik:1.7.2-alpine
     restart: always
     volumes:
-      - ./traefik/traefik.toml:/traefik.toml # Comment out for development
-      #- ./traefik/traefik.dev.toml:/traefik.toml # Uncomment for development
+      - ./traefik/traefik.toml.template:/traefik.toml.template
+      #- ./traefik/traefik.dev.toml:/traefik.dev.toml # Uncomment for development
       - ./traefik/acme.json:/acme.json
+      - ./cca-operator/templater.sh:/templater.sh
+      - ./docker-compose/traefik-secrets.sh:/traefik-secrets.sh
+      - ./traefik/entrypoint.sh:/entrypoint.sh
     networks:
       - ckan-multi
+    entrypoint: ["/bin/sh", "-c", "/entrypoint.sh"]
 
   varnish:
     image: million12/varnish

docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini.template

@@ -153,6 +153,10 @@ ckanext.querytool.allow_nav_bar = True
 
 ## Google Analytics
 
+ckanext.gtm.gtm_id = {{GTM_ID}}
+
+# OR
+
 googleanalytics.id = {{GA_ID}}
 googleanalytics.account = {{GA_ACCOUNT}}
 googleanalytics.username = {{GA_USERNAME}}
@@ -189,11 +193,11 @@ ckanext.xloader.jobs_db.uri = postgresql://postgres:123456@jobs-db/postgres
 
 #email_to = errors@example.com
 #error_email_from = ckan-errors@example.com
-# smtp.server = {{SMTP_SERVER}}
-# smtp.starttls = True
-# smtp.user = {{SMTP_USER}}
-# smtp.password = {{SMTP_PASSWORD}}
-# smtp.mail_from = vitalplatform@vitalstrategies.org
+smtp.server = {{SMTP_SERVER}}
+smtp.starttls = True
+smtp.user = {{SMTP_USER}}
+smtp.password = {{SMTP_PASSWORD}}
+smtp.mail_from = {{SMTP_MAIL_FROM}}
 
 ###
 
@@ -207,9 +211,9 @@ ckan.harvest.mq.hostname = redis
 ckan.harvest.mq.redis_db = 9
 
 ## Sentry settings
-# sentry_dsn = {{SENTRY_DSN}}
-# ckan.sentry.configure_logging = True
-# ckan.sentry.log_level = ERROR
+ckan.sentry.dsn = {{SENTRY_DSN}}
+ckan.sentry.configure_logging = True
+ckan.sentry.log_level = ERROR
 
 
 ## Logging configuration

docker-compose/ckan-secrets.dat

@@ -16,11 +16,14 @@ ckan required SMTP_USER info Enter SMTP server username
 ckan optional SMTP_PASSWORD empty Enter SMTP server password
 ckan optional SMTP_MAIL_FROM empty Enter SMTP mail from
 ckan required CKAN_SITE_URL http://ckan:5000 Enter Website URL (including https:// or http://)
-ckan optional AWS_ACCESS_KEY_ID empty Enter AWS secret key (if any)
-ckan optional AWS_SECRET_ACCESS_KEY empty AWS secret access key (if any)
-ckan optional SENTRY_DSN https://<token>@sentry.io/<id> Enter Sentry DSN URL with token and ID
-ckan optional GA_ID empty Enter Google Analytics ID
-ckan optional GA_ACCOUNT empty Enter Google Analytics account name
-ckan optional GA_PASSWORD empty Enter Google Analytics password
-ckan optional GA_USERNAME empty Enter Google Analytics username
-datapusher optional TIMEZONE UTC Enter Datapusher timezone 
+ckan optional AWS_ACCESS_KEY_ID empty Enter AWS secret key [Skip if not using AWS]
+ckan optional AWS_SECRET_ACCESS_KEY empty AWS secret access key [Skip if not using AWS]
+ckan optional SENTRY_DSN https://<token>@sentry.io/<id> Enter Sentry DSN URL with token and ID [Skip if not using Sentry]
+ckan optional GTM_ID empty Enter Google Tag Manager ID [Skip if not using the new Google Analytics (GTM) or not using Google Analytics at all]
+ckan optional GA_ID empty Enter Google Analytics ID [Skip if using GTM or not using Google Analytics at all]
+ckan optional GA_ACCOUNT empty Enter Google Analytics account name [Skip if using GTM or not using Google Analytics at all]
+ckan optional GA_PASSWORD empty Enter Google Analytics password [Skip if using GTM or not using Google Analytics at all]
+ckan optional GA_USERNAME empty Enter Google Analytics username [Skip if using GTM or not using Google Analytics at all]
+datapusher optional TIMEZONE UTC Enter Datapusher timezone
+traefik optional CERTIFICATE_EMAIL Enter email address for Let's Encrypt certificate [Skip if using self-signed certificate]
+traefik optional CERTIFICATE_DOMAIN Enter domain for Let's Encrypt certificate [Skip if using self-signed certificate]

traefik/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+set -e
+
+# Update the Traefik configuration file with secrets if not development mode
+if [ -f /traefik.dev.toml ]; then
+    echo "Using development configuration"
+    cp /traefik.dev.toml /traefik.toml
+else
+    if [ ! -f /traefik.toml.template ]; then
+        echo "Traefik template file does not exist, exiting"
+        exit 1
+    fi
+    if [ ! -f /traefik-secrets.sh ]; then
+        echo "Traefik secrets file does not exist. Please run 'make secret' to generate it before starting the container"
+        exit 1
+    fi
+    if [ ! -f /templater.sh ]; then
+        echo "Templater script does not exist, exiting"
+        exit 1
+    fi
+
+    echo "Traefik configuration file does not exist, templating"
+
+    chmod +x /templater.sh
+    ./templater.sh /traefik.toml.template -f /traefik-secrets.sh > traefik.toml
+fi
+
+# Fix acme.json file permissions: set to 600
+chmod 600 /acme.json
+
+# first arg is `-f` or `--some-option`
+if [ "${1#-}" != "$1" ]; then
+    set -- traefik "$@"
+fi
+
+# if our command is a valid Traefik subcommand, let's invoke it through Traefik instead
+# (this allows for "docker run traefik version", etc)
+if traefik "$1" --help | grep -s -q "help"; then
+    set -- traefik "$@"
+fi
+
+exec "$@"

traefik/traefik.toml.template

@@ -0,0 +1,49 @@
+debug = false
+defaultEntryPoints = ["http", "https"]
+
+[entryPoints]
+    [entryPoints.http]
+      address = ":80"
+
+    [entryPoints.https]
+      address = ":443"
+        [entryPoints.https.tls]
+    [entryPoints.api]
+      address = ":8081"
+
+    [api]
+      entryPoint = "api"
+
+    [ping]
+      entryPoint = "http"
+
+    [acme]
+      email = "{{CERTIFICATE_EMAIL}}"
+      storage = "/acme.json"
+      entryPoint = "https"
+      onHostRule = true
+
+      [[acme.domains]]
+        main = "{{CERTIFICATE_DOMAIN}}"
+
+      [acme.httpChallenge]
+        entryPoint = "http"
+
+    [accessLog]
+
+    [file]
+      watch = true
+
+    [backends]
+      [backends.ckan]
+        [backends.ckan.servers.server1]
+          url = "http://nginx:8080"
+
+    [frontends]
+      [frontends.ckan]
+        backend="ckan"
+        passHostHeader = true
+        [frontends.ckan.headers]
+          SSLRedirect = true
+        [frontends.ckan.routes.route1]
+          rule = "Host:{{CERTIFICATE_DOMAIN}}"