Merge pull request apache#6 from riskive/ZFE-74262-nomad-release

Deploy superset to nomad from superset-external ZFE-74262

.drone.yml

@@ -19,5 +19,5 @@ steps:
       - apk add -U make openssh-client jq zip
       - mkdir ~/.ssh && echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519 && chmod 0600 ~/.ssh/id_ed25519
       - ssh-keyscan -H github.com >> ~/.ssh/known_hosts
-      - make -C .terra/ init
-      - make -C .terra/ validate
+      - make -C .terra/superset init
+      - make -C .terra/superset validate

.terra/superset.tf → .terra/superset/superset.tf

@@ -138,6 +138,8 @@ module "nomad-job" {
   app               = "${var.app}"
   ecr_url           = "${var.ecr_url}"
   git_sha           = "${var.git_sha}"
+  docker_file        = "../../Dockerfile"
+  docker_path       = "../.."
   rendered_template = "${data.template_file.nomad_job_spec.rendered}"
 }
 
@@ -146,19 +148,59 @@ resource "consul_keys" "superset-keys" {
   token      = "${var.consul_token}"
 
   key {
-    path  = "${var.app}/superset/env/celery_url"
+    path  = "${var.app}/superset/env/redis_host"
     value = "${aws_elasticache_cluster.cache.cache_nodes.0.address}"
   }
 
   key {
-    path  = "${var.app}/superset/env/celery_port"
+    path  = "${var.app}/superset/env/redis_port"
     value = "${aws_elasticache_cluster.cache.cache_nodes.0.port}"
   }
 
   key {
     path  = "${var.app}/superset/env/env"
     value = "${var.env}"
   }
+
+  key {
+    path  = "${var.app}/superset/env/server_limit_request_field_size"
+    value = "8190"
+  }
+
+  key {
+    path  = "${var.app}/superset/env/server_limit_request_line"
+    value = "4094"
+  }
+
+  key {
+    path  = "${var.app}/superset/env/server_limit_request_fields"
+    value = "20"
+  }
+
+  key {
+    path  = "${var.app}/superset/env/gunicorn_timeout"
+    value = "300"
+  }
+
+  key {
+    path  = "${var.app}/superset/env/server_worker_amount"
+    value = "6"
+  }
+
+  key {
+    path  = "${var.app}/superset/env/superset_certfile"
+    value = "/app/server.crt"
+  }
+
+  key {
+    path  = "${var.app}/superset/env/superset_keyfile"
+    value = "/app/server.key"
+  }
+
+  key {
+    path  = "${var.app}/superset/env/gunicorn_keepalive"
+    value = "65"
+  }
 }
 
 

Dockerfile

@@ -69,8 +69,17 @@ RUN mkdir -p ${PYTHONPATH} \
     libsasl2-modules-gssapi-mit \
     libpq-dev \
     libecpg-dev \
+    chromium \
+    wget \
+    unzip \
     && rm -rf /var/lib/apt/lists/*
 
+RUN export CHROMEDRIVER_VERSION=$(curl --silent https://chromedriver.storage.googleapis.com/LATEST_RELEASE_111) && \
+    wget -q https://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip && \
+    unzip chromedriver_linux64.zip -d /usr/bin && \
+    chmod 755 /usr/bin/chromedriver && \
+    rm -f chromedriver_linux64.zip
+
 COPY ./requirements/*.txt  /app/requirements/
 COPY setup.py MANIFEST.in README.md /app/
 
@@ -87,77 +96,23 @@ COPY --from=superset-node /app/superset/static/assets /app/superset/static/asset
 ## Lastly, let's install superset itself
 COPY superset /app/superset
 COPY setup.py MANIFEST.in README.md /app/
+COPY ./docker/run-server.sh /usr/bin/run-server.sh
+RUN chmod a+x /usr/bin/run-server.sh
+
 RUN cd /app \
-    && chown -R superset:superset * \
+    && chown -R superset:superset /app \
     && pip install -e . \
     && flask fab babel-compile --target superset/translations
 
-COPY ./docker/run-server.sh /usr/bin/
-
-RUN chmod a+x /usr/bin/run-server.sh
-
 WORKDIR /app
-
 USER superset
 
+COPY config/superset_config.py /app/superset_config.py
+COPY config/custom_sso_security_manager.py /app/custom_sso_security_manager.py
+COPY config/macros.py  /app/macros.py
+
 HEALTHCHECK CMD curl -f "http://localhost:$SUPERSET_PORT/health"
 
 EXPOSE ${SUPERSET_PORT}
 
 CMD /usr/bin/run-server.sh
-
-######################################################################
-# Dev image...
-######################################################################
-FROM lean AS dev
-ARG GECKODRIVER_VERSION=0.29.0
-ARG FIREFOX_VERSION=106.0.3
-
-COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/
-
-USER root
-
-RUN apt-get update -y \
-    && apt-get install -y --no-install-recommends \
-    libnss3 \
-    libdbus-glib-1-2 \
-    libgtk-3-0 \
-    libx11-xcb1 \
-    libasound2 \
-    libxtst6 \
-    wget
-
-ENV GECKODRIVER_VERSION=0.29.0
-
-# Install GeckoDriver WebDriver
-RUN apt-get update && \
-    wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
-    apt-get install -y --no-install-recommends ./google-chrome-stable_current_amd64.deb && \
-    rm -f google-chrome-stable_current_amd64.deb
-
-RUN export CHROMEDRIVER_VERSION=$(curl --silent https://chromedriver.storage.googleapis.com/LATEST_RELEASE_111) && \
-    wget -q https://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip && \
-    unzip chromedriver_linux64.zip -d /usr/bin && \
-    chmod 755 /usr/bin/chromedriver && \
-    rm -f chromedriver_linux64.zip
-# Cache everything for dev purposes...
-RUN cd /app \
-    && pip install --no-cache -r requirements/docker.txt \
-    && pip install --no-cache -r requirements/requirements-local.txt || true
-USER superset
-
-
-######################################################################
-# CI image...
-######################################################################
-FROM lean AS ci
-
-COPY --chown=superset ./docker/docker-bootstrap.sh /app/docker/
-COPY --chown=superset ./docker/docker-init.sh /app/docker/
-COPY --chown=superset ./docker/docker-ci.sh /app/docker/
-COPY config/superset_config.py /app/superset_config.py
-COPY config/custom_sso_security_manager.py /app/custom_sso_security_manager.py
-
-RUN chmod a+x /app/docker/*.sh
-
-CMD /app/docker/docker-ci.sh

Makefile

@@ -1,6 +1,6 @@
 run:
 	echo "Running superset locally with postgres and redis"
-	docker-compose up -d 
+	docker-compose up -d
 
 build:
 	echo "Building superset locally"

README.md

@@ -31,8 +31,9 @@ This repository  contains a copy of superset repo, to be compiled and used as a
 ## How to run
 to run it locally take in note that needs atleast 3gb on docker memory to run properly
 
-1. `make build`: Generates local superset image
-2. `make run`: Runs local superset image
+1. Duplicate /docker/.env.sample to /docker/.env
+2. `make build`: Generates local superset image
+3. `make run`: Runs local superset image
 
 Note: the local build will need you to compile the assets on the host machine: