From 3bca3ad6ccc2d5eb6b6c8eb7011073b44b8a425f Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Wed, 22 Mar 2023 05:04:56 -0400 Subject: [PATCH 01/10] Test commit --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9cd9f28..5da0f57 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,9 @@ It is ready when **Starting uWSGI** appears ### Manual Install +#### Ubuntu +**in-progress** + **Read the [Admin Guide](./docs/Decider_Admin_Guide_v1.0.0.pdf)** There are some issues in the instructions... ***Working on it, simplifying them*** From dd6a10e184e00d3921f1d2c6205fb8fe174f461c Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Wed, 22 Mar 2023 02:07:20 -0700 Subject: [PATCH 02/10] Test commit 2 - centos --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 5da0f57..60750d5 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,9 @@ It is ready when **Starting uWSGI** appears #### Ubuntu **in-progress** +#### CentOS +**in-progress** + **Read the [Admin Guide](./docs/Decider_Admin_Guide_v1.0.0.pdf)** There are some issues in the instructions... ***Working on it, simplifying them*** From 3de4786d7e638508e870533b9016697207ed9581 Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Thu, 23 Mar 2023 10:55:01 -0400 Subject: [PATCH 03/10] Ubuntu Manual Install Save Progress - Change DB username from 'user' (reserved word) to 'deciderdbuser'. - Split .env.example into .env.docker and .env.manual. - Add uwsgi.ini and decider.service files for manual install route. - Add current ubuntu install instructions md file. --- .env.example => .env.docker | 6 +- .env.manual | 26 +++++++ README.md | 2 +- app/env_vars.py | 2 +- decider.service | 7 ++ docs/install/Ubuntu_22.04.2.md | 132 +++++++++++++++++++++++++++++++++ uwsgi.ini | 11 +++ 7 files changed, 183 insertions(+), 3 deletions(-) rename .env.example => .env.docker (83%) create mode 100644 .env.manual create mode 100644 decider.service create mode 100644 docs/install/Ubuntu_22.04.2.md create mode 100644 uwsgi.ini diff --git a/.env.example b/.env.docker similarity index 83% rename from .env.example rename to .env.docker index 051f163..aa5ee56 100644 --- a/.env.example +++ b/.env.docker @@ -1,5 +1,9 @@ +# DOCKER DEPLOYMENT +# =-=-=-=-=-=-=-=-= +# Default / Example Environment File + # database & login credentials to access it -DB_USERNAME=user +DB_USERNAME=deciderdbuser DB_PASSWORD=password DB_DATABASE=decider diff --git a/.env.manual b/.env.manual new file mode 100644 index 0000000..cff5ecb --- /dev/null +++ b/.env.manual @@ -0,0 +1,26 @@ +# MANUAL INSTALL +# =-=-=-=-=-=-=- +# Default / Example Environment File + +# database & login credentials to access it +DB_USERNAME=deciderdbuser +DB_PASSWORD=password +DB_DATABASE=decider + +# key to encrypt cart content with +CART_ENC_KEY=12345678901234567890123456789012 + +# admin user account for the app +ADMIN_EMAIL=admin@admin.com +ADMIN_PASS=admin + +# only influences Docker, not uWSGI +# app is accessible at: http(s)://WEB_IP:WEB_PORT/ +# WEB_HTTPS_ON is off when empty, on when has any value +WEB_IP=127.0.0.1 +WEB_PORT=8001 +WEB_HTTPS_ON='' + +# defaults for postgres hosted on the same system +DB_HOSTNAME=localhost +DB_PORT=5432 diff --git a/README.md b/README.md index 5da0f57..507baa6 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ This project makes use of MITRE ATT&CK - [ATT&CK Terms of Use](https://attack.mi ```bash git clone https://github.com/cisagov/decider.git cd decider -cp .env.example .env +cp .env.docker .env # if you want HTTPS instead of HTTP # - edit .env diff --git a/app/env_vars.py b/app/env_vars.py index 806211a..70aeec0 100644 --- a/app/env_vars.py +++ b/app/env_vars.py @@ -19,6 +19,6 @@ except KeyError: print( "Failed to find all of the required environment variables mentioned in: app/env_vars.py.\n" - "Either modify .env using .env.example as a template, or edit the environment variables before launch." + "Either modify .env using .env.docker / .env.manual as a template, or edit the environment variables before launch." ) sys.exit(1) diff --git a/decider.service b/decider.service new file mode 100644 index 0000000..2eb12b3 --- /dev/null +++ b/decider.service @@ -0,0 +1,7 @@ +[Unit] +Description = Decider +[Service] +Type=simple +ExecStart=/opt/decider/1.0.0/venv/bin/uwsgi --ini /opt/decider/1.0.0/uwsgi.ini +[Install] +WantedBy=multi-user.target diff --git a/docs/install/Ubuntu_22.04.2.md b/docs/install/Ubuntu_22.04.2.md new file mode 100644 index 0000000..a86bfa1 --- /dev/null +++ b/docs/install/Ubuntu_22.04.2.md @@ -0,0 +1,132 @@ +# Ubuntu Desktop (Jammy) 22.04.2 LTS + + +### Update Package Sources +```bash +sudo apt update + +# (optional) +# sudo apt upgrade +``` + + +### Create Decider Service Account + Group +```bash +sudo adduser --no-create-home --system --shell /bin/false decider +sudo usermod -L decider +sudo groupadd decider +sudo usermod -aG decider decider +``` + + +### Install PostgreSQL +```bash +sudo apt install -y postgresql postgresql-contrib +sudo systemctl start postgresql +sudo systemctl enable postgresql +sudo systemctl status postgresql +``` + + +### Clone Repository +```bash +sudo apt install -y git +sudo mkdir /opt/decider +sudo chown decider:decider /opt/decider +sudo -u decider -g decider git clone https://github.com/cisagov/decider.git /opt/decider/1.0.0 +``` + + +### Install Python 3.8.10 (as Ubuntu 22.04 has Python 3.10.6) +- [Build Dependencies Reference](https://devguide.python.org/getting-started/setup-building/index.html#install-dependencies) +```bash +sudo apt install -y build-essential gdb lcov pkg-config \ + libbz2-dev libffi-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ + libncurses5-dev libreadline6-dev libsqlite3-dev libssl-dev \ + lzma lzma-dev tk-dev uuid-dev zlib1g-dev + +wget https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tar.xz +tar -xf Python-3.8.10.tar.xz +cd Python-3.8.10 + +./configure --prefix=/opt/decider/python3.8.10 --exec_prefix=/opt/decider/python3.8.10 --enable-optimizations +make -j $(( $(nproc) + 1 )) + +sudo mkdir /opt/decider/python3.8.10 +sudo make altinstall +sudo chown -R decider:decider /opt/decider/python3.8.10 + +cd .. +sudo rm -rf Python-3.8.10 +rm Python-3.8.10.tar.xz +``` + + +### Create & Populate virtualenv +```bash +sudo -u decider -g decider /opt/decider/python3.8.10/bin/python3.8 -m venv /opt/decider/1.0.0/venv/ +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m pip --no-cache-dir install wheel==0.37.1 +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m pip --no-cache-dir install -r /opt/decider/1.0.0/requirements.txt +``` + + +### Create user.json file & Initialize DB +```bash +sudo -u decider -g decider cp /opt/decider/1.0.0/.env.manual /opt/decider/1.0.0/.env +sudo -u decider -g decider chmod 660 /opt/decider/1.0.0/.env +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python /opt/decider/1.0.0/initial_setup.py +sudo -u postgres psql -a -f /opt/decider/1.0.0/init.sql +sudo -u decider -g decider rm /opt/decider/1.0.0/init.sql +``` + + +### Configure Logging +- [Configuring Logging](https://docs.python.org/3.8/howto/logging.html#configuring-logging) +```bash +# (optional) +# sudo -u decider -g decider nano --restricted /opt/decider/1.0.0/app/logging_conf.yaml +``` + + +### Build Database +```bash +cd /opt/decider/1.0.0/ +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m app.utils.db.actions.full_build --config DefaultConfig +sudo -u decider -g decider rm /opt/decider/1.0.0/app/utils/jsons/source/user.json +``` + +### Add UFW Exception +```bash +# (optional - only needed if using & running UFW) +# sudo ufw allow 443/tcp +``` + + +### Generate Self-Signed SSL Cert / Add Your Own +- **If you have your own cert already** - don't run the code, just write these 2 files: + - /opt/decider/1.0.0/app/utils/certs/decider.key + - /opt/decider/1.0.0/app/utils/certs/decider.crt +```bash +sudo -u decider -g decider openssl genrsa \ + -out /opt/decider/1.0.0/app/utils/certs/decider.key 2048 +sudo -u decider -g decider openssl req -new \ + -key /opt/decider/1.0.0/app/utils/certs/decider.key \ + -out /opt/decider/1.0.0/app/utils/certs/decider.csr +sudo -u decider -g decider openssl x509 -req -days 365 \ + -in /opt/decider/1.0.0/app/utils/certs/decider.csr \ + -signkey /opt/decider/1.0.0/app/utils/certs/decider.key \ + -out /opt/decider/1.0.0/app/utils/certs/decider.crt +``` + + +**ToDo: Remove** - Nukes and remakes Postgres +```bash +sudo pg_ctlcluster stop 14 main; +sudo systemctl stop postgresql; +sudo pg_dropcluster 14 main; +sudo systemctl start postgresql; +sudo pg_createcluster 14 main; +sudo pg_ctlcluster start 14 main; +sudo pg_ctlcluster status 14 main; +sudo systemctl status postgresql; +``` \ No newline at end of file diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100644 index 0000000..d5238b3 --- /dev/null +++ b/uwsgi.ini @@ -0,0 +1,11 @@ +[uwsgi] +chdir=/opt/decider/1.0.0 +module = decider:app +master = true +processes = 5 +pyargv = --config DefaultConfig +shared-socket = 0.0.0.0:443 +uid = decider +gid = decider +https = =0,/opt/decider/1.0.0/app/utils/certs/decider.crt,/opt/decider/1.0.0/app/utils/certs/decider.key +enable-threads = true From b83cc7be39fea56866385e506c00841405df4bfe Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Thu, 23 Mar 2023 12:11:49 -0400 Subject: [PATCH 04/10] Finished Ubuntu install instructions. Created a script that performs the install. --- docs/install/Ubuntu_22.04.2.md | 85 ++++++++++++++++++++++++-------- docs/install/ubuntu_22_04_2.sh | 89 ++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 19 deletions(-) create mode 100755 docs/install/ubuntu_22_04_2.sh diff --git a/docs/install/Ubuntu_22.04.2.md b/docs/install/Ubuntu_22.04.2.md index a86bfa1..c0a1af5 100644 --- a/docs/install/Ubuntu_22.04.2.md +++ b/docs/install/Ubuntu_22.04.2.md @@ -1,7 +1,30 @@ # Ubuntu Desktop (Jammy) 22.04.2 LTS +## Install Script +- See **ubuntu_22_04_2.sh** for the adventurous.. **otherwise just follow this file** + - Not idempotent + - No error handling + - Doesn't ask you to change default passwords + - Good for setup on a clean OS install + + +## Install Note +- Some files are created during the installation (in current dir) + - Best to give yourself a temp dir + - Make sure to delete this temp folder post-intall +```bash +cd +mkdir decider_temp +cd decider_temp +``` + + +## Install Instructions + + ### Update Package Sources +- Ensures package listing is up-to-date ```bash sudo apt update @@ -11,6 +34,7 @@ sudo apt update ### Create Decider Service Account + Group +- Dedicated no-home, no-login, shell-less user prevents app from accessing more ```bash sudo adduser --no-create-home --system --shell /bin/false decider sudo usermod -L decider @@ -32,12 +56,14 @@ sudo systemctl status postgresql ```bash sudo apt install -y git sudo mkdir /opt/decider -sudo chown decider:decider /opt/decider -sudo -u decider -g decider git clone https://github.com/cisagov/decider.git /opt/decider/1.0.0 +git clone https://github.com/cisagov/decider.git +sudo cp -a ./decider/. /opt/decider/1.0.0 +sudo chown -R decider:decider /opt/decider ``` ### Install Python 3.8.10 (as Ubuntu 22.04 has Python 3.10.6) +- Useful as a means of isolation as well (never depends on system versions) - [Build Dependencies Reference](https://devguide.python.org/getting-started/setup-building/index.html#install-dependencies) ```bash sudo apt install -y build-essential gdb lcov pkg-config \ @@ -57,16 +83,18 @@ sudo make altinstall sudo chown -R decider:decider /opt/decider/python3.8.10 cd .. -sudo rm -rf Python-3.8.10 -rm Python-3.8.10.tar.xz ``` ### Create & Populate virtualenv +- Useful instead of installing directly into Decider's own Py3.8.10 - as future versions could change packages in use ```bash -sudo -u decider -g decider /opt/decider/python3.8.10/bin/python3.8 -m venv /opt/decider/1.0.0/venv/ -sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m pip --no-cache-dir install wheel==0.37.1 -sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m pip --no-cache-dir install -r /opt/decider/1.0.0/requirements.txt +sudo -u decider -g decider /opt/decider/python3.8.10/bin/python3.8 -m \ + venv /opt/decider/1.0.0/venv/ +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ + pip --no-cache-dir install wheel==0.37.1 +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ + pip --no-cache-dir install -r /opt/decider/1.0.0/requirements.txt ``` @@ -75,12 +103,13 @@ sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m pip --no-cache- sudo -u decider -g decider cp /opt/decider/1.0.0/.env.manual /opt/decider/1.0.0/.env sudo -u decider -g decider chmod 660 /opt/decider/1.0.0/.env sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python /opt/decider/1.0.0/initial_setup.py -sudo -u postgres psql -a -f /opt/decider/1.0.0/init.sql +sudo -i -u postgres psql -a -f /opt/decider/1.0.0/init.sql sudo -u decider -g decider rm /opt/decider/1.0.0/init.sql ``` ### Configure Logging +- Logs DEBUG to decider.log and stdout by default - [Configuring Logging](https://docs.python.org/3.8/howto/logging.html#configuring-logging) ```bash # (optional) @@ -88,10 +117,19 @@ sudo -u decider -g decider rm /opt/decider/1.0.0/init.sql ``` +### Configure Content to be Built onto the DB (optional) +- ATT&CK Enterprise v11.0 & v12.0 are built by default (as of Mar 2023) + - This includes co-occurrences for each version (**Frequently Appears With** on Tech success pages) +- Configuration Information + - Visit the **Admin Guide** (Decider_Admin_Guide_v1.0.0.pdf in docs) + - Go to the section **Database Setup** (bottom of page 12) + + ### Build Database ```bash cd /opt/decider/1.0.0/ -sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m app.utils.db.actions.full_build --config DefaultConfig +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ + app.utils.db.actions.full_build --config DefaultConfig sudo -u decider -g decider rm /opt/decider/1.0.0/app/utils/jsons/source/user.json ``` @@ -119,14 +157,23 @@ sudo -u decider -g decider openssl x509 -req -days 365 \ ``` -**ToDo: Remove** - Nukes and remakes Postgres +### Launch Decider +- Runs as a systemd service ```bash -sudo pg_ctlcluster stop 14 main; -sudo systemctl stop postgresql; -sudo pg_dropcluster 14 main; -sudo systemctl start postgresql; -sudo pg_createcluster 14 main; -sudo pg_ctlcluster start 14 main; -sudo pg_ctlcluster status 14 main; -sudo systemctl status postgresql; -``` \ No newline at end of file +# (optional - allows tweaking uwsgi threads, decider port, etc) +# sudo -u decider -g decider nano --restricted /opt/decider/1.0.0/uwsgi.ini + +# (alternative - Decider can be launched without systemd) +# sudo /opt/decider/1.0.0/venv/bin/uwsgi --ini /opt/decider/1.0.0/uwsgi.ini + +sudo cp /opt/decider/1.0.0/decider.service /etc/systemd/system/decider.service +sudo chmod 644 /etc/systemd/system/decider.service +sudo systemctl start decider +sudo systemctl status decider +sudo systemctl enable decider +``` + + +### Default Login +- **email:** admin@admin.com +- **password:** admin diff --git a/docs/install/ubuntu_22_04_2.sh b/docs/install/ubuntu_22_04_2.sh new file mode 100755 index 0000000..e5498b5 --- /dev/null +++ b/docs/install/ubuntu_22_04_2.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# This is NOT idempotent +# This is intended for a fresh OS install +# This has no error handling + +# Ubuntu Desktop (Jammy) 22.04.2 LTS + +# temp dir for git clone & python building +cd +mkdir decider_temp +cd decider_temp + +# ensure updated listing +sudo apt update + +# decider service account +sudo adduser --no-create-home --system --shell /bin/false decider +sudo usermod -L decider +sudo groupadd decider +sudo usermod -aG decider decider + +# install postgres +sudo apt install -y postgresql postgresql-contrib +sudo systemctl start postgresql +sudo systemctl enable postgresql +sudo systemctl status postgresql + +# clone repo +sudo apt install -y git +sudo mkdir /opt/decider +git clone https://github.com/cisagov/decider.git +sudo cp -a ./decider/. /opt/decider/1.0.0 +sudo chown -R decider:decider /opt/decider + +# build python 3.8.10 +sudo apt install -y build-essential gdb lcov pkg-config \ + libbz2-dev libffi-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ + libncurses5-dev libreadline6-dev libsqlite3-dev libssl-dev \ + lzma lzma-dev tk-dev uuid-dev zlib1g-dev +wget https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tar.xz +tar -xf Python-3.8.10.tar.xz +cd Python-3.8.10 +./configure --prefix=/opt/decider/python3.8.10 --exec_prefix=/opt/decider/python3.8.10 --enable-optimizations +make -j $(( $(nproc) + 1 )) +sudo mkdir /opt/decider/python3.8.10 +sudo make altinstall +sudo chown -R decider:decider /opt/decider/python3.8.10 +cd .. + +# setup venv +sudo -u decider -g decider /opt/decider/python3.8.10/bin/python3.8 -m \ + venv /opt/decider/1.0.0/venv/ +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ + pip --no-cache-dir install wheel==0.37.1 +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ + pip --no-cache-dir install -r /opt/decider/1.0.0/requirements.txt + +# create user.json (for build), create/run/rm init.sql (for DB init) +sudo -u decider -g decider cp /opt/decider/1.0.0/.env.manual /opt/decider/1.0.0/.env +sudo -u decider -g decider chmod 660 /opt/decider/1.0.0/.env +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python /opt/decider/1.0.0/initial_setup.py +sudo -i -u postgres psql -a -f /opt/decider/1.0.0/init.sql +sudo -u decider -g decider rm /opt/decider/1.0.0/init.sql + +# build database +cd /opt/decider/1.0.0/ +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ + app.utils.db.actions.full_build --config DefaultConfig +sudo -u decider -g decider rm /opt/decider/1.0.0/app/utils/jsons/source/user.json + +# generate self-signed ssl cert +sudo -u decider -g decider openssl genrsa \ + -out /opt/decider/1.0.0/app/utils/certs/decider.key 2048 +sudo -u decider -g decider openssl req -new \ + -key /opt/decider/1.0.0/app/utils/certs/decider.key \ + -out /opt/decider/1.0.0/app/utils/certs/decider.csr +sudo -u decider -g decider openssl x509 -req -days 365 \ + -in /opt/decider/1.0.0/app/utils/certs/decider.csr \ + -signkey /opt/decider/1.0.0/app/utils/certs/decider.key \ + -out /opt/decider/1.0.0/app/utils/certs/decider.crt + +# copy service file and start +sudo cp /opt/decider/1.0.0/decider.service /etc/systemd/system/decider.service +sudo chmod 644 /etc/systemd/system/decider.service +sudo systemctl start decider +sudo systemctl status decider +sudo systemctl enable decider +echo "Default Login: admin@admin.com admin" From 4e3cf5ab1a35cc2c0fb604b305f4295b33801a96 Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Thu, 23 Mar 2023 12:07:49 -0700 Subject: [PATCH 05/10] Added CentOS 7 install instructions --- docs/install/CentOS_7.md | 217 +++++++++++++++++++++++++++++++++ docs/install/ubuntu_22_04_2.sh | 2 - 2 files changed, 217 insertions(+), 2 deletions(-) create mode 100644 docs/install/CentOS_7.md diff --git a/docs/install/CentOS_7.md b/docs/install/CentOS_7.md new file mode 100644 index 0000000..7271181 --- /dev/null +++ b/docs/install/CentOS_7.md @@ -0,0 +1,217 @@ +# CentOS 7 + + +## Install Note +- Some files are created during the installation (in current dir) + - Best to give yourself a temp dir + - Make sure to delete this temp folder post-intall +```bash +cd ;\ +mkdir decider_temp ;\ +cd decider_temp +``` + + +## Install Instructions + + +### Add Yourself to Sudoers +- Administrators are likely already in there +```bash +su +EDITOR=nano visudo + +# Comment-block is editor view (Scroll Down) +# user "damion" was added in this example + +# ## Allow root to run any commands anywhere +# root ALL=(ALL) ALL +# damion ALL=(ALL) ALL + +exit +``` + + +### Update Package Sources +- Ensures package listing is up-to-date +```bash +yum check-update +``` + + +### Create Decider Service Account + Group +- Dedicated no-home, no-login, shell-less user prevents app from accessing more +```bash +sudo adduser --no-create-home --system --shell /bin/false decider ;\ +sudo usermod -L decider ;\ +sudo groupadd decider ;\ +sudo usermod -aG decider decider +``` + + +### Install PostgreSQL +- Adds postgres to repositories (GPG keys too), then installs and enables it +```bash + +sudo yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm ;\ +sudo yum -y list postgres* ;\ +sudo yum -y install postgresql12 postgresql12-server postgresql12-contrib postgresql12-libs ;\ +sudo systemctl enable postgresql-12 +``` + + +### Initialize & Start Postgres +```bash +sudo /usr/pgsql-12/bin/postgresql-12-setup initdb ;\ +sudo systemctl restart postgresql-12 ;\ +sudo systemctl status postgresql-12 +``` + + +### Clone Repository +```bash +sudo yum install -y git ;\ +sudo mkdir /opt/decider ;\ +git clone https://github.com/cisagov/decider.git ;\ +sudo cp -a ./decider/. /opt/decider/1.0.0 ;\ +sudo chown -R decider:decider /opt/decider +``` + + +### Install Python 3.8.10 (as CentOS 7 has Python 3.6.8) +- Useful as a means of isolation as well (never depends on system versions) +- [Build Dependencies Reference](https://devguide.python.org/getting-started/setup-building/index.html#install-dependencies) +```bash +sudo yum install -y yum-utils openssl-devel ;\ +sudo yum-builddep -y python3 ;\ +wget https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tar.xz ;\ +tar -xf Python-3.8.10.tar.xz ;\ +cd Python-3.8.10 ;\ +./configure --prefix=/opt/decider/python3.8.10 --exec_prefix=/opt/decider/python3.8.10 --enable-optimizations ;\ +make -j $(( $(nproc) + 1 )) ;\ +sudo mkdir /opt/decider/python3.8.10 ;\ +sudo make altinstall ;\ +sudo chown -R decider:decider /opt/decider/python3.8.10 ;\ +cd .. +``` +################################################# WE ARE HERE + +### Create & Populate virtualenv +- Useful instead of installing directly into Decider's own Py3.8.10 - as future versions could change packages in use +```bash +sudo -u decider -g decider /opt/decider/python3.8.10/bin/python3.8 -m \ + venv /opt/decider/1.0.0/venv/ ;\ +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ + pip --no-cache-dir install wheel==0.37.1 ;\ +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ + pip --no-cache-dir install -r /opt/decider/1.0.0/requirements.txt +``` + + +### Create user.json file & Initialize DB +```bash +sudo -u decider -g decider cp /opt/decider/1.0.0/.env.manual /opt/decider/1.0.0/.env ;\ +sudo -u decider -g decider chmod 660 /opt/decider/1.0.0/.env ;\ +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python /opt/decider/1.0.0/initial_setup.py ;\ +sudo -i -u postgres psql -a -f /opt/decider/1.0.0/init.sql ;\ +sudo -u decider -g decider rm /opt/decider/1.0.0/init.sql +``` + + +### Modify Postgres's Authentication Away From Ident +- Our user uses a password, it is not a system account + - SQLAlchemy connects to Postgres over ipv4 or ipv6 - which is 'host' type + - Solves problem of `(psycopg2.OperationalError) FATAL: Ident authentication failed for user "deciderdbuser"` +```bash +# shows file location (already in next command) +sudo -i -u postgres psql -U postgres -c 'SHOW hba_file' ;\ +sudo -i -u postgres nano /var/lib/pgsql/12/data/pg_hba.conf ;\ +sudo -i -u postgres psql -c 'SELECT pg_reload_conf()'; + +# EDIT TO MAKE WHEN EDITOR APPEARS (Scroll Down) +# +# # TYPE DATABASE USER ADDRESS METHOD +# +# # "local" is for Unix domain socket connections only +# local all all peer +# # IPv4 local connections: +# host all all 127.0.0.1/32 ident <---CHANGE-THIS-TO-md5---| +# # IPv6 local connections: +# host all all ::1/128 ident <---CHANGE-THIS-TO-md5---| +# # Allow replication connections from localhost, by a user with the +# # replication privilege. +# local replication all peer +# host replication all 127.0.0.1/32 ident +# host replication all ::1/128 ident +``` + + +### Configure Logging +- Logs DEBUG to decider.log and stdout by default +- [Configuring Logging](https://docs.python.org/3.8/howto/logging.html#configuring-logging) +```bash +# (optional) +# sudo -u decider -g decider nano --restricted /opt/decider/1.0.0/app/logging_conf.yaml +``` + + +### Configure Content to be Built onto the DB (optional) +- ATT&CK Enterprise v11.0 & v12.0 are built by default (as of Mar 2023) + - This includes co-occurrences for each version (**Frequently Appears With** on Tech success pages) +- Configuration Information + - Visit the **Admin Guide** (Decider_Admin_Guide_v1.0.0.pdf in docs) + - Go to the section **Database Setup** (bottom of page 12) + + +### Build Database +```bash +cd /opt/decider/1.0.0/ ;\ +sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ + app.utils.db.actions.full_build --config DefaultConfig ;\ +sudo -u decider -g decider rm /opt/decider/1.0.0/app/utils/jsons/source/user.json +``` + +### Add UFW Exception +```bash +sudo firewall-cmd --zone=public --add-port=443/tcp --permanent ;\ +sudo firewall-cmd --reload +``` + + +### Generate Self-Signed SSL Cert / Add Your Own +- **If you have your own cert already** - don't run the code, just write these 2 files: + - /opt/decider/1.0.0/app/utils/certs/decider.key + - /opt/decider/1.0.0/app/utils/certs/decider.crt +```bash +sudo -u decider -g decider RANDFILE=/opt/decider/1.0.0/app/utils/certs/.rnd openssl genrsa \ + -out /opt/decider/1.0.0/app/utils/certs/decider.key 2048 ;\ +sudo -u decider -g decider RANDFILE=/opt/decider/1.0.0/app/utils/certs/.rnd openssl req -new \ + -key /opt/decider/1.0.0/app/utils/certs/decider.key \ + -out /opt/decider/1.0.0/app/utils/certs/decider.csr ;\ +sudo -u decider -g decider RANDFILE=/opt/decider/1.0.0/app/utils/certs/.rnd openssl x509 -req -days 365 \ + -in /opt/decider/1.0.0/app/utils/certs/decider.csr \ + -signkey /opt/decider/1.0.0/app/utils/certs/decider.key \ + -out /opt/decider/1.0.0/app/utils/certs/decider.crt +``` + + +### Launch Decider +- Runs as a systemd service +```bash +# (optional - allows tweaking uwsgi threads, decider port, etc) +# sudo -u decider -g decider nano --restricted /opt/decider/1.0.0/uwsgi.ini + +# (alternative - Decider can be launched without systemd) +# sudo /opt/decider/1.0.0/venv/bin/uwsgi --ini /opt/decider/1.0.0/uwsgi.ini + +sudo cp /opt/decider/1.0.0/decider.service /etc/systemd/system/decider.service ;\ +sudo chmod 644 /etc/systemd/system/decider.service ;\ +sudo systemctl start decider ;\ +sudo systemctl status decider ;\ +sudo systemctl enable decider +``` + + +### Default Login +- **email:** admin@admin.com +- **password:** admin diff --git a/docs/install/ubuntu_22_04_2.sh b/docs/install/ubuntu_22_04_2.sh index e5498b5..347a3cb 100755 --- a/docs/install/ubuntu_22_04_2.sh +++ b/docs/install/ubuntu_22_04_2.sh @@ -4,8 +4,6 @@ # This is intended for a fresh OS install # This has no error handling -# Ubuntu Desktop (Jammy) 22.04.2 LTS - # temp dir for git clone & python building cd mkdir decider_temp From cfe1f0eecb2da513bc4a1e172935295644270654 Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Thu, 23 Mar 2023 12:35:49 -0700 Subject: [PATCH 06/10] Update install docs to comment on bracketed paste --- docs/install/CentOS_7.md | 5 ++++- docs/install/Ubuntu_22.04.2.md | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/install/CentOS_7.md b/docs/install/CentOS_7.md index 7271181..3e14139 100644 --- a/docs/install/CentOS_7.md +++ b/docs/install/CentOS_7.md @@ -2,6 +2,8 @@ ## Install Note +- Assumes terminal bracketed paste mode is off, hence the `;\` everywhere + - Without these, a sudo prompt eats later lines of a pasted block - Some files are created during the installation (in current dir) - Best to give yourself a temp dir - Make sure to delete this temp folder post-intall @@ -82,6 +84,7 @@ sudo chown -R decider:decider /opt/decider - Useful as a means of isolation as well (never depends on system versions) - [Build Dependencies Reference](https://devguide.python.org/getting-started/setup-building/index.html#install-dependencies) ```bash +sudo yum groupinstall -y 'Development Tools' ;\ sudo yum install -y yum-utils openssl-devel ;\ sudo yum-builddep -y python3 ;\ wget https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tar.xz ;\ @@ -94,7 +97,7 @@ sudo make altinstall ;\ sudo chown -R decider:decider /opt/decider/python3.8.10 ;\ cd .. ``` -################################################# WE ARE HERE + ### Create & Populate virtualenv - Useful instead of installing directly into Decider's own Py3.8.10 - as future versions could change packages in use diff --git a/docs/install/Ubuntu_22.04.2.md b/docs/install/Ubuntu_22.04.2.md index c0a1af5..f9cc629 100644 --- a/docs/install/Ubuntu_22.04.2.md +++ b/docs/install/Ubuntu_22.04.2.md @@ -2,6 +2,8 @@ ## Install Script +- Assumes terminal bracketed paste mode is on (gnome default) + - If it is off, a sudo prompt eats later lines of a pasted block - See **ubuntu_22_04_2.sh** for the adventurous.. **otherwise just follow this file** - Not idempotent - No error handling From 7a40b5aba860dc7c102c3bd5acc801854ba90504 Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Thu, 23 Mar 2023 13:23:46 -0700 Subject: [PATCH 07/10] Fix CentOS Python Compile --- docs/install/CentOS_7.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/install/CentOS_7.md b/docs/install/CentOS_7.md index 3e14139..75fd55d 100644 --- a/docs/install/CentOS_7.md +++ b/docs/install/CentOS_7.md @@ -91,7 +91,6 @@ wget https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tar.xz ;\ tar -xf Python-3.8.10.tar.xz ;\ cd Python-3.8.10 ;\ ./configure --prefix=/opt/decider/python3.8.10 --exec_prefix=/opt/decider/python3.8.10 --enable-optimizations ;\ -make -j $(( $(nproc) + 1 )) ;\ sudo mkdir /opt/decider/python3.8.10 ;\ sudo make altinstall ;\ sudo chown -R decider:decider /opt/decider/python3.8.10 ;\ From adeb64904a56cc55b537a4155fe4a1b3d0faaeac Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Thu, 23 Mar 2023 16:43:12 -0400 Subject: [PATCH 08/10] Fix CentOS install Make Ubuntu install match CentOS install more closely --- docs/install/CentOS_7.md | 2 +- docs/install/Ubuntu_22.04.2.md | 11 +++-------- docs/install/ubuntu_22_04_2.sh | 7 +++---- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/docs/install/CentOS_7.md b/docs/install/CentOS_7.md index 75fd55d..023d096 100644 --- a/docs/install/CentOS_7.md +++ b/docs/install/CentOS_7.md @@ -173,7 +173,7 @@ sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ sudo -u decider -g decider rm /opt/decider/1.0.0/app/utils/jsons/source/user.json ``` -### Add UFW Exception +### Add Firewall Exception ```bash sudo firewall-cmd --zone=public --add-port=443/tcp --permanent ;\ sudo firewall-cmd --reload diff --git a/docs/install/Ubuntu_22.04.2.md b/docs/install/Ubuntu_22.04.2.md index f9cc629..0f240b3 100644 --- a/docs/install/Ubuntu_22.04.2.md +++ b/docs/install/Ubuntu_22.04.2.md @@ -72,18 +72,13 @@ sudo apt install -y build-essential gdb lcov pkg-config \ libbz2-dev libffi-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \ libncurses5-dev libreadline6-dev libsqlite3-dev libssl-dev \ lzma lzma-dev tk-dev uuid-dev zlib1g-dev - wget https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tar.xz tar -xf Python-3.8.10.tar.xz cd Python-3.8.10 - ./configure --prefix=/opt/decider/python3.8.10 --exec_prefix=/opt/decider/python3.8.10 --enable-optimizations -make -j $(( $(nproc) + 1 )) - sudo mkdir /opt/decider/python3.8.10 sudo make altinstall sudo chown -R decider:decider /opt/decider/python3.8.10 - cd .. ``` @@ -147,12 +142,12 @@ sudo -u decider -g decider rm /opt/decider/1.0.0/app/utils/jsons/source/user.jso - /opt/decider/1.0.0/app/utils/certs/decider.key - /opt/decider/1.0.0/app/utils/certs/decider.crt ```bash -sudo -u decider -g decider openssl genrsa \ +sudo -u decider -g decider RANDFILE=/opt/decider/1.0.0/app/utils/certs/.rnd openssl genrsa \ -out /opt/decider/1.0.0/app/utils/certs/decider.key 2048 -sudo -u decider -g decider openssl req -new \ +sudo -u decider -g decider RANDFILE=/opt/decider/1.0.0/app/utils/certs/.rnd openssl req -new \ -key /opt/decider/1.0.0/app/utils/certs/decider.key \ -out /opt/decider/1.0.0/app/utils/certs/decider.csr -sudo -u decider -g decider openssl x509 -req -days 365 \ +sudo -u decider -g decider RANDFILE=/opt/decider/1.0.0/app/utils/certs/.rnd openssl x509 -req -days 365 \ -in /opt/decider/1.0.0/app/utils/certs/decider.csr \ -signkey /opt/decider/1.0.0/app/utils/certs/decider.key \ -out /opt/decider/1.0.0/app/utils/certs/decider.crt diff --git a/docs/install/ubuntu_22_04_2.sh b/docs/install/ubuntu_22_04_2.sh index 347a3cb..481d32a 100755 --- a/docs/install/ubuntu_22_04_2.sh +++ b/docs/install/ubuntu_22_04_2.sh @@ -40,7 +40,6 @@ wget https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tar.xz tar -xf Python-3.8.10.tar.xz cd Python-3.8.10 ./configure --prefix=/opt/decider/python3.8.10 --exec_prefix=/opt/decider/python3.8.10 --enable-optimizations -make -j $(( $(nproc) + 1 )) sudo mkdir /opt/decider/python3.8.10 sudo make altinstall sudo chown -R decider:decider /opt/decider/python3.8.10 @@ -68,12 +67,12 @@ sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ sudo -u decider -g decider rm /opt/decider/1.0.0/app/utils/jsons/source/user.json # generate self-signed ssl cert -sudo -u decider -g decider openssl genrsa \ +sudo -u decider -g decider RANDFILE=/opt/decider/1.0.0/app/utils/certs/.rnd openssl genrsa \ -out /opt/decider/1.0.0/app/utils/certs/decider.key 2048 -sudo -u decider -g decider openssl req -new \ +sudo -u decider -g decider RANDFILE=/opt/decider/1.0.0/app/utils/certs/.rnd openssl req -new \ -key /opt/decider/1.0.0/app/utils/certs/decider.key \ -out /opt/decider/1.0.0/app/utils/certs/decider.csr -sudo -u decider -g decider openssl x509 -req -days 365 \ +sudo -u decider -g decider RANDFILE=/opt/decider/1.0.0/app/utils/certs/.rnd openssl x509 -req -days 365 \ -in /opt/decider/1.0.0/app/utils/certs/decider.csr \ -signkey /opt/decider/1.0.0/app/utils/certs/decider.key \ -out /opt/decider/1.0.0/app/utils/certs/decider.crt From 04167683b52d30a3a2615bdf6463a93970af9eb1 Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Thu, 23 Mar 2023 16:54:28 -0400 Subject: [PATCH 09/10] Update readme to mention new install instructions --- README.md | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index e58e686..a2916a6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # Decider +## Notifications +- Manual installation for Ubuntu & CentOS is much nicer. + - Scroll down to **Manual Install** for details! +- Will be adding information about hardware requirements soon + ## What is it? ### The Short @@ -92,24 +97,23 @@ It is ready when **Starting uWSGI** appears ### Manual Install -#### Ubuntu -**in-progress** +#### Ubuntu 22.04 +[Ubuntu Install Guide](docs/install/Ubuntu_22.04.2.md) -#### CentOS -**in-progress** +#### CentOS 7 +[CentOS Install Guide](docs/install/CentOS_7.md) -**Read the [Admin Guide](./docs/Decider_Admin_Guide_v1.0.0.pdf)** +#### Other OSes +Read the Ubuntu & CentOS guides and recreate actions according to your platform. -There are some issues in the instructions... ***Working on it, simplifying them*** +##### Windows +`open()` in Python uses the system's default text encoding +- This is `utf-8` on macOS and Linux +- This is `windows-1252` on Windows + - This causes issues in reading the jsons for the database build process + - Adding `encoding='utf-8'` as an arg in each `open()` ***may*** allow Windows deployment -Help Tips: -- Use Python 3.8.10 / 3.8.x on Linux / mac -- Follow the order of instructions -- Watch out using `sudo` with `python` - it won't keep the venv you're in by default -- If just running for yourself locally: - - Don't create a system account for decider - - Don't use uWSGI - - Use the built-in debug Flask server -- Mac M1 users should install Postgres before installing the pip requirements - - `brew install postgresql` - - **Explained:** *psycopg2-binary* isn't using a pre-built binary and tries to compile from scratch, and it can't find *pg_config*. +##### macOS +(M1 users at least) Make sure to (1) install Postgres before (2) installing the pip requirements +1. `brew install postgresql` +2. `pip install -r requirements.txt` From 7de558a4d71c16128a438c260065b1f5ff54b266 Mon Sep 17 00:00:00 2001 From: Damion R Mounts Date: Thu, 23 Mar 2023 17:01:34 -0400 Subject: [PATCH 10/10] Mention that changing passwords from default is good --- docs/install/CentOS_7.md | 1 + docs/install/Ubuntu_22.04.2.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/install/CentOS_7.md b/docs/install/CentOS_7.md index 023d096..d76c2c3 100644 --- a/docs/install/CentOS_7.md +++ b/docs/install/CentOS_7.md @@ -111,6 +111,7 @@ sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ ### Create user.json file & Initialize DB +- **'Optional:'** Change default passwords in .env after copy command ```bash sudo -u decider -g decider cp /opt/decider/1.0.0/.env.manual /opt/decider/1.0.0/.env ;\ sudo -u decider -g decider chmod 660 /opt/decider/1.0.0/.env ;\ diff --git a/docs/install/Ubuntu_22.04.2.md b/docs/install/Ubuntu_22.04.2.md index 0f240b3..538b520 100644 --- a/docs/install/Ubuntu_22.04.2.md +++ b/docs/install/Ubuntu_22.04.2.md @@ -96,6 +96,7 @@ sudo -u decider -g decider /opt/decider/1.0.0/venv/bin/python -m \ ### Create user.json file & Initialize DB +- **'Optional:'** Change default passwords in .env after copy command ```bash sudo -u decider -g decider cp /opt/decider/1.0.0/.env.manual /opt/decider/1.0.0/.env sudo -u decider -g decider chmod 660 /opt/decider/1.0.0/.env