Merge pull request paritytech#66 from subspace/working-build-run-inst…

…ructions

Working build run instructions

.dockerignore

@@ -0,0 +1,10 @@
+*
+# Just enough files for `git rev-parse --short HEAD` to work (used in Substrate node build)
+!/.git/refs/heads
+!/.git/config
+!/.git/HEAD
+!/crates
+!/node-template-subspace
+!/substrate
+!/Cargo.lock
+!/Cargo.toml

crates/subspace-farmer/Dockerfile → Dockerfile-farmer

@@ -7,38 +7,32 @@ RUN \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         ca-certificates \
         curl \
-        gcc \
+        git \
         llvm \
-        clang \
-        make \
-        m4 && \
+        clang && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
 
+RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown
+
 COPY Cargo.lock /code/Cargo.lock
 COPY Cargo.toml /code/Cargo.toml
 
-# Hack to make Cargo download and cache dependencies
-RUN \
-    mkdir src && \
-    echo "fn main() {}" > src/main.rs && \
-    /root/.cargo/bin/cargo build --release && \
-    rm -rf src
+# Up until this line all Rust images in this repo should be the same to share the same layers
 
-COPY src /code/src
+COPY crates /code/crates
+COPY substrate /code/substrate
+COPY node-template-subspace /code/node-template-subspace
 
 RUN \
-    # TODO: Next line is a workaround for https://github.com/rust-lang/cargo/issues/7969
-    touch src/main.rs && \
-    /root/.cargo/bin/cargo test --release && \
-    /root/.cargo/bin/cargo build --release && \
+    /root/.cargo/bin/cargo build --release --bin subspace-farmer && \
     mv target/release/subspace-farmer subspace-farmer && \
     rm -rf target
 
 FROM ubuntu:20.04
 
 COPY --from=0 /code/subspace-farmer /subspace-farmer
 
-ENV SUBSAPCE_DIR=/var/subspace
+ENV SUBSPACE_DIR=/var/subspace
 
 RUN mkdir /var/subspace && chown nobody:nogroup /var/subspace
 

Dockerfile-node

@@ -0,0 +1,50 @@
+FROM ubuntu:20.04
+
+WORKDIR /code
+
+RUN \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        git \
+        llvm \
+        clang && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
+
+RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown
+
+COPY Cargo.lock /code/Cargo.lock
+COPY Cargo.toml /code/Cargo.toml
+
+# Up until this line all Rust images in this repo should be the same to share the same layers
+
+COPY .git /code/.git
+# Just an empty directory for Git to recognize it is indeed a Git repository
+RUN mkdir /code/.git/objects
+COPY crates /code/crates
+COPY substrate /code/substrate
+COPY node-template-subspace /code/node-template-subspace
+
+RUN \
+    /root/.cargo/bin/cargo build --release --bin node-template-subspace && \
+    mv target/release/node-template-subspace subspace-node && \
+    rm -rf target
+
+FROM ubuntu:20.04
+
+RUN \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=0 /code/subspace-node /subspace-node
+
+RUN mkdir /var/subspace && chown nobody:nogroup /var/subspace
+
+VOLUME /var/subspace
+
+USER nobody:nogroup
+
+ENTRYPOINT ["/subspace-node"]

README.md

@@ -18,7 +18,7 @@ The structure of this repository is the following:
 
 This is a monorepo with multiple binaries and the workflow is typical for Rust projects:
 
-- `cargo run --bin node-template-subspace -- --dev --tmp` to run [a node](node-template-subspace)
+- `cargo run --release --bin node-template-subspace -- --dev --tmp` to run [a node](node-template-subspace)
 - `cargo run --release --bin subspace-farmer -- farm` to [start farming](crates/subspace-farmer#start-the-farmer)
 
 NOTE: You need to have `nightly` version of Rust toolchain with `wasm32-unknown-unknown` target available or else you'll get a compilation error.

crates/subspace-farmer/README.md

@@ -1,14 +1,14 @@
 <div align="center">
   <h1><code>subspace-farmer</code></h1>
-  <strong>A proof-of-concept farmer for the <a href="https://subspace.network/">Subspace Network Blockchain</a></strong>
+  <strong>Farmer for the <a href="https://subspace.network/">Subspace Network Blockchain</a></strong>
 </div>
 
 ## Overview
 **Notes:** The code is un-audited and not production ready, use it at your own risk.
 
 This repo is an implementation of a Farmer for [Subspace Network Blockchain](https://subspace.network).
 
-Subspace is a proof-of-storage blockchain that resolves the farmer's dilemma, to learn more read our <a href="https://drive.google.com/file/d/1v847u_XeVf0SBz7Y7LEMXi72QfqirstL/view">white paper</a>.
+Subspace is a proof-of-storage blockchain that resolves the farmer's dilemma, to learn more read our [white paper](https://drive.google.com/file/d/1v847u_XeVf0SBz7Y7LEMXi72QfqirstL/view).
 
 ## Some Notes on Plotting
 
@@ -55,20 +55,14 @@ Use `--help` to find out all available commands and their options:
 subspace-farmer --help
 ```
 
-### Create a New Plot
+### Start the farmer
 ```
-subspace-farmer plot <optional parameters> <piece-count> <seed>
+subspace-farmer farm
 ```
 
-This will create a 1 GB plot:
-```
-subspace-farmer plot 256000 test
-```
+This will connect to local node and will try to solve on every slot notification, while also plotting all existing and new history of the blockchain in parallel.
 
-For all supported options check help:
-```
-subspace-farmer plot --help
-```
+*NOTE: You need to have a subspace-client node running before starting farmer, otherwise it will not be able to start*
 
 By default, plots are written to the OS-specific users local data directory.
 
@@ -84,37 +78,25 @@ Windows
 {FOLDERID_LocalAppData}             C:\Users\Alice\AppData\Local
 ```
 
-### Start the farmer
-```
-RUST_LOG=debug subspace-farmer farm
-```
-
-This will connect to local node and will try to solve on every slot notification.
-
-*NOTE: You need to have a subspace-client node running before starting farmer, otherwise it will not be able to start*
-
-
 ## Design
 
-The farmer has two modes: plotting and farming.
+The farmer typically runs two processes in parallel: plotting and farming.
 
 ### Plotting
-1. A genesis piece is created from a short seed.
-2. A new Schnorr key pair is generated, and the farmer ID is derived from the public key.
-3. New encodings are created by applying the time-asymmetric SLOTH permutation as `encode(genesis_piece, farmer_public_key_hash, plot_index)`
-4. Each encoding is written directly to disk.
-5. A commitment, or tag, to each encoding is created as `hmac(encoding, salt)` and stored within a binary search tree (BST).
+0. A new Schnorr key pair is generated, and the farmer ID is derived from the public key.
+1. For every archived piece of the history encoding is created by applying the time-asymmetric SLOTH permutation as `encode(genesis_piece, farmer_public_key_hash, plot_index)`
+2. Each encoding is written directly to disk.
+3. A commitment, or tag, to each encoding is created as `hmac(encoding, salt)` and stored within a binary search tree (BST).
 
 This process currently takes ~ 36 hours per TiB on a quad-core machine, but for 1 GiB plotting should take between a few seconds and a few minutes.
 
 ### Solving
-Once plotting is complete the farmer may join the network and participate in consensus.
-
 1. Connect to a client and subscribe to `slot_notifications` via JSON-RPC.
 2. Given a global challenge as `hash(epoch_randomness || slot_index)` and `SOLUTION_RANGE`.
 3. Derive local challenge as `hash(global_challenge || farmer_public_key_hash)`.
 4. Query the BST for the nearest tag to the local challenge.
 5. If it within `SOLUTION_RANGE` return a `SOLUTION` else return `None`
+6. All the above can and will happen in parallel to plotting process, so it is possible to participate right away
 
 
 

crates/subspace-farmer/src/main.rs

@@ -37,6 +37,7 @@ const BATCH_SIZE: u64 = (16 * 1024 * 1024 / PIECE_SIZE) as u64;
 // TODO: Move to codec
 // const CUDA_BATCH_SIZE: u64 = (32 * 1024) as u64;
 
+// TODO: Separate commands for erasing the plot and wiping everyting
 #[derive(Debug, Clap)]
 #[clap(about, version)]
 #[clap(setting = AppSettings::ColoredHelp)]

node-template-subspace/docker-compose.yml

@@ -1,13 +1,17 @@
 version: "3"
 services:
   node:
-    image: subspacelabs/node-template-subspace
+    image: subspacelabs/subspace-node
     networks:
       - default
       - subspace
+    volumes:
+      - source: subspace-node
+        target: /var/subspace
+        type: volume
     command: [
         "--dev",
-        "--tmp",
+        "--base-path", "/var/subspace",
         "--ws-external",
         "--bootnodes", "/ip4/${BOOTSTRAP_CLIENT_IP}/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp"
     ]
@@ -31,6 +35,5 @@ networks:
     name: subspace
 
 volumes:
+  subspace-node:
   subspace-farmer:
-    external: true
-    name: subspace-farmer-${INSTANCE_ID}

node-template-subspace/run-node-farmer-pair.sh

@@ -6,26 +6,19 @@ if [ $# -eq 0 ]; then
     exit 1
 fi
 
-BOOTSTRAP_CLIENT_IP=${2:-$(docker inspect -f "{{.NetworkSettings.Networks.subspace.IPAddress}}" node-template-subspace)}
+BOOTSTRAP_CLIENT_IP=${2:-$(docker inspect -f "{{.NetworkSettings.Networks.subspace.IPAddress}}" subspace-node)}
 
 cd $(dirname ${BASH_SOURCE[0]})
 
 export BOOTSTRAP_CLIENT_IP
 export INSTANCE_ID="$1"
 export COMPOSE_PROJECT_NAME="subspace-$INSTANCE_ID"
 stop() {
-  docker-compose down -t 3 || /bin/true
-  docker volume rm subspace-farmer-$INSTANCE_ID
+  docker-compose down -t 10 || /bin/true
 }
 
 trap 'stop' SIGINT
 
 docker-compose pull
 
-docker volume create subspace-farmer-$INSTANCE_ID
-docker run --rm -it \
-  --name subspace-farmer-$INSTANCE_ID \
-  --mount source=subspace-farmer-$INSTANCE_ID,target=/var/subspace \
-  subspacelabs/subspace-farmer plot 256000 subspace
-
 docker-compose up

node-template-subspace/start-farmer.sh

@@ -18,21 +18,15 @@ run-farm() {
     --mount source=subspace-farmer,target=/var/subspace \
     subspacelabs/subspace-farmer \
         farm \
-        --ws-server ws://node-template-subspace-full:9944
+        --ws-server ws://subspace-node-full:9944
 }
 
-plot-1gb() {
-    echo "Plotting 1gb..."
-    docker run --rm -it \
-    --name subspace-farmer \
-    --mount source=subspace-farmer,target=/var/subspace \
-    subspacelabs/subspace-farmer plot 256000 subspace
-}
 wipe() {
     echo "Wiping prior installation..."
-    docker container kill subspace-farmer
+    docker container kill subspace-farmer || /bin/true
     docker volume rm subspace-farmer
 }
+
 erase() {
     echo "Erasing plot..."
     docker container kill subspace-farmer
@@ -41,6 +35,7 @@ erase() {
     --mount source=subspace-farmer,target=/var/subspace \
     subspacelabs/subspace-farmer erase-plot
 }
+
 ##
 # Color  Variables
 ##
@@ -66,21 +61,19 @@ menu(){
     -=[Subspace - Subspace Testnet]=-
     ----------------------------------
     $(ColorGreen '1)') Setup/Update Farmer
-    $(ColorGreen '2)') Plot 1GB of Data
-    $(ColorGreen '3)') Run Farmer
-    $(ColorGreen '4)') Wipe Farmer
-    $(ColorGreen '5)') Erase Plot
+    $(ColorGreen '2)') Run Farmer
+    $(ColorGreen '3)') Wipe Farmer
+    $(ColorGreen '4)') Erase Plot
     $(ColorGreen '0)') Exit
     $(ColorBlue 'Choose an option:') $clear"
 
 
     read a
     case $a in
         1) setup ; menu ;;
-        2) plot-1gb ; menu ;;
-        3) run-farm ; menu ;;
-        4) wipe ; menu ;;
-        5) erase ; menu ;;
+        2) run-farm ; menu ;;
+        3) wipe ; menu ;;
+        4) erase ; menu ;;
         0) exit 0 ;;
         *) echo -e "Not a Valid Option, Try Again..."; menu;;
     esac