first commit

.gitignore

@@ -0,0 +1 @@
+ripley

README.md

@@ -0,0 +1,98 @@
+# ripley - replay HTTP
+
+Ripley replays HTTP traffic at multiples of the original rate. It simulates traffic ramp up or down by specifying rate phases for each run. For example, you can replay HTTP requests at twice the original rate for ten minutes, then three times the original rate for five minutes, then ten times the original rate for an hour and so on. Ripley's original use case is load testing by replaying HTTP access logs from production applications.
+
+## Quickstart
+
+Clone and build ripley
+
+```bash
+git clone git@github.com:loveholidays/ripley.git
+cd ripley
+go build -o ripley main.go
+```
+
+Run a web server to replay traffic against
+
+```bash
+go run etc/dummyweb.go
+```
+
+Loop 10 times over a set of HTTP requests at 1x rate for 10 seconds, then at 5x for 10 seconds, then at 10x for the remaining requests
+
+```bash
+seq 10 | xargs -i cat etc/requests.jsonl | ./ripley -pace "10s@1 10s@5 1h@10"
+```
+
+## Replaying HTTP traffic
+
+Ripley reads a representation of HTTP requests in [JSON Lines format](https://jsonlines.org/) from `STDIN` and replays them at different rates in phases as specified by the `-pace` flag.
+
+An example ripley request:
+
+```JSON
+{
+    "url": "http://localhost:8080/",
+    "verb": "GET",
+    "timestamp": "2021-11-08T18:59:59.9Z",
+    "headers": {"Accept": "text/plain"}
+}
+```
+
+`url`, `verb` and `timestamp` are required, `headers` are optional.
+
+`-pace` specifies rate phases in `[duration]@[rate]` format. For example, `10s@5 5m@10 1h30m@100` means replay traffic at 5x for 10 seconds, 10x for 5 minutes and 100x for one and a half hours. The run will stop either when ripley stops receiving requests from `STDIN` or when the last phase elapses, whichever happens first.
+
+Ripley writes request results as JSON Lines to `STDOUT`
+
+```bash
+echo '{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:50.9Z"}' | ./ripley | jq
+```
+
+produces
+
+```JSON
+{
+  "statusCode": 200,
+  "latency": 3915447,
+  "request": {
+    "verb": "GET",
+    "url": "http://localhost:8080/",
+    "body": "",
+    "timestamp": "2021-11-08T18:59:50.9Z",
+    "headers": null
+  }
+}
+```
+
+Results output can be suppressed using the `-silent` flag.
+
+It is possible to collect and print a run's statistics:
+
+```bash
+seq 10 | xargs -i cat etc/requests.jsonl | ./ripley -pace "10s@1 10s@5 1h@10" -silent -stats | jq
+```
+
+```JSON
+{
+  "totalRequests": 100,
+  "statusCodes": {
+    "200": 100
+  },
+  "latencyMicroseconds": {
+    "max": 2960,
+    "mean": 2008.25,
+    "median": 2085.5,
+    "min": 815,
+    "p95": 2577,
+    "p99": 2876,
+    "stdDev": 449.1945986986041
+  }
+}
+```
+
+## Running the tests
+
+```bash
+go test pkg/*go
+```

etc/dummyweb.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"log"
+	"net/http"
+	"time"
+)
+
+func handler(w http.ResponseWriter, r *http.Request) {
+	log.Printf("%v\n", time.Now().Format(time.UnixDate))
+	w.Write([]byte("hi\n"))
+}
+
+func main() {
+	http.HandleFunc("/", handler)
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}

etc/requests.jsonl

@@ -0,0 +1,10 @@
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:50.9Z"}
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:51.9Z"}
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:52.9Z"}
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:53.9Z"}
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:54.9Z"}
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:55.9Z"}
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:56.9Z"}
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:57.9Z"}
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:58.9Z"}
+{"url": "http://localhost:8080/", "verb": "GET", "timestamp": "2021-11-08T18:59:59.9Z", "headers": {"Accept": "text/plain"}}

go.mod

@@ -0,0 +1,5 @@
+module github.com/loveholidays/ripley
+
+go 1.17
+
+require github.com/montanaflynn/stats v0.6.6

go.sum

@@ -0,0 +1,2 @@
+github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ=
+github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=

main.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"flag"
+	"github.com/loveholidays/ripley/pkg"
+)
+
+func main() {
+	paceStr := flag.String("pace", "10s@1", `[duration]@[rate], e.g. "1m@1 30s@1.5 1h@2"`)
+	silent := flag.Bool("silent", false, "Suppress output")
+	printStats := flag.Bool("stats", false, "Collect and print statistics before the program exits")
+
+	flag.Parse()
+
+	ripley.Replay(*paceStr, *silent, *printStats)
+}

pkg/client.go

@@ -0,0 +1,48 @@
+package ripley
+
+import (
+	"net/http"
+	"time"
+)
+
+type result struct {
+	StatusCode int           `json:"statusCode"`
+	Latency    time.Duration `json:"latency"`
+	Request    *request      `json:"request"`
+	err        error
+}
+
+func startClientWorkers(numWorkers int, requests <-chan *request, results chan<- *result) {
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	for i := 0; i <= numWorkers; i++ {
+		go doHttpRequest(client, requests, results)
+	}
+}
+
+func doHttpRequest(client *http.Client, requests <-chan *request, results chan<- *result) {
+	for req := range requests {
+		latencyStart := time.Now()
+		httpReq, err := req.httpRequest()
+
+		if err != nil {
+			results <- &result{err: err}
+			return
+		}
+
+		resp, err := client.Do(httpReq)
+
+		if err != nil {
+			results <- &result{err: err}
+			return
+		}
+
+		latency := time.Now().Sub(latencyStart)
+		results <- &result{StatusCode: resp.StatusCode, Latency: latency, Request: req}
+	}
+}

pkg/pace.go

@@ -0,0 +1,82 @@
+package ripley
+
+import (
+	"strconv"
+	"strings"
+	"time"
+)
+
+type pacer struct {
+	phases          []*phase
+	lastRequestTime time.Time
+	done            bool
+}
+
+type phase struct {
+	duration time.Duration
+	rate     float64
+}
+
+func newPacer(phasesStr string) (*pacer, error) {
+	phases, err := parsePhases(phasesStr)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &pacer{phases: phases}, nil
+}
+
+func (p *pacer) start() {
+	// Run a timer for the first phase's duration
+	time.AfterFunc(p.phases[0].duration, p.onPhaseElapsed)
+}
+
+func (p *pacer) onPhaseElapsed() {
+	// Pop phase
+	p.phases = p.phases[1:]
+
+	if len(p.phases) == 0 {
+		p.done = true
+	} else {
+		// Create a timer with next phase
+		time.AfterFunc(p.phases[0].duration, p.onPhaseElapsed)
+	}
+}
+
+func (p *pacer) waitDuration(t time.Time) time.Duration {
+	// If there are no more phases left, continue with the last phase's rate
+	if p.lastRequestTime.IsZero() {
+		p.lastRequestTime = t
+	}
+
+	duration := t.Sub(p.lastRequestTime)
+	p.lastRequestTime = t
+	return time.Duration(float64(duration) / p.phases[0].rate)
+}
+
+// Format is [duration]@[rate] [duration]@[rate]..."
+// e.g. "5s@1 10m@2"
+func parsePhases(phasesStr string) ([]*phase, error) {
+	var phases []*phase
+
+	for _, durationAtRate := range strings.Split(phasesStr, " ") {
+		tokens := strings.Split(durationAtRate, "@")
+
+		duration, err := time.ParseDuration(tokens[0])
+
+		if err != nil {
+			return nil, err
+		}
+
+		rate, err := strconv.ParseFloat(tokens[1], 64)
+
+		if err != nil {
+			return nil, err
+		}
+
+		phases = append(phases, &phase{duration, rate})
+	}
+
+	return phases, nil
+}