Loki Logstash Plugin (#1822)

* Logstash plugin * include_labels * include_labels * Removes binary. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Improve documentation and remove the push path. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Move to cmd. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add more precision for jruby. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Update docs/clients/logstash/README.md * p Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * ignore Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * remove ignore file/ Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * More precision for installing jruby Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Rename without Grafana Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * A lot of refactoring and testing. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * change delay logic * Fully tested version. Now testing/writing docs and charts. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Forgot to save merge. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * working version. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Makefile + easier docker build. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * adds ci to build logstash image. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fix build for logstash. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Adds example with helm charts. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fix target to send 10 logs with logstash. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Improved documentation. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * add missing helm add repo for external repo Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Review comment. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes loki service in Promtail. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Update loki-stack version Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> Co-authored-by: Cyril Tovena <cyril.tovena@gmail.com>
grafana · Jul 17, 2020 · 4c60907 · 4c60907
1 parent 99604a5
commit 4c60907
Show file tree

Hide file tree

Showing 23 changed files with 1,285 additions and 4 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -11,8 +11,7 @@ version: 2
   branches: { only: master }
   <<: *tags
 
-.no-master:
-  &no-master # contrary to tags, the branches must be excluded
+.no-master: &no-master # contrary to tags, the branches must be excluded
   branches: { ignore: master }
 
 workflows:
@@ -136,6 +135,7 @@ jobs:
       - run:
           name: Run Chart Tests
           command: |
+            helm repo add elastic https://helm.elastic.co
             ct lint --chart-dirs=production/helm --check-version-increment=false --validate-maintainers=false
             ct install --build-id=${CIRCLE_BUILD_NUM} --charts production/helm/loki-stack
 

diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet
@@ -108,6 +108,30 @@ local fluentd() = pipeline('fluentd-amd64') + arch_image('amd64', 'latest,master
   depends_on: ['check'],
 };
 
+local logstash() = pipeline('logstash-amd64') + arch_image('amd64', 'latest,master') {
+  steps+: [
+    // dry run for everything that is not tag or master
+    docker('amd64', 'logstash') {
+      depends_on: ['image-tag'],
+      when: condition('exclude').tagMaster,
+      settings+: {
+        dry_run: true,
+        repo: 'grafana/logstash-output-loki',
+      },
+    },
+  ] + [
+    // publish for tag or master
+    docker('amd64', 'logstash') {
+      depends_on: ['image-tag'],
+      when: condition('include').tagMaster,
+      settings+: {
+        repo: 'grafana/logstash-output-loki',
+      },
+    },
+  ],
+  depends_on: ['check'],
+};
+
 local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) {
   steps+: [
     // dry run for everything that is not tag or master
@@ -206,6 +230,7 @@ local manifest(apps) = pipeline('manifest') {
 ] + [
   fluentbit(),
   fluentd(),
+  logstash(),
 ] + [
   manifest(['promtail', 'loki', 'loki-canary']) {
     trigger: condition('include').tagMaster,

diff --git a/.drone/drone.yml b/.drone/drone.yml
@@ -570,6 +570,62 @@ steps:
 depends_on:
 - check
 
+---
+kind: pipeline
+name: logstash-amd64
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: image-tag
+  image: alpine
+  commands:
+  - apk add --no-cache bash git
+  - git fetch origin --tags
+  - echo $(./tools/image-tag)-amd64 > .tags
+  - echo ",latest,master" >> .tags
+
+- name: build-logstash-image
+  image: plugins/docker
+  settings:
+    dockerfile: cmd/logstash/Dockerfile
+    dry_run: true
+    password:
+      from_secret: docker_password
+    repo: grafana/logstash-output-loki
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+      exclude:
+      - refs/heads/master
+      - refs/heads/k??
+      - refs/tags/v*
+  depends_on:
+  - image-tag
+
+- name: publish-logstash-image
+  image: plugins/docker
+  settings:
+    dockerfile: cmd/logstash/Dockerfile
+    password:
+      from_secret: docker_password
+    repo: grafana/logstash-output-loki
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/heads/master
+    - refs/heads/k??
+    - refs/tags/v*
+  depends_on:
+  - image-tag
+
+depends_on:
+- check
+
 ---
 kind: pipeline
 name: manifest

diff --git a/Makefile b/Makefile
@@ -436,6 +436,25 @@ fluentd-test: LOKI_URL ?= http://localhost:3100/loki/api/
 fluentd-test:
 	LOKI_URL="$(LOKI_URL)" docker-compose -f cmd/fluentd/docker/docker-compose.yml up --build #$(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG)
 
+##################
+# logstash plugin #
+##################
+logstash-image:
+	$(SUDO) docker build -t $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f cmd/logstash/Dockerfile ./
+
+# Send 10 lines to the local Loki instance.
+logstash-push-test-logs: LOKI_URL ?= http://host.docker.internal:3100/loki/api/v1/push
+logstash-push-test-logs:
+	$(SUDO) docker run -e LOKI_URL="$(LOKI_URL)" -v `pwd`/cmd/logstash/loki-test.conf:/home/logstash/loki.conf --rm \
+		$(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG) -f loki.conf
+
+logstash-push:
+	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG)
+
+# Enter an env already configure to build and test logstash output plugin.
+logstash-env:
+	$(SUDO) docker run -v  `pwd`/cmd/logstash:/home/logstash/ -it --rm --entrypoint /bin/sh $(IMAGE_PREFIX)/logstash-output-loki:$(IMAGE_TAG)
+
 ########################
 # Bigtable Backup Tool #
 ########################

diff --git a/cmd/logstash/.gitignore b/cmd/logstash/.gitignore
@@ -0,0 +1,7 @@
+*.gem
+.ruby-version
+.bundle
+logstash
+path
+!lib
+.rakeTasks
diff --git a/cmd/logstash/Dockerfile b/cmd/logstash/Dockerfile
@@ -0,0 +1,24 @@
+FROM logstash:7.6.2
+
+USER logstash
+ENV PATH /usr/share/logstash/vendor/jruby/bin:/usr/share/logstash/vendor/bundle/jruby/2.5.0/bin:$PATH
+ENV LOGSTASH_PATH="/usr/share/logstash"
+ENV GEM_PATH /usr/share/logstash/vendor/bundle/jruby/2.5.0
+ENV GEM_HOME /usr/share/logstash/vendor/bundle/jruby/2.5.0
+
+RUN gem install bundler:2.1.4
+
+COPY --chown=logstash:logstash ./cmd/logstash/ /home/logstash/
+WORKDIR /home/logstash/
+
+
+RUN bundle install --path=/usr/share/logstash/vendor/bundle && \
+    bundle exec rake vendor && \
+    bundle exec rspec
+
+RUN cat logstash-output-loki.gemspec | grep s.version | awk '{print $3}' |  cut -d "'" -f 2 > VERSION
+
+RUN gem build logstash-output-loki.gemspec && \
+    PLUGIN_VERSION=$(cat VERSION); /usr/share/logstash/bin/logstash-plugin install logstash-output-loki-${PLUGIN_VERSION}.gem
+
+EXPOSE 5044
diff --git a/cmd/logstash/Gemfile b/cmd/logstash/Gemfile
@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "./logstash"
+
+if Dir.exist?(logstash_path)
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+else
+  raise 'missing logstash vendoring'
+end
+
+gem "webmock", "~> 3.8"
diff --git a/cmd/logstash/README.md b/cmd/logstash/README.md
@@ -0,0 +1,74 @@
+# Contributing to Loki Logstash Output Plugin
+
+For information about hwo to use this plugin see this [documentation](../../docs/clients/logstash/README.md).
+
+## Install dependencies
+
+First you need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
+
+After setting up `rbenv`. Install JRuby
+
+```bash
+rbenv install jruby-9.2.10.0
+rbenv local jruby-9.2.10.0
+```
+
+Check that the environment is configured
+
+```bash
+ruby --version
+jruby 9.2.10
+```
+
+You should use make sure you are running jruby and not ruby. If the command below still shows ruby and not jruby, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
+
+```bash
+export PATH="$HOME/.rbenv/bin:$PATH"
+eval "$(rbenv init -)"
+```
+
+Then install bundler
+`gem install bundler:2.1.4`
+
+Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
+
+## Install dependencies and Build plugin
+
+### Install required packages
+
+```bash
+git clone git@github.com:elastic/logstash.git
+cd logstash
+git checkout tags/v7.6.2
+export LOGSTASH_PATH=`pwd`
+export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
+export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
+./gradlew assemble
+cd ..
+ruby -S bundle install
+ruby -S bundle exec rake vendor
+```
+
+### Build the plugin
+
+`gem build logstash-output-loki.gemspec`
+
+### Test
+
+`ruby -S bundle exec rspec`
+
+Alternatively if you don't want to install JRuby. Enter inside logstash-loki container.
+
+```bash
+docker build -t logstash-loki ./
+docker run -v  `pwd`/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
+bundle exec rspec
+```
+
+## Install plugin to local logstash
+
+`bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem`
+
+## Send sample event and check plugin is working
+
+`bin/logstash -f loki.conf`
diff --git a/cmd/logstash/Rakefile b/cmd/logstash/Rakefile
@@ -0,0 +1 @@
+require "logstash/devutils/rake"