Add gitlab pipeline for host injection

.gitlab-ci.yml

@@ -1,22 +1,74 @@
 stages:
+  - package
   - deploy
   - macrobenchmarks
   - microbenchmarks
 
-include: ".gitlab/benchmarks.yml"
+include:
+  - remote: https://gitlab-templates.ddbuild.io/apm/packaging.yml
+  - local: ".gitlab/benchmarks.yml"
 
 # -----------------------------------------------------
 # Reliability Environment configuration
 # -----------------------------------------------------
 
 variables:
+  RUBY_PACKAGE_VERSION:
+    description: "The version of the gem to build the rpm/deb package"
   DOWNSTREAM_BRANCH:
     value: "master"
     description: "Run a specific datadog-reliability-env branch downstream"
 
-.common: &common
+default:
   tags: [ "runner:main", "size:large" ]
 
+install-base-ruby-gems:
+  image: registry.ddbuild.io/images/mirror/ruby:3.2.2
+  stage: package
+  rules:
+    - if: $RUBY_PACKAGE_VERSION
+      when: on_success
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+      when: on_success
+  script:
+    - .gitlab/install-gem-to-folder.sh
+  artifacts:
+    paths:
+      - vendor
+
+install-version-dependant-gems:
+  image: registry.ddbuild.io/images/mirror/ruby:$RUBY_VERSION
+  parallel:
+    matrix:
+      - RUBY_VERSION: [ "3.0.0", "3.1.4"]
+  rules:
+    - if: $RUBY_PACKAGE_VERSION
+      when: on_success
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+      when: on_success
+  stage: package
+  needs: [ "install-base-ruby-gems" ]
+  script:
+    - .gitlab/install-gem-to-folder-version-specific.sh
+  artifacts:
+    paths:
+      - vendor
+
+package:
+  extends: .package
+  needs: ["install-version-dependant-gems", "install-base-ruby-gems"]
+  rules:
+    - if: $RUBY_PACKAGE_VERSION
+      when: on_success
+    - if: '$CI_COMMIT_TAG =~ /^v.*/'
+      when: on_success
+  script:
+    - ls ../vendor
+    - ../.gitlab/build-deb-rpm.sh
+
+.release-package:
+  stage: deploy
+
 deploy_to_reliability_env:
   stage: deploy
   rules:

.gitlab/build-deb-rpm.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ -n "$CI_COMMIT_TAG" ] && [ -z "$RUBY_PACKAGE_VERSION" ]; then
+  RUBY_PACKAGE_VERSION=${CI_COMMIT_TAG##v}
+fi
+
+source common_build_functions.sh
+
+# Not sure how it is called
+fpm_wrapper "datadog-apm-library-ruby" "$RUBY_PACKAGE_VERSION" \
+ --input-type dir \
+ --url "https://github.com/DataDog/dd-trace-rb" \
+ --description "Datadog APM client library for Ruby" \
+ --license "BSD-3-Clause" \
+ --chdir=../vendor \
+ --prefix "$LIBRARIES_INSTALL_BASE/ruby" \
+ .=.

.gitlab/install-gem-to-folder-version-specific.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cd vendor
+
+export INSTALL_DDTRACE_NATIVE_DEPS=true
+ruby install_ddtrace_deps.rb

.gitlab/install-gem-to-folder.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+if [ -n "$CI_COMMIT_TAG" ] && [ -z "$RUBY_PACKAGE_VERSION" ]; then
+  RUBY_PACKAGE_VERSION=${CI_COMMIT_TAG##v}
+fi
+
+mkdir -p vendor
+cat > Gemfile << EOF
+    source "https://rubygems.org"
+
+    gem 'ddtrace', "$RUBY_PACKAGE_VERSION"
+EOF
+
+bundle lock
+
+cp Gemfile vendor
+cp Gemfile.lock vendor
+cp .gitlab/install_ddtrace_deps.rb vendor
+cp lib-injection/host_inject.rb vendor
+
+export INSTALL_DDTRACE_NATIVE_DEPS=true
+export INSTALL_DDTRACE_NON_NATIVE_DEPS=true
+
+ruby vendor/install_ddtrace_deps.rb

.gitlab/install_ddtrace_deps.rb

@@ -0,0 +1,78 @@
+# require "fileutils"
+require "open3"
+require "bundler"
+
+lock_file_path = "./Gemfile.lock"
+install_dir = "./vendor"
+
+# lock_file_path = "/opt/datadog/apm/library/ruby/Gemfile.lock"
+# install_dir = "./vendor"
+
+lock_file_parser = Bundler::LockfileParser.new(Bundler.read_file(lock_file_path))
+
+gem_version_mapping = lock_file_parser.specs.each_with_object({}) do |spec, hash|
+  hash[spec.name] = spec.version.to_s
+  hash
+end
+
+def install_ddtrace_deps_without_native_extensions(gem_version_mapping: ,location: )
+  [
+    "debase-ruby_core_source",
+    "libdatadog",
+    "libddwaf"
+  ].each do |gem|
+    gem_install_cmd = "gem install #{gem} "\
+    "--version #{gem_version_mapping[gem]} "\
+    "--install-dir #{location} "\
+    "--no-document "\
+    "--ignore-dependencies "
+
+    STDOUT.puts "Execute: #{gem_install_cmd}"
+    output, status = Open3.capture2e(gem_install_cmd)
+    STDOUT.puts output
+
+    if status.success?
+      next
+    else
+      break
+    end
+  end
+end
+
+def install_ddtrace_deps_with_native_extensions(gem_version_mapping: ,location: )
+  [
+    "msgpack",
+    "ffi",
+    "ddtrace"
+  ].each do |gem|
+    gem_install_cmd = "gem install #{gem} "\
+      "--version #{gem_version_mapping[gem]} "\
+      "--install-dir #{location} "\
+      "--no-document "\
+      "--ignore-dependencies "
+
+    STDOUT.puts "Execute: #{gem_install_cmd}"
+    output, status = Open3.capture2e(gem_install_cmd)
+    STDOUT.puts output
+
+    if status.success?
+      next
+    else
+      break
+    end
+  end
+end
+
+if ENV["INSTALL_DDTRACE_NON_NATIVE_DEPS"] == 'true'
+  install_ddtrace_deps_without_native_extensions(
+    gem_version_mapping: gem_version_mapping,
+    location: install_dir
+  )
+end
+
+if ENV["INSTALL_DDTRACE_NATIVE_DEPS"] == 'true'
+  install_ddtrace_deps_with_native_extensions(
+    gem_version_mapping: gem_version_mapping,
+    location: "#{install_dir}/#{RUBY_VERSION.split(".")[0..1].join(".")}"
+  )
+end

lib-injection/host_inject.rb

@@ -0,0 +1,97 @@
+return if ENV["DD_TRACE_SKIP_LIB_INJECTION"] == "true"
+
+Gem.paths = {
+  "GEM_PATH" => "/opt/datadog/apm/library/ruby/#{RUBY_VERSION.split(".")[0..1].join(".")}:/opt/datadog/apm/library/ruby:#{ENV["GEM_PATH"]}"
+}
+
+ENV["GEM_PATH"] = Gem.path.join(":")
+
+begin
+  require "open3"
+  require "bundler"
+  require "shellwords"
+  require "fileutils"
+
+  failure_prefix = "Datadog lib injection failed:"
+  support_message = "For help solving this issue, please contact Datadog support at https://docs.datadoghq.com/help/."
+
+  unless Bundler::SharedHelpers.in_bundle?
+    STDOUT.puts "[ddtrace] Not in bundle... skipping host injection" if ENV["DD_TRACE_DEBUG"] == "true"
+    return
+  end
+
+  if Bundler.frozen_bundle?
+    warn "[ddtrace] #{failure_prefix} Cannot inject with frozen Gemfile, run `bundle config unset deployment` to allow lib injection. To learn more about bundler deployment, check https://bundler.io/guides/deploying.html#deploying-your-application. #{support_message}"
+    return
+  end
+
+  _, status = Open3.capture2e({"DD_TRACE_SKIP_LIB_INJECTION" => "true"}, "bundle show ddtrace")
+
+  if status.success?
+    STDOUT.puts "[ddtrace] ddtrace already installed... skipping host injection" if ENV["DD_TRACE_DEBUG"] == "true"
+    return
+  end
+
+  # lock_file_parser = Bundler::LockfileParser.new(Bundler.read_file(Bundler.default_lockfile))
+  lock_file_parser = Bundler::LockfileParser.new(Bundler.read_file("/opt/datadog/apm/library/ruby/Gemfile.lock"))
+  gem_version_mapping = lock_file_parser.specs.each_with_object({}) do |spec, hash|
+    hash[spec.name] = spec.version.to_s
+    hash
+  end
+
+  [
+    "msgpack",
+    "ffi",
+    "debase-ruby_core_source",
+    "libdatadog",
+    "libddwaf",
+    "ddtrace"
+  ].each do |g|
+    _, status = Open3.capture2e({"DD_TRACE_SKIP_LIB_INJECTION" => "true"}, "bundle show #{g}")
+    if status.success?
+      STDOUT.puts "[ddtrace] #{g} already installed... skipping..." if ENV["DD_TRACE_DEBUG"] == "true"
+      next
+    else
+      bundle_add_cmd = "bundle add #{g} --skip-install --version #{gem_version_mapping[g]} "
+
+      if g == "ddtrace"
+        bundle_add_cmd << "--require ddtrace/auto_instrument"
+      end
+
+      STDOUT.puts "[ddtrace] Performing lib injection with `#{bundle_add_cmd}`" if ENV["DD_TRACE_DEBUG"] == "true"
+
+      gemfile = Bundler::SharedHelpers.default_gemfile
+      lockfile = Bundler::SharedHelpers.default_lockfile
+
+      datadog_gemfile = gemfile.dirname + "datadog-Gemfile"
+      datadog_lockfile = lockfile.dirname + "datadog-Gemfile.lock"
+
+      begin
+        # Copies for trial
+        FileUtils.cp gemfile, datadog_gemfile
+        FileUtils.cp lockfile, datadog_lockfile
+
+        output, status = Open3.capture2e(
+          {"DD_TRACE_SKIP_LIB_INJECTION" => "true", "BUNDLE_GEMFILE" => datadog_gemfile.to_s},
+          bundle_add_cmd
+        )
+
+        if status.success?
+          STDOUT.puts "[ddtrace] Datadog lib injection successfully added #{g} to the application."
+
+          FileUtils.cp datadog_gemfile, gemfile
+          FileUtils.cp datadog_lockfile, lockfile
+        else
+          warn "[ddtrace] #{failure_prefix} Unable to add ddtrace. Error output:\n#{output.split("\n").map { |l| "[ddtrace] #{l}" }.join("\n")}\n#{support_message}"
+        end
+      ensure
+        # Remove the copies
+        FileUtils.rm datadog_gemfile
+        FileUtils.rm datadog_lockfile
+      end
+
+    end
+  end
+rescue Exception => e
+  warn "[ddtrace] #{failure_prefix} #{e.class.name} #{e.message}\nBacktrace: #{e.backtrace.join("\n")}\n#{support_message}"
+end