Design observable metrics configuration syntax

[jfallows]

Zilla engine is designed to be observable at near-zero performance impact, so we can use that to provide low overhead metrics.

As with other first class concepts in zilla.yaml, such as vaults and guards, we need a general concept of metrics, independent of how they are stored locally or transmitted remotely, with more specific implementation for observability standards such as OpenTelemetry.

Metrics can fall into different categories, such as counter, gauge or histogram, which has implications on how metrics are stored in Zilla engine before being periodically exported.

[jfallows]

Proposal for enhancement to zilla.json for telemetry metrics.

{
  "name": "example",
  "telemetry":
  {
    "attributes": <-- default attributes for all bindings
    {
      "service.instance.id": "{{env.SERVICE_INSTANCE_ID}}"  <-- horizonally scaled instances remain unique
      "service.namespace": "example"
    },
    "metrics":     <-- semantics and storage requirements implicitly contributed metrics-stream, metrics-http, etc
    [
      "http.duration",
      "http.request.size",
      "http.response.size",
      "http.active_requests",
      "stream.initial.opens",
      "stream.initial.closes",
      "stream.initial.errors",
      "stream.initial.bytes",
      "stream.reply.opens",
      "stream.reply.closes",
      "stream.reply.errors",
      "stream.reply.bytes"
    ],
    "exporters":
    {
      "otlp0":
      {
        "type": "otlp",
        "endpoint": "otlp://myserver.local:55690"
        "endpoint": "https://example.com:4318"  <-- /v1/traces, /v1/metrics, /v1/logs (json or protobuf, grpc)
        "vault": ...
      }
    }
  },
  "bindings":
  {
    "tcp0":
    {
      "type": "tcp",
      "kind": "server",
      "options":
      {
        "host": "localhost",
        "port": 8080
      },
      "telemetry": <-- default telemetry (exit or routes)
      {
        "metrics":
        [
          "stream.*" <-- pattern or explicit
        ],
        "exporters": [ "otlp0" ]
      },
      "exit": "http0"
    },
    "http0":
    {
       "type": "http",
       "kind": "server",
       "exit": "http-kafka0"
    },
    "http-kafka0":
    {
      "type": "http-kafka",
      "kind": "proxy",
      "telemetry":  <-- default telemetry (exit or routes)
      {
        "metrics":
        [
          "http.server.*" <-- pattern or explicit
        ],
        "attributes":
        {
          "service.name": "http-kafka0"
        },
         "exporters": [ "otlp0" ] <-- exporters not required, can use ./zilla metrics to observe locally
      },
      "routes":
      [
        {
          "when":
          [
            {
              "method": "GET",
              "path": "/tasks/{id}"
            }
          ],
          "telemetry": <-- route-specific telemetry (incremental)
          {
            "attributes":
            {
              "http.target": "/tasks/{id}"
            }
          },
          "exit": "kafka_cache_client0",
          "with":
          {
            "capability": "fetch",
            "topic": "task-snapshots",
            "filters":
            [
              {
                "key": "${params.id}"
              }
            ]
          }
        },
        {
          "when":
          [
            {
              "method": "PUT",
              "path": "/tasks/{id}"
            }
          ],
          "exit": "kafka_cache_client0",
          "with":
          {
            "capability": "produce",
            "topic": "task-commands",
            "key": "${params.id}",
            "reply-to": "task-replies",
            "overrides":
            {
              "zilla:domain-model": "RenameTaskCommand"
            }
          },
          "telemetry": <-- route-specific telemetry (incremental)
          {
            "attributes":
            {
              "http.target": "/tasks/{id}"
            }
          }
        }
      ]
    }
  }
}