operator: Add automatic stream sharding support

operator/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+- [11091](https://github.com/grafana/loki/pull/11091) **periklis**: Add automatic stream sharding support
 - [11022](https://github.com/grafana/loki/pull/11022) **JoaoBraveCoding**: Remove outdated BoltDB dashboards 
 - [10932](https://github.com/grafana/loki/pull/10932) **JoaoBraveCoding**: Adds new value v13 to schema
 - [11232](https://github.com/grafana/loki/pull/11232) **periklis**: Update dependencies and dev tools

operator/apis/loki/v1/lokistack_types.go

@@ -635,6 +635,14 @@ type QueryLimitSpec struct {
 
 // IngestionLimitSpec defines the limits applied at the ingestion path.
 type IngestionLimitSpec struct {
+	// DesiredRate defines the desired ingestion rate per second that LokiStack should
+	// target applying automatic stream sharding. Unit MB.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Desired Ingestion Rate (in MB)"
+	DesiredRate int32 `json:"desiredRate,omitempty"`
+
 	// IngestionRate defines the sample size per second. Units MB.
 	//
 	// +optional

operator/apis/loki/v1/v1.go

@@ -83,6 +83,8 @@ var (
 	ErrReplicationSpecConflict = errors.New("replicationSpec and replicationFactor (deprecated) cannot be used at the same time")
 	// ErrIPv6InstanceAddrTypeNotAllowed when the default InstanceAddrType is used with enableIPv6.
 	ErrIPv6InstanceAddrTypeNotAllowed = errors.New(`instanceAddrType "default" cannot be used with enableIPv6 at the same time`)
+	// ErrDesiredRateAndPerStreamRateNotAllowed when there's a conflict in the rate limiting configuration because both desiredRate and perStreamRateLimit are set.
+	ErrDesiredRateAndPerStreamRateNotAllowed = errors.New("fields desiredRate and perStreamRateLimit cannot be used together")
 
 	// ErrRuleMustMatchNamespace indicates that an expression used in an alerting or recording rule is missing
 	// matchers for a namespace.

operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -294,6 +294,12 @@ spec:
       - description: Global defines the limits applied globally across the cluster.
         displayName: Global Limits
         path: limits.global
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.global.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.
@@ -378,6 +384,12 @@ spec:
       - description: Tenants defines the limits applied per tenant.
         displayName: Limits per Tenant
         path: limits.tenants
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.tenants.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.

operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml

@@ -98,6 +98,12 @@ spec:
                         description: IngestionLimits defines the limits applied on
                           ingested log streams.
                         properties:
+                          desiredRate:
+                            description: DesiredRate defines the desired ingestion
+                              rate per second that LokiStack should target applying
+                              automatic stream sharding. Unit MB.
+                            format: int32
+                            type: integer
                           ingestionBurstSize:
                             description: IngestionBurstSize defines the local rate-limited
                               sample size per distributor replica. It should be set
@@ -223,6 +229,12 @@ spec:
                           description: IngestionLimits defines the limits applied
                             on ingested log streams.
                           properties:
+                            desiredRate:
+                              description: DesiredRate defines the desired ingestion
+                                rate per second that LokiStack should target applying
+                                automatic stream sharding. Unit MB.
+                              format: int32
+                              type: integer
                             ingestionBurstSize:
                               description: IngestionBurstSize defines the local rate-limited
                                 sample size per distributor replica. It should be

operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml

@@ -294,6 +294,12 @@ spec:
       - description: Global defines the limits applied globally across the cluster.
         displayName: Global Limits
         path: limits.global
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.global.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.
@@ -378,6 +384,12 @@ spec:
       - description: Tenants defines the limits applied per tenant.
         displayName: Limits per Tenant
         path: limits.tenants
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.tenants.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.

operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml

@@ -98,6 +98,12 @@ spec:
                         description: IngestionLimits defines the limits applied on
                           ingested log streams.
                         properties:
+                          desiredRate:
+                            description: DesiredRate defines the desired ingestion
+                              rate per second that LokiStack should target applying
+                              automatic stream sharding. Unit MB.
+                            format: int32
+                            type: integer
                           ingestionBurstSize:
                             description: IngestionBurstSize defines the local rate-limited
                               sample size per distributor replica. It should be set
@@ -223,6 +229,12 @@ spec:
                           description: IngestionLimits defines the limits applied
                             on ingested log streams.
                           properties:
+                            desiredRate:
+                              description: DesiredRate defines the desired ingestion
+                                rate per second that LokiStack should target applying
+                                automatic stream sharding. Unit MB.
+                              format: int32
+                              type: integer
                             ingestionBurstSize:
                               description: IngestionBurstSize defines the local rate-limited
                                 sample size per distributor replica. It should be

operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -307,6 +307,12 @@ spec:
       - description: Global defines the limits applied globally across the cluster.
         displayName: Global Limits
         path: limits.global
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.global.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.
@@ -391,6 +397,12 @@ spec:
       - description: Tenants defines the limits applied per tenant.
         displayName: Limits per Tenant
         path: limits.tenants
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.tenants.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.

operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml

@@ -98,6 +98,12 @@ spec:
                         description: IngestionLimits defines the limits applied on
                           ingested log streams.
                         properties:
+                          desiredRate:
+                            description: DesiredRate defines the desired ingestion
+                              rate per second that LokiStack should target applying
+                              automatic stream sharding. Unit MB.
+                            format: int32
+                            type: integer
                           ingestionBurstSize:
                             description: IngestionBurstSize defines the local rate-limited
                               sample size per distributor replica. It should be set
@@ -223,6 +229,12 @@ spec:
                           description: IngestionLimits defines the limits applied
                             on ingested log streams.
                           properties:
+                            desiredRate:
+                              description: DesiredRate defines the desired ingestion
+                                rate per second that LokiStack should target applying
+                                automatic stream sharding. Unit MB.
+                              format: int32
+                              type: integer
                             ingestionBurstSize:
                               description: IngestionBurstSize defines the local rate-limited
                                 sample size per distributor replica. It should be

operator/config/crd/bases/loki.grafana.com_lokistacks.yaml

@@ -80,6 +80,12 @@ spec:
                         description: IngestionLimits defines the limits applied on
                           ingested log streams.
                         properties:
+                          desiredRate:
+                            description: DesiredRate defines the desired ingestion
+                              rate per second that LokiStack should target applying
+                              automatic stream sharding. Unit MB.
+                            format: int32
+                            type: integer
                           ingestionBurstSize:
                             description: IngestionBurstSize defines the local rate-limited
                               sample size per distributor replica. It should be set
@@ -205,6 +211,12 @@ spec:
                           description: IngestionLimits defines the limits applied
                             on ingested log streams.
                           properties:
+                            desiredRate:
+                              description: DesiredRate defines the desired ingestion
+                                rate per second that LokiStack should target applying
+                                automatic stream sharding. Unit MB.
+                              format: int32
+                              type: integer
                             ingestionBurstSize:
                               description: IngestionBurstSize defines the local rate-limited
                                 sample size per distributor replica. It should be

operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml

@@ -207,6 +207,12 @@ spec:
       - description: Global defines the limits applied globally across the cluster.
         displayName: Global Limits
         path: limits.global
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.global.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.
@@ -291,6 +297,12 @@ spec:
       - description: Tenants defines the limits applied per tenant.
         displayName: Limits per Tenant
         path: limits.tenants
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.tenants.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.

operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml

@@ -207,6 +207,12 @@ spec:
       - description: Global defines the limits applied globally across the cluster.
         displayName: Global Limits
         path: limits.global
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.global.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.
@@ -291,6 +297,12 @@ spec:
       - description: Tenants defines the limits applied per tenant.
         displayName: Limits per Tenant
         path: limits.tenants
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.tenants.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.

operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml

@@ -219,6 +219,12 @@ spec:
       - description: Global defines the limits applied globally across the cluster.
         displayName: Global Limits
         path: limits.global
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.global.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.
@@ -303,6 +309,12 @@ spec:
       - description: Tenants defines the limits applied per tenant.
         displayName: Limits per Tenant
         path: limits.tenants
+      - description: DesiredRate defines the desired ingestion rate per second that
+          LokiStack should target applying automatic stream sharding. Unit MB.
+        displayName: Desired Ingestion Rate (in MB)
+        path: limits.tenants.ingestion.desiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: IngestionBurstSize defines the local rate-limited sample size
           per distributor replica. It should be set to the set at least to the maximum
           logs size expected in a single push request.

operator/docs/operator/api.md

@@ -1082,6 +1082,19 @@ MemberListSpec
 <tbody>
 <tr>
 <td>
+<code>desiredRate</code><br/>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>DesiredRate defines the desired ingestion rate per second that LokiStack should
+target applying automatic stream sharding. Unit MB.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>ingestionRate</code><br/>
 <em>
 int32

operator/internal/manifests/internal/config/build.go

@@ -25,7 +25,9 @@ var (
 	//go:embed loki-runtime-config.yaml
 	lokiRuntimeConfigYAMLTmplFile embed.FS
 
-	lokiConfigYAMLTmpl = template.Must(template.ParseFS(lokiConfigYAMLTmplFile, "loki-config.yaml"))
+	lokiConfigYAMLTmpl = template.Must(template.New("loki-config.yaml").Funcs(template.FuncMap{
+		"toBytes": toBytes,
+	}).ParseFS(lokiConfigYAMLTmplFile, "loki-config.yaml"))
 
 	lokiRuntimeConfigYAMLTmpl = template.Must(template.ParseFS(lokiRuntimeConfigYAMLTmplFile, "loki-runtime-config.yaml"))
 )
@@ -54,3 +56,7 @@ func Build(opts Options) ([]byte, []byte, error) {
 	}
 	return cfg, rcfg, nil
 }
+
+func toBytes(s int32) int32 {
+	return int32(float32(s) * (1 << 20))
+}