operator: Add automatic stream sharding support

operator/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+- [11091](https://github.com/grafana/loki/pull/11091) **periklis**: Add automatic stream sharding support
 - [11022](https://github.com/grafana/loki/pull/11022) **JoaoBraveCoding**: Remove outdated BoltDB dashboards 
 - [10932](https://github.com/grafana/loki/pull/10932) **JoaoBraveCoding**: Adds new value v13 to schema
 - [11232](https://github.com/grafana/loki/pull/11232) **periklis**: Update dependencies and dev tools

operator/apis/loki/v1/lokistack_types.go

@@ -690,6 +690,14 @@ type IngestionLimitSpec struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Max Line Size"
 	MaxLineSize int32 `json:"maxLineSize,omitempty"`
 
+	// PerStreamDesiredRate defines the desired ingestion rate per second that LokiStack should
+	// target applying automatic stream sharding. Units MB.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Per Stream Desired Rate (in MB)"
+	PerStreamDesiredRate int32 `json:"perStreamDesiredRate,omitempty"`
+
 	// PerStreamRateLimit defines the maximum byte rate per second per stream. Units MB.
 	//
 	// +optional

operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.5.0
-    createdAt: "2023-11-03T11:44:16Z"
+    createdAt: "2023-11-23T11:25:33Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     operators.operatorframework.io/builder: operator-sdk-unknown
@@ -336,6 +336,12 @@ spec:
         path: limits.global.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.global.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)
@@ -420,6 +426,12 @@ spec:
         path: limits.tenants.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.tenants.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)

operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml

@@ -135,6 +135,12 @@ spec:
                               on ingestion path. Units in Bytes.
                             format: int32
                             type: integer
+                          perStreamDesiredRate:
+                            description: PerStreamDesiredRate defines the desired
+                              ingestion rate per second that LokiStack should target
+                              applying automatic stream sharding. Units MB.
+                            format: int32
+                            type: integer
                           perStreamRateLimit:
                             description: PerStreamRateLimit defines the maximum byte
                               rate per second per stream. Units MB.
@@ -262,6 +268,12 @@ spec:
                                 on ingestion path. Units in Bytes.
                               format: int32
                               type: integer
+                            perStreamDesiredRate:
+                              description: PerStreamDesiredRate defines the desired
+                                ingestion rate per second that LokiStack should target
+                                applying automatic stream sharding. Units MB.
+                              format: int32
+                              type: integer
                             perStreamRateLimit:
                               description: PerStreamRateLimit defines the maximum
                                 byte rate per second per stream. Units MB.

operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml

@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.5.0
-    createdAt: "2023-11-03T11:44:14Z"
+    createdAt: "2023-11-23T11:25:30Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     operators.operatorframework.io/builder: operator-sdk-unknown
@@ -336,6 +336,12 @@ spec:
         path: limits.global.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.global.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)
@@ -420,6 +426,12 @@ spec:
         path: limits.tenants.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.tenants.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)

operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml

@@ -135,6 +135,12 @@ spec:
                               on ingestion path. Units in Bytes.
                             format: int32
                             type: integer
+                          perStreamDesiredRate:
+                            description: PerStreamDesiredRate defines the desired
+                              ingestion rate per second that LokiStack should target
+                              applying automatic stream sharding. Units MB.
+                            format: int32
+                            type: integer
                           perStreamRateLimit:
                             description: PerStreamRateLimit defines the maximum byte
                               rate per second per stream. Units MB.
@@ -262,6 +268,12 @@ spec:
                                 on ingestion path. Units in Bytes.
                               format: int32
                               type: integer
+                            perStreamDesiredRate:
+                              description: PerStreamDesiredRate defines the desired
+                                ingestion rate per second that LokiStack should target
+                                applying automatic stream sharding. Units MB.
+                              format: int32
+                              type: integer
                             perStreamRateLimit:
                               description: PerStreamRateLimit defines the maximum
                                 byte rate per second per stream. Units MB.

operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: quay.io/openshift-logging/loki-operator:0.1.0
-    createdAt: "2023-11-03T11:44:18Z"
+    createdAt: "2023-11-23T11:25:35Z"
     description: |
       The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging.
       ## Prerequisites and Requirements
@@ -349,6 +349,12 @@ spec:
         path: limits.global.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.global.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)
@@ -433,6 +439,12 @@ spec:
         path: limits.tenants.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.tenants.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)

operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml

@@ -135,6 +135,12 @@ spec:
                               on ingestion path. Units in Bytes.
                             format: int32
                             type: integer
+                          perStreamDesiredRate:
+                            description: PerStreamDesiredRate defines the desired
+                              ingestion rate per second that LokiStack should target
+                              applying automatic stream sharding. Units MB.
+                            format: int32
+                            type: integer
                           perStreamRateLimit:
                             description: PerStreamRateLimit defines the maximum byte
                               rate per second per stream. Units MB.
@@ -262,6 +268,12 @@ spec:
                                 on ingestion path. Units in Bytes.
                               format: int32
                               type: integer
+                            perStreamDesiredRate:
+                              description: PerStreamDesiredRate defines the desired
+                                ingestion rate per second that LokiStack should target
+                                applying automatic stream sharding. Units MB.
+                              format: int32
+                              type: integer
                             perStreamRateLimit:
                               description: PerStreamRateLimit defines the maximum
                                 byte rate per second per stream. Units MB.

operator/config/crd/bases/loki.grafana.com_lokistacks.yaml

@@ -117,6 +117,12 @@ spec:
                               on ingestion path. Units in Bytes.
                             format: int32
                             type: integer
+                          perStreamDesiredRate:
+                            description: PerStreamDesiredRate defines the desired
+                              ingestion rate per second that LokiStack should target
+                              applying automatic stream sharding. Units MB.
+                            format: int32
+                            type: integer
                           perStreamRateLimit:
                             description: PerStreamRateLimit defines the maximum byte
                               rate per second per stream. Units MB.
@@ -244,6 +250,12 @@ spec:
                                 on ingestion path. Units in Bytes.
                               format: int32
                               type: integer
+                            perStreamDesiredRate:
+                              description: PerStreamDesiredRate defines the desired
+                                ingestion rate per second that LokiStack should target
+                                applying automatic stream sharding. Units MB.
+                              format: int32
+                              type: integer
                             perStreamRateLimit:
                               description: PerStreamRateLimit defines the maximum
                                 byte rate per second per stream. Units MB.

operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml

@@ -249,6 +249,12 @@ spec:
         path: limits.global.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.global.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)
@@ -333,6 +339,12 @@ spec:
         path: limits.tenants.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.tenants.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)

operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml

@@ -249,6 +249,12 @@ spec:
         path: limits.global.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.global.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)
@@ -333,6 +339,12 @@ spec:
         path: limits.tenants.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.tenants.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)

operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml

@@ -261,6 +261,12 @@ spec:
         path: limits.global.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.global.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)
@@ -345,6 +351,12 @@ spec:
         path: limits.tenants.ingestion.maxLineSize
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
+      - description: PerStreamDesiredRate defines the desired ingestion rate per second
+          that LokiStack should target applying automatic stream sharding. Units MB.
+        displayName: Per Stream Desired Rate (in MB)
+        path: limits.tenants.ingestion.perStreamDesiredRate
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
       - description: PerStreamRateLimit defines the maximum byte rate per second per
           stream. Units MB.
         displayName: Maximum byte rate per second per stream (in MB)

operator/docs/operator/api.md

@@ -1172,6 +1172,19 @@ int32
 </tr>
 <tr>
 <td>
+<code>perStreamDesiredRate</code><br/>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>PerStreamDesiredRate defines the desired ingestion rate per second that LokiStack should
+target applying automatic stream sharding. Units MB.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>perStreamRateLimit</code><br/>
 <em>
 int32