Merge branch 'main' into fix-3575

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -72,6 +72,14 @@ body:
         be automatically formatted into code, so no need for backticks. Or paste
         gists, pastebins links here
       render: Shell
+  - type: textarea
+    id: stuck_node
+    attributes:
+      label: Is the node "stuck"? Has it stopped syncing?
+      description: Please share the state of the node e.g. what height is stuck at, what is the state of DAS?
+      placeholder: >
+        Please consult our RPC docs https://node-rpc-docs.celestia.org/ and share output from any or all of
+        Das.SamplingStats, Header.SyncState and Header.NetworkHead
   - type: textarea
     id: misc
     attributes:

.github/workflows/go-ci.yml

@@ -41,7 +41,7 @@ jobs:
           go-version: ${{ inputs.go-version }}
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6.0.1
+        uses: golangci/golangci-lint-action@v6.1.0
         with:
           args: --timeout 10m
           version: v1.59

.github/workflows/labels.yml

@@ -17,3 +17,33 @@ jobs:
           mode: minimum
           count: 1
           labels: "kind:fix, kind:misc, kind:break!, kind:refactor, kind:feat, kind:deps, kind:docs, kind:ci, kind:chore, kind:testing" # yamllint disable-line rule:line-length
+
+  # will attempt to apply a breaking label
+  # on opening the PR but not enforce it on repeated changes
+  # so we don't get trapped by false positives (for now)
+  # we can expand to all cases after
+  apply-breaking:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'opened' && github.actor != 'dependabot[bot]' }}
+    permissions:
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run check for breaking
+        id: breaking_change
+        run: |
+          git fetch origin main
+          make detect-breaking
+
+      - name: Add label if breaking changes detected
+        if: failure()
+        run: gh issue edit "$NUMBER" --add-label "$LABELS"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          NUMBER: ${{ github.event.pull_request.number }}
+          LABELS: kind:break!

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=$BUILDPLATFORM docker.io/golang:1.22.4-alpine3.20 as builder
+FROM --platform=$BUILDPLATFORM docker.io/golang:1.22.5-alpine3.20 as builder
 
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
@@ -25,7 +25,7 @@ RUN uname -a &&\
     CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
     make build && make cel-key
 
-FROM docker.io/alpine:3.20.1
+FROM docker.io/alpine:3.20.2
 
 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000
 ARG UID=10001

Makefile

@@ -39,7 +39,7 @@ build:
 	@go build -o build/ ${LDFLAGS} ./cmd/celestia
 .PHONY: build
 
-## build-jemalloc: Build celestia-node binary with jemalloc allocator for BadgerDB instead of Go's native one
+## build-jemalloc: Build celestia-node binary with jemalloc allocator for BadgerDB.
 build-jemalloc: jemalloc
 	@echo "--> Building Celestia with jemalloc"
 	@go build -o build/ ${LDFLAGS} -tags jemalloc ./cmd/celestia
@@ -51,14 +51,14 @@ clean:
 	@rm -rf build/*
 .PHONY: clean
 
-## cover: generate to code coverage report.
+## cover: Generate code coverage report.
 cover:
 	@echo "--> Generating Code Coverage"
 	@go install github.com/ory/go-acc@latest
 	@go-acc -o coverage.txt `go list ./... | grep -v nodebuilder/tests` -- -v
 .PHONY: cover
 
-## deps: install dependencies.
+## deps: Install dependencies.
 deps:
 	@echo "--> Installing Dependencies"
 	@go mod download
@@ -73,7 +73,7 @@ else
 endif
 .PHONY: install
 
-## install-global: Install the celestia-node binary (only for systems that support GNU coreutils, i.e. Linux).
+## install-global: Install the celestia-node binary (only for systems that support GNU coreutils like Linux).
 install-global:
 	@echo "--> Installing Celestia"
 	@install -v ./build/* -t ${PREFIX}/bin
@@ -109,7 +109,8 @@ install-key:
 	@go install ./cmd/cel-key
 .PHONY: install-key
 
-## fmt: Formats only *.go (excluding *.pb.go *pb_test.go). Runs `gofmt & goimports` internally.
+## fmt: Formats only *.go (excluding *.pb.go *pb_test.go).
+# Runs `gofmt & goimports` internally.
 fmt: sort-imports
 	@find . -name '*.go' -type f -not -path "*.git*" -not -name '*.pb.go' -not -name '*pb_test.go' | xargs gofmt -w -s
 	@go mod tidy -compat=1.20
@@ -118,39 +119,40 @@ fmt: sort-imports
 	@markdownlint --fix --quiet --config .markdownlint.yaml .
 .PHONY: fmt
 
-## lint: Linting *.go files using golangci-lint. Look for .golangci.yml for the list of linters. Also lint *.md files using markdownlint.
+## lint: Lint *.go files with golangci-lint and *.md files with markdownlint.
+# Look at .golangci.yml for the list of linters.
 lint: lint-imports
 	@echo "--> Running linter"
 	@golangci-lint run
 	@markdownlint --config .markdownlint.yaml '**/*.md'
 	@cfmt --m=120 ./...
 .PHONY: lint
 
-## test-unit: Running unit tests
+## test-unit: Run unit tests.
 test-unit:
 	@echo "--> Running unit tests"
 	@go test $(VERBOSE) -covermode=atomic -coverprofile=coverage.txt `go list ./... | grep -v nodebuilder/tests` $(LOG_AND_FILTER)
 .PHONY: test-unit
 
-## test-unit-race: Running unit tests with data race detector
+## test-unit-race: Run unit tests with data race detector.
 test-unit-race:
 	@echo "--> Running unit tests with data race detector"
 	@go test $(VERBOSE) -race -covermode=atomic -coverprofile=coverage.txt `go list ./... | grep -v nodebuilder/tests` $(LOG_AND_FILTER)
 .PHONY: test-unit-race
 
-## test-integration: Running /integration tests located in nodebuilder/tests
+## test-integration: Run integration tests located in nodebuilder/tests.
 test-integration:
 	@echo "--> Running integrations tests $(VERBOSE) -tags=$(TAGS) $(INTEGRATION_RUN_LENGTH)"
 	@go test $(VERBOSE) -tags=$(TAGS) $(INTEGRATION_RUN_LENGTH) ./nodebuilder/tests
 .PHONY: test-integration
 
-## test-integration-race: Running integration tests with data race detector located in node/tests
+## test-integration-race: Run integration tests with data race detector located in nodebuilder/tests.
 test-integration-race:
 	@echo "--> Running integration tests with data race detector -tags=$(TAGS)"
 	@go test -race -tags=$(TAGS) ./nodebuilder/tests
 .PHONY: test-integration-race
 
-## benchmark: Running all benchmarks
+## benchmark: Run all benchmarks.
 benchmark:
 	@echo "--> Running benchmarks"
 	@go test -run="none" -bench=. -benchtime=100x -benchmem ./...
@@ -173,14 +175,14 @@ pb-gen:
 	done;
 .PHONY: pb-gen
 
-## openrpc-gen: Generate OpenRPC spec for Celestia-Node's RPC api
+## openrpc-gen: Generate OpenRPC spec for celestia-node's RPC API.
 openrpc-gen:
 	@go run ${LDFLAGS} ./cmd/celestia docgen
 .PHONY: openrpc-gen
 
 ## lint-imports: Lint only Go imports.
-## flag -set-exit-status doesn't exit with code 1 as it should, so we use find until it is fixed by goimports-reviser
 lint-imports:
+# flag -set-exit-status doesn't exit with code 1 as it should, so we use find until it is fixed by goimports-reviser
 	@echo "--> Running imports linter"
 	@for file in `find . -type f -name '*.go'`; \
 		do goimports-reviser -list-diff -set-exit-status -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/celestia-node" -output stdout $$file \
@@ -199,13 +201,13 @@ adr-gen:
 	@curl -sSL https://raw.githubusercontent.com/celestiaorg/.github/main/adr-template.md > docs/architecture/adr-$(NUM)-$(TITLE).md
 .PHONY: adr-gen
 
-## telemetry-infra-up: launches local telemetry infrastructure. This includes grafana, jaeger, loki, pyroscope, and an otel-collector.
-## you can access the grafana instance at localhost:3000 and login with admin:admin.
+## telemetry-infra-up: Launch local telemetry infra (grafana, jaeger, loki, pyroscope, and otel-collector).
+# You can access the grafana instance at localhost:3000 and login with admin:admin.
 telemetry-infra-up:
 	PWD="${DIR_FULLPATH}/docker/telemetry" docker-compose -f ./docker/telemetry/docker-compose.yml up
 .PHONY: telemetry-infra-up
 
-## telemetry-infra-down: tears the telemetry infrastructure down. The stores for grafana, prometheus, and loki will persist.
+## telemetry-infra-down: Tears the telemetry infra down. Persists the stores for grafana, prometheus, and loki.
 telemetry-infra-down:
 	PWD="${DIR_FULLPATH}/docker/telemetry" docker-compose -f ./docker/telemetry/docker-compose.yml down
 .PHONY: telemetry-infra-down
@@ -222,17 +224,44 @@ goreleaser-build:
 	goreleaser build --snapshot --clean --single-target
 .PHONY: goreleaser-build
 
-## goreleaser-release: Builds the release celestia binaries as defined in .goreleaser.yaml. This requires there be a git tag for the release in the local git history.
+## goreleaser-release: Builds the release celestia binaries as defined in .goreleaser.yaml.
+# This requires there be a git tag for the release in the local git history.
 goreleaser-release:
 	goreleaser release --clean --fail-fast --skip-publish
 .PHONY: goreleaser-release
 
+# detect changed files and parse output
+# to inspect changes to nodebuilder/**/config.go fields
+CHANGED_FILES      = $(shell git diff --name-only origin/main...HEAD)
+detect-breaking:
+	@BREAK=false
+	@for file in ${CHANGED_FILES}; do \
+		if echo $$file | grep -qE '\.proto$$'; then \
+			BREAK=true; \
+		fi; \
+		if echo $$file | grep -qE 'nodebuilder/.*/config\.go'; then \
+			DIFF_OUTPUT=$$(git diff origin/main...HEAD $$file); \
+			if echo "$$DIFF_OUTPUT" | grep -qE 'type Config struct|^\s+\w+\s+Config'; then \
+				BREAK=true; \
+			fi; \
+		fi; \
+	done; \
+	if [ "$$BREAK" = true ]; then \
+		echo "break detected"; \
+		exit 1; \
+	else \
+		echo "no break detected"; \
+		exit 0; \
+	fi
+.PHONY: detect-breaking
+
+
 # Copied from https://github.com/dgraph-io/badger/blob/main/Makefile
 USER_ID      = $(shell id -u)
 HAS_JEMALLOC = $(shell test -f /usr/local/lib/libjemalloc.a && echo "jemalloc")
 JEMALLOC_URL = "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
 
-## jemalloc installs jemalloc allocator
+## jemalloc: Install jemalloc allocator.
 jemalloc:
 	@if [ -z "$(HAS_JEMALLOC)" ] ; then \
 		mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ; \

api/gateway/share.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/gorilla/mux"
 
-	"github.com/celestiaorg/celestia-app/pkg/shares"
+	"github.com/celestiaorg/go-square/shares"
 
 	"github.com/celestiaorg/celestia-node/share"
 )

api/gateway/share_test.go

@@ -5,10 +5,10 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	coretypes "github.com/tendermint/tendermint/types"
 
-	"github.com/celestiaorg/celestia-app/pkg/appconsts"
-	"github.com/celestiaorg/celestia-app/pkg/shares"
+	"github.com/celestiaorg/celestia-app/v2/pkg/appconsts"
+	"github.com/celestiaorg/go-square/blob"
+	"github.com/celestiaorg/go-square/shares"
 
 	"github.com/celestiaorg/celestia-node/share/sharetest"
 )
@@ -23,13 +23,13 @@ func Test_dataFromShares(t *testing.T) {
 	ns := sharetest.RandV0Namespace()
 	sss := shares.NewSparseShareSplitter()
 	for _, data := range testData {
-		b := coretypes.Blob{
+		b := blob.Blob{
 			Data:             data,
-			NamespaceID:      ns.ID(),
-			NamespaceVersion: ns.Version(),
-			ShareVersion:     appconsts.ShareVersionZero,
+			NamespaceId:      ns.ID(),
+			NamespaceVersion: uint32(ns.Version()),
+			ShareVersion:     uint32(appconsts.ShareVersionZero),
 		}
-		err := sss.Write(b)
+		err := sss.Write(&b)
 		require.NoError(t, err)
 	}
 

api/rpc/client/client.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/celestiaorg/celestia-node/api/rpc/perms"
 	"github.com/celestiaorg/celestia-node/nodebuilder/blob"
+	"github.com/celestiaorg/celestia-node/nodebuilder/blobstream"
 	"github.com/celestiaorg/celestia-node/nodebuilder/da"
 	"github.com/celestiaorg/celestia-node/nodebuilder/das"
 	"github.com/celestiaorg/celestia-node/nodebuilder/fraud"
@@ -27,15 +28,16 @@ var (
 )
 
 type Client struct {
-	Fraud  fraud.API
-	Header header.API
-	State  state.API
-	Share  share.API
-	DAS    das.API
-	P2P    p2p.API
-	Node   node.API
-	Blob   blob.API
-	DA     da.API
+	Fraud      fraud.API
+	Header     header.API
+	State      state.API
+	Share      share.API
+	DAS        das.API
+	P2P        p2p.API
+	Node       node.API
+	Blob       blob.API
+	DA         da.API
+	Blobstream blobstream.API
 
 	closer multiClientCloser
 }
@@ -116,14 +118,15 @@ func newClient(ctx context.Context, addr string, authHeader http.Header, config
 func moduleMap(client *Client) map[string]interface{} {
 	// TODO: this duplication of strings many times across the codebase can be avoided with issue #1176
 	return map[string]interface{}{
-		"share":  &client.Share.Internal,
-		"state":  &client.State.Internal,
-		"header": &client.Header.Internal,
-		"fraud":  &client.Fraud.Internal,
-		"das":    &client.DAS.Internal,
-		"p2p":    &client.P2P.Internal,
-		"node":   &client.Node.Internal,
-		"blob":   &client.Blob.Internal,
-		"da":     &client.DA.Internal,
+		"share":      &client.Share.Internal,
+		"state":      &client.State.Internal,
+		"header":     &client.Header.Internal,
+		"fraud":      &client.Fraud.Internal,
+		"das":        &client.DAS.Internal,
+		"p2p":        &client.P2P.Internal,
+		"node":       &client.Node.Internal,
+		"blob":       &client.Blob.Internal,
+		"da":         &client.DA.Internal,
+		"blobstream": &client.Blobstream.Internal,
 	}
 }