Add metadata storage and indexer

changelog/unreleased/metadata-storage.md

@@ -0,0 +1,5 @@
+Enhancement: Add metadata storage layer and indexer
+
+We ported over and enhanced the metadata storage layer and indexer from ocis-pkg so that it can be used by reva services as well.
+
+https://github.com/cs3org/reva/pull/2585

go.mod

@@ -4,6 +4,7 @@ require (
 	bou.ke/monkey v1.0.2
 	contrib.go.opencensus.io/exporter/prometheus v0.4.0
 	github.com/BurntSushi/toml v1.0.0
+	github.com/CiscoM31/godata v1.0.5 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible
@@ -41,6 +42,7 @@ require (
 	github.com/hashicorp/go-hclog v1.1.0
 	github.com/hashicorp/go-plugin v1.4.3
 	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/iancoleman/strcase v0.2.0 // indirect
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/juliangruber/go-intersect v1.1.0
 	github.com/mattn/go-sqlite3 v1.14.10

go.sum

@@ -82,6 +82,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/CiscoM31/godata v1.0.5 h1:AITXpa/5ybXEq59A0nqUGiS7ZXVJnQtFw5o09tyN/UA=
+github.com/CiscoM31/godata v1.0.5/go.mod h1:wcmFm66GMdOE316TgwFO1wo5ainCvTK26omd93oZf2M=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
@@ -597,6 +599,8 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
+github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhKWFeDesPjMj+wCHReeknARU3wqlyN4=

pkg/storage/utils/indexer/errors/errors.go

@@ -0,0 +1,57 @@
+// Copyright 2018-2022 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package errors
+
+import (
+	"fmt"
+
+	"github.com/cs3org/reva/pkg/storage/utils/indexer/option"
+)
+
+// AlreadyExistsErr implements the Error interface.
+type AlreadyExistsErr struct {
+	TypeName, Value string
+	IndexBy         option.IndexBy
+}
+
+func (e *AlreadyExistsErr) Error() string {
+	return fmt.Sprintf("%s with %s=%s does already exist", e.TypeName, e.IndexBy.String(), e.Value)
+}
+
+// IsAlreadyExistsErr checks whether an error is of type AlreadyExistsErr.
+func IsAlreadyExistsErr(e error) bool {
+	_, ok := e.(*AlreadyExistsErr)
+	return ok
+}
+
+// NotFoundErr implements the Error interface.
+type NotFoundErr struct {
+	TypeName, Value string
+	IndexBy         option.IndexBy
+}
+
+func (e *NotFoundErr) Error() string {
+	return fmt.Sprintf("%s with %s=%s not found", e.TypeName, e.IndexBy.String(), e.Value)
+}
+
+// IsNotFoundErr checks whether an error is of type IsNotFoundErr.
+func IsNotFoundErr(e error) bool {
+	_, ok := e.(*NotFoundErr)
+	return ok
+}

pkg/storage/utils/indexer/helper.go

@@ -0,0 +1,32 @@
+// Copyright 2018-2022 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package indexer
+
+// dedup removes duplicate values in given slice
+func dedup(s []string) []string {
+	var out []string
+	exists := make(map[string]bool)
+	for _, ss := range s {
+		if _, ok := exists[ss]; !ok {
+			out = append(out, ss)
+			exists[ss] = true
+		}
+	}
+	return out
+}

pkg/storage/utils/indexer/helper_test.go

@@ -0,0 +1,55 @@
+// Copyright 2018-2022 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package indexer
+
+import (
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gmeasure"
+)
+
+var _ = Describe("Helper", func() {
+	Describe("dedup", func() {
+		It("dedups reasonably fast", func() {
+			experiment := gmeasure.NewExperiment("deduplicating string slices")
+			AddReportEntry(experiment.Name, experiment)
+
+			experiment.Sample(func(idx int) {
+				slice := []string{}
+				for i := 0; i < 900; i++ {
+					slice = append(slice, strconv.Itoa(i))
+				}
+				for i := 0; i < 100; i++ {
+					slice = append(slice, strconv.Itoa(i))
+				}
+				experiment.MeasureDuration("repagination", func() {
+					dedupped := dedup(slice)
+					Expect(len(dedupped)).To(Equal(900))
+				})
+			}, gmeasure.SamplingConfig{N: 100000, Duration: 10 * time.Second})
+
+			repaginationStats := experiment.GetStats("repagination")
+			medianDuration := repaginationStats.DurationFor(gmeasure.StatMedian)
+			Expect(medianDuration).To(BeNumerically("<", 1200*time.Microsecond))
+		})
+	})
+})