diff --git a/go.mod b/go.mod index 4f31ede3..eecce7d9 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,11 @@ require ( emperror.dev/errors v0.8.1 github.com/Masterminds/sprig/v3 v3.2.3 github.com/aws/aws-sdk-go v1.44.300 + github.com/aws/aws-sdk-go-v2 v1.19.0 + github.com/aws/aws-sdk-go-v2/config v1.18.28 + github.com/aws/aws-sdk-go-v2/credentials v1.13.27 + github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 + github.com/aws/smithy-go v1.13.5 github.com/coreos/go-oidc/v3 v3.6.0 github.com/dimiro1/health v0.0.0-20191019130555-c5cbb4d46ffc github.com/dustin/go-humanize v1.0.1 @@ -36,6 +41,19 @@ require ( github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 9695a5a1..77051ec3 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,42 @@ github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3 github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.44.300 h1:Zn+3lqgYahIf9yfrwZ+g+hq/c3KzUBaQ8wqY/ZXiAbY= github.com/aws/aws-sdk-go v1.44.300/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= +github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw= +github.com/aws/aws-sdk-go-v2/config v1.18.28/go.mod h1:nIL+4/8JdAuNHEjn/gPEXqtnS02Q3NXB/9Z7o5xE4+A= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27 h1:dz0yr/yR1jweAnsCx+BmjerUILVPQ6FS5AwF/OyG1kA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27/go.mod h1:syOqAek45ZXZp29HlnRS/BNgMIW6uiRmeuQsz4Qh2UE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 h1:kP3Me6Fy3vdi+9uHd7YLr6ewPxRL+PU6y15urfTaamU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5/go.mod h1:Gj7tm95r+QsDoN2Fhuz/3npQvcZbkEf5mL70n3Xfluc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35/go.mod h1:ipR5PvpSPqIqL5Mi82BxLnfMkHVbmco8kUwO2xrCi0M= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36/go.mod h1:Rmw2M1hMVTwiUhjwMoIBFWFJMhvJbct06sSidxInkhY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 h1:cZG7psLfqpkB6H+fIrgUDWmlzM474St1LP0jcz272yI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27/go.mod h1:ZdjYvJpDlefgh8/hWelJhqgqJeodxu4SmbVsSdBlL7E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 h1:Bje8Xkh2OWpjBdNfXLrnn8eZg569dUQmhgtydxAYyP0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30/go.mod h1:qQtIBl5OVMfmeQkz8HaVyh5DzFmmFXyvK27UgIgOr4c= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29/go.mod h1:fDbkK4o7fpPXWn8YAPmTieAMuB9mk/VgvW64uaUqxd4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 h1:hx4WksB0NRQ9utR+2c3gEGzl6uKj3eM6PMQ6tN3lgXs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4/go.mod h1:JniVpqvw90sVjNqanGLufrVapWySL28fhBlYgl96Q/w= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 h1:PalLOEGZ/4XfQxpGZFTLaoJSmPoybnqJYotaIZEf/Rg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0/go.mod h1:PwyKKVL0cNkC37QwLcrhyeCrAk+5bY8O2ou7USyAS2A= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13/go.mod h1:DfX0sWuT46KpcqbMhJ9QWtxAIP1VozkDWf8VAkByjYY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13/go.mod h1:BzqsVVFduubEmzrVtUFQQIQdFqvUItF8XUq2EnS8Wog= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 h1:e5mnydVdCVWxP+5rPAGi2PYxC7u2OZgH1ypC114H04U= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3/go.mod h1:yVGZA1CPkmUhBdA039jXNJJG7/6t+G+EBWmFq23xqnY= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -155,6 +191,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= diff --git a/pkg/s3-proxy/config/config.go b/pkg/s3-proxy/config/config.go index f7c84648..050d764a 100644 --- a/pkg/s3-proxy/config/config.go +++ b/pkg/s3-proxy/config/config.go @@ -44,7 +44,7 @@ const DefaultLogFormat = "json" const DefaultBucketRegion = "us-east-1" // DefaultBucketS3ListMaxKeys Default bucket S3 list max keys. -const DefaultBucketS3ListMaxKeys int64 = 1000 +const DefaultBucketS3ListMaxKeys int32 = 1000 // DefaultTemplateFolderListPath Default template folder list path. const DefaultTemplateFolderListPath = "templates/folder-list.tpl" @@ -474,8 +474,7 @@ type BucketConfig struct { Prefix string `mapstructure:"prefix"` Region string `mapstructure:"region"` S3Endpoint string `mapstructure:"s3Endpoint"` - S3ListMaxKeys int64 `mapstructure:"s3ListMaxKeys" validate:"gt=0"` - DisableSSL bool `mapstructure:"disableSSL"` + S3ListMaxKeys int32 `mapstructure:"s3ListMaxKeys" validate:"gt=0"` } // BucketRequestConfig Bucket request configuration. diff --git a/pkg/s3-proxy/s3client/http-client.go b/pkg/s3-proxy/s3client/http-client.go new file mode 100644 index 00000000..b0968760 --- /dev/null +++ b/pkg/s3-proxy/s3client/http-client.go @@ -0,0 +1,39 @@ +package s3client + +import ( + "net/http" + + "github.com/aws/aws-sdk-go-v2/aws" +) + +type httpCl struct { + httpClient aws.HTTPClient + headers map[string]string +} + +func (h httpCl) Do(req *http.Request) (*http.Response, error) { + // Add all custom headers + for k, v := range h.headers { + req.Header.Add(k, v) + } + + return h.httpClient.Do(req) +} + +func customizeHTTPClient(cli aws.HTTPClient, headers map[string]string) aws.HTTPClient { + // Init original http client + oriHC := cli + // Check if it is nil + if oriHC == nil { + // Set global default http client + oriHC = http.DefaultClient + } + + // Create custom http client + h := &httpCl{ + httpClient: oriHC, + headers: headers, + } + + return h +} diff --git a/pkg/s3-proxy/s3client/manager.go b/pkg/s3-proxy/s3client/manager.go index 8baedf52..012429dc 100644 --- a/pkg/s3-proxy/s3client/manager.go +++ b/pkg/s3-proxy/s3client/manager.go @@ -1,11 +1,13 @@ package s3client import ( + "context" + "emperror.dev/errors" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/config" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/metrics" "github.com/thoas/go-funk" @@ -63,34 +65,44 @@ func (m *manager) Load() error { } func newClient(tgt *config.TargetConfig, metricsCtx metrics.Client) (Client, error) { - sessionConfig := &aws.Config{ - Region: aws.String(tgt.Bucket.Region), + params := []func(*awscfg.LoadOptions) error{ + awscfg.WithRegion(tgt.Bucket.Region), } // Load credentials if they exists if tgt.Bucket.Credentials != nil && tgt.Bucket.Credentials.AccessKey != nil && tgt.Bucket.Credentials.SecretKey != nil { - sessionConfig.Credentials = credentials.NewStaticCredentials(tgt.Bucket.Credentials.AccessKey.Value, tgt.Bucket.Credentials.SecretKey.Value, "") + params = append(params, awscfg.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(tgt.Bucket.Credentials.AccessKey.Value, tgt.Bucket.Credentials.SecretKey.Value, ""), + )) } // Load custom endpoint if it exists if tgt.Bucket.S3Endpoint != "" { - sessionConfig.Endpoint = aws.String(tgt.Bucket.S3Endpoint) - sessionConfig.S3ForcePathStyle = aws.Bool(true) - } - // Check if ssl needs to be disabled - if tgt.Bucket.DisableSSL { - sessionConfig.DisableSSL = aws.Bool(true) + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: tgt.Bucket.S3Endpoint, + SigningRegion: region, + HostnameImmutable: true, + }, nil + }) + + params = append(params, awscfg.WithEndpointResolverWithOptions(customResolver)) } - // Create session - sess, err := session.NewSession(sessionConfig) + + // Create configuration + cfg, err := awscfg.LoadDefaultConfig(context.TODO(), params...) // Check error if err != nil { return nil, errors.WithStack(err) } // Create s3 client - svcClient := s3.New(sess) + svcClient := s3.NewFromConfig(cfg) + // Create presign client + presignCl := s3.NewPresignClient(svcClient) return &s3Context{ - svcClient: svcClient, - target: tgt, - metricsCtx: metricsCtx, + svcClient: svcClient, + presignClient: presignCl, + target: tgt, + metricsCtx: metricsCtx, }, nil } diff --git a/pkg/s3-proxy/s3client/s3Context.go b/pkg/s3-proxy/s3client/s3Context.go index e9f2eb49..9bb26b64 100644 --- a/pkg/s3-proxy/s3client/s3Context.go +++ b/pkg/s3-proxy/s3client/s3Context.go @@ -6,20 +6,20 @@ import ( "time" "emperror.dev/errors" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/config" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/metrics" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/tracing" ) type s3Context struct { - svcClient s3iface.S3API - target *config.TargetConfig - metricsCtx metrics.Client + svcClient *s3.Client + presignClient *s3.PresignClient + target *config.TargetConfig + metricsCtx metrics.Client } // ListObjectsOperation List objects operation. @@ -37,7 +37,7 @@ const PutObjectOperation = "put-object" // DeleteObjectOperation Delete object operation. const DeleteObjectOperation = "delete-object" -const s3MaxKeys int64 = 1000 +const s3MaxKeys int32 = 1000 func (s3ctx *s3Context) buildGetObjectInputFromInput(input *GetInput) *s3.GetObjectInput { s3Input := &s3.GetObjectInput{ @@ -65,24 +65,24 @@ func (s3ctx *s3Context) buildGetObjectInputFromInput(input *GetInput) *s3.GetObj return s3Input } -func (s3ctx *s3Context) GetObjectSignedURL(_ context.Context, input *GetInput, expiration time.Duration) (string, error) { +func (s3ctx *s3Context) GetObjectSignedURL(ctx context.Context, input *GetInput, expiration time.Duration) (string, error) { // Build input s3Input := s3ctx.buildGetObjectInputFromInput(input) // Build object request - req, _ := s3ctx.svcClient.GetObjectRequest(s3Input) - // Build url - urlStr, err := req.Presign(expiration) + req, err := s3ctx.presignClient.PresignGetObject(ctx, s3Input, s3.WithPresignExpires(expiration)) // Check error if err != nil { - // Try to cast error into an AWS Error if possible - //nolint: errorlint // Cast - aerr, ok := err.(awserr.Error) - if ok { - // Check if it is a not found case - if aerr.Code() == s3.ErrCodeNoSuchKey { - return "", ErrNotFound - } else if aerr.Code() == "PreconditionFailed" { + var nsk *types.NoSuchKey + // Check no such key error + if errors.As(err, &nsk) { + return "", ErrNotFound + } + + // Check precondition failed + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + if apiErr.ErrorCode() == "PreconditionFailed" { return "", ErrPreconditionFailed } } @@ -90,7 +90,7 @@ func (s3ctx *s3Context) GetObjectSignedURL(_ context.Context, input *GetInput, e return "", errors.WithStack(err) } - return urlStr, nil + return req.URL, nil } // ListFilesAndDirectories List files and directories. @@ -131,62 +131,58 @@ func (s3ctx *s3Context) ListFilesAndDirectories(ctx context.Context, key string) childTrace.SetTag("s3-proxy.target-name", s3ctx.target.Name) // Request S3 - err := s3ctx.svcClient.ListObjectsV2PagesWithContext( + page, err := s3ctx.svcClient.ListObjectsV2( ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(s3ctx.target.Bucket.Name), Prefix: aws.String(key), Delimiter: aws.String("/"), - MaxKeys: aws.Int64(maxKeys), + MaxKeys: maxKeys, ContinuationToken: nextToken, }, - func(page *s3.ListObjectsV2Output, lastPage bool) bool { - // Store next token - nextToken = page.NextContinuationToken - - // Check if keycount exists - if page.KeyCount != nil { - // Remove current keys to tmp max elements - tmpMaxElements -= *page.KeyCount - // Update max keys if needed - if tmpMaxElements < maxKeys { - maxKeys = tmpMaxElements - } - } - - // Manage loop control - loopControl = nextToken != nil && tmpMaxElements > 0 - - // Manage folders - for _, item := range page.CommonPrefixes { - name := strings.TrimPrefix(*item.Prefix, key) - folders = append(folders, &ListElementOutput{ - Type: FolderType, - Key: *item.Prefix, - Name: name, - }) - } - - // Manage files - for _, item := range page.Contents { - name := strings.TrimPrefix(*item.Key, key) - if name != "" { - files = append(files, &ListElementOutput{ - Type: FileType, - ETag: *item.ETag, - Name: name, - LastModified: *item.LastModified, - Size: *item.Size, - Key: *item.Key, - }) - } - } - - return lastPage + func(o *s3.Options) { + o.HTTPClient = customizeHTTPClient(o.HTTPClient, requestHeaders) }, - addHeadersToRequest(requestHeaders), ) + // Store next token + nextToken = page.NextContinuationToken + + // Remove current keys to tmp max elements + tmpMaxElements -= page.KeyCount + // Update max keys if needed + if tmpMaxElements < maxKeys { + maxKeys = tmpMaxElements + } + + // Manage loop control + loopControl = nextToken != nil && tmpMaxElements > 0 + + // Manage folders + for _, item := range page.CommonPrefixes { + name := strings.TrimPrefix(*item.Prefix, key) + folders = append(folders, &ListElementOutput{ + Type: FolderType, + Key: *item.Prefix, + Name: name, + }) + } + + // Manage files + for _, item := range page.Contents { + name := strings.TrimPrefix(*item.Key, key) + if name != "" { + files = append(files, &ListElementOutput{ + Type: FileType, + ETag: *item.ETag, + Name: name, + LastModified: *item.LastModified, + Size: item.Size, + Key: *item.Key, + }) + } + } + // Metrics s3ctx.metricsCtx.IncS3Operations(s3ctx.target.Name, s3ctx.target.Bucket.Name, ListObjectsOperation) @@ -237,27 +233,30 @@ func (s3ctx *s3Context) GetObject(ctx context.Context, input *GetInput) (*GetOut requestHeaders = s3ctx.target.Bucket.RequestConfig.GetHeaders } - obj, err := s3ctx.svcClient.GetObjectWithContext( + obj, err := s3ctx.svcClient.GetObject( ctx, s3Input, - addHeadersToRequest(requestHeaders), + func(o *s3.Options) { + o.HTTPClient = customizeHTTPClient(o.HTTPClient, requestHeaders) + }, ) // Metrics s3ctx.metricsCtx.IncS3Operations(s3ctx.target.Name, s3ctx.target.Bucket.Name, GetObjectOperation) // Check if error exists if err != nil { - // Try to cast error into an AWS Error if possible - //nolint: errorlint // Cast - aerr, ok := err.(awserr.Error) - if ok { - // Check if it is a not found case - //nolint: gocritic // Because don't want to write a switch for the moment - if aerr.Code() == s3.ErrCodeNoSuchKey { - return nil, nil, ErrNotFound - } else if aerr.Code() == "NotModified" { - return nil, nil, ErrNotModified - } else if aerr.Code() == "PreconditionFailed" { + var nsk *types.NoSuchKey + // Check no such key error + if errors.As(err, &nsk) { + return nil, nil, ErrNotFound + } + + // Check precondition failed + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + if apiErr.ErrorCode() == "PreconditionFailed" { return nil, nil, ErrPreconditionFailed + } else if apiErr.ErrorCode() == "NotModified" { + return nil, nil, ErrNotModified } } @@ -265,12 +264,9 @@ func (s3ctx *s3Context) GetObject(ctx context.Context, input *GetInput) (*GetOut } // Build output output := &GetOutput{ - Body: obj.Body, - } - - // Metadata transformation - if obj.Metadata != nil { - output.Metadata = aws.StringValueMap(obj.Metadata) + Body: obj.Body, + ContentLength: obj.ContentLength, + Metadata: obj.Metadata, } if obj.CacheControl != nil { @@ -278,7 +274,7 @@ func (s3ctx *s3Context) GetObject(ctx context.Context, input *GetInput) (*GetOut } if obj.Expires != nil { - output.Expires = *obj.Expires + output.Expires = obj.Expires.Format(time.RFC1123) } if obj.ContentDisposition != nil { @@ -293,10 +289,6 @@ func (s3ctx *s3Context) GetObject(ctx context.Context, input *GetInput) (*GetOut output.ContentLanguage = *obj.ContentLanguage } - if obj.ContentLength != nil { - output.ContentLength = *obj.ContentLength - } - if obj.ContentRange != nil { output.ContentRange = *obj.ContentRange } @@ -339,10 +331,11 @@ func (s3ctx *s3Context) PutObject(ctx context.Context, input *PutInput) (*Result inp := &s3.PutObjectInput{ Body: input.Body, - ContentLength: aws.Int64(input.ContentSize), + ContentLength: input.ContentSize, Bucket: aws.String(s3ctx.target.Bucket.Name), Key: aws.String(input.Key), Expires: input.Expires, + Metadata: input.Metadata, } // Manage ACL @@ -352,7 +345,7 @@ func (s3ctx *s3Context) PutObject(ctx context.Context, input *PutInput) (*Result s3ctx.target.Actions.PUT.Config.CannedACL != nil && *s3ctx.target.Actions.PUT.Config.CannedACL != "" { // Inject ACL - inp.ACL = s3ctx.target.Actions.PUT.Config.CannedACL + inp.ACL = types.ObjectCannedACL(*s3ctx.target.Actions.PUT.Config.CannedACL) } // Manage cache control case if input.CacheControl != "" { @@ -374,13 +367,9 @@ func (s3ctx *s3Context) PutObject(ctx context.Context, input *PutInput) (*Result if input.ContentType != "" { inp.ContentType = aws.String(input.ContentType) } - // Manage metadata case - if input.Metadata != nil { - inp.Metadata = aws.StringMap(input.Metadata) - } // Manage storage class if input.StorageClass != "" { - inp.StorageClass = aws.String(input.StorageClass) + inp.StorageClass = types.StorageClass(input.StorageClass) } // Init & get request headers @@ -390,10 +379,12 @@ func (s3ctx *s3Context) PutObject(ctx context.Context, input *PutInput) (*Result } // Upload to S3 bucket - _, err := s3ctx.svcClient.PutObjectWithContext( + _, err := s3ctx.svcClient.PutObject( ctx, inp, - addHeadersToRequest(requestHeaders), + func(o *s3.Options) { + o.HTTPClient = customizeHTTPClient(o.HTTPClient, requestHeaders) + }, ) // Check error if err != nil { @@ -435,24 +426,24 @@ func (s3ctx *s3Context) HeadObject(ctx context.Context, key string) (*HeadOutput } // Head object in bucket - _, err := s3ctx.svcClient.HeadObjectWithContext( + _, err := s3ctx.svcClient.HeadObject( ctx, &s3.HeadObjectInput{ Bucket: aws.String(s3ctx.target.Bucket.Name), Key: aws.String(key), }, - addHeadersToRequest(requestHeaders), + func(o *s3.Options) { + o.HTTPClient = customizeHTTPClient(o.HTTPClient, requestHeaders) + }, ) // Metrics s3ctx.metricsCtx.IncS3Operations(s3ctx.target.Name, s3ctx.target.Bucket.Name, HeadObjectOperation) // Test error if err != nil { - // Try to cast error into an AWS Error if possible - //nolint: errorlint // Cast - aerr, ok := err.(awserr.Error) - if ok { - // Issue not fixed: https://github.com/aws/aws-sdk-go/issues/1208 - if aerr.Code() == "NotFound" { + // Check precondition failed + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + if apiErr.ErrorCode() == "NotFound" { return nil, ErrNotFound } } @@ -488,13 +479,15 @@ func (s3ctx *s3Context) DeleteObject(ctx context.Context, key string) (*ResultIn } // Delete object - _, err := s3ctx.svcClient.DeleteObjectWithContext( + _, err := s3ctx.svcClient.DeleteObject( ctx, &s3.DeleteObjectInput{ Bucket: aws.String(s3ctx.target.Bucket.Name), Key: aws.String(key), }, - addHeadersToRequest(requestHeaders), + func(o *s3.Options) { + o.HTTPClient = customizeHTTPClient(o.HTTPClient, requestHeaders) + }, ) // Check error if err != nil { @@ -515,12 +508,3 @@ func (s3ctx *s3Context) DeleteObject(ctx context.Context, key string) (*ResultIn // Return return info, nil } - -func addHeadersToRequest(headers map[string]string) func(r *request.Request) { - return func(r *request.Request) { - // Loop over them - for k, v := range headers { - r.HTTPRequest.Header.Set(k, v) - } - } -} diff --git a/pkg/s3-proxy/server/server_integration_test.go b/pkg/s3-proxy/server/server_integration_test.go index a47e3416..c01e77dc 100644 --- a/pkg/s3-proxy/server/server_integration_test.go +++ b/pkg/s3-proxy/server/server_integration_test.go @@ -4,6 +4,7 @@ package server import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -19,8 +20,8 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" "github.com/golang/mock/gomock" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/config" cmocks "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/config/mocks" @@ -80,13 +81,13 @@ func TestPublicRouter(t *testing.T) { expectedHeaderContains map[string][]string validateS3Object bool expectedS3ObjectKey string - expectedS3ObjectMetadata map[string]*string + expectedS3ObjectMetadata map[string]string expectedS3ObjectCacheControl *string expectedS3ObjectContentDisposition *string expectedS3ObjectContentEncoding *string expectedS3ObjectContentLanguage *string - expectedS3ObjectExpires *string - expectedS3ObjectStorageClass *string + expectedS3ObjectExpires string + expectedS3ObjectStorageClass string notExpectedBody string wantErr bool }{ @@ -109,7 +110,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -149,7 +149,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -188,7 +187,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -231,7 +229,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -280,7 +277,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -320,7 +316,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -370,7 +365,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -413,7 +407,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -459,7 +452,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -510,7 +502,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -535,7 +526,7 @@ func TestPublicRouter(t *testing.T) { }, }, { - name: "GET a file with success (redirect to signed url enabled)", + name: "GET a file with success _redirect to signed url enabled_", args: args{ cfg: &config.Config{ Server: svrCfg, @@ -553,7 +544,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -586,7 +576,7 @@ func TestPublicRouter(t *testing.T) { "X-Amz-Credential=" + accessKey, region, "X-Amz-Expires=60", - "X-Amz-SignedHeaders=host&X-Amz-Signature=", + "X-Amz-SignedHeaders=host&x-id=GetObject&X-Amz-Signature=", }, }, }, @@ -609,7 +599,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -654,7 +643,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -706,7 +694,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -752,7 +739,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -800,7 +786,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -870,7 +855,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -940,7 +924,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1011,7 +994,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1084,7 +1066,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1157,7 +1138,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1223,7 +1203,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1290,7 +1269,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1347,7 +1325,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1417,7 +1394,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1491,7 +1467,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1562,7 +1537,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1638,7 +1612,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1714,7 +1687,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1792,7 +1764,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1863,7 +1834,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -1934,7 +1904,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2014,7 +1983,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2094,7 +2062,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2168,7 +2135,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2249,7 +2215,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2324,7 +2289,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2399,7 +2363,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2456,7 +2419,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2481,7 +2443,7 @@ func TestPublicRouter(t *testing.T) { }, }, { - name: "GET index document with index document enabled with success", + name: "GET index document with index document enabled with success on signed url", args: args{ cfg: &config.Config{ Server: svrCfg, @@ -2499,7 +2461,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2533,7 +2494,7 @@ func TestPublicRouter(t *testing.T) { "X-Amz-Credential=" + accessKey, region, "X-Amz-Expires=60", - "X-Amz-SignedHeaders=host&X-Amz-Signature=", + "X-Amz-SignedHeaders=host&x-id=GetObject&X-Amz-Signature=", }, }, }, @@ -2556,7 +2517,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2599,7 +2559,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2637,7 +2596,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2693,7 +2651,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2739,7 +2696,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2794,7 +2750,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2826,8 +2781,8 @@ func TestPublicRouter(t *testing.T) { }, validateS3Object: true, expectedS3ObjectKey: "folder1/test2.txt", - expectedS3ObjectStorageClass: aws.String("Standard"), - expectedS3ObjectMetadata: aws.StringMap(map[string]string{"Meta1": "meta1"}), + expectedS3ObjectStorageClass: "Standard", + expectedS3ObjectMetadata: map[string]string{"meta1": "meta1"}, }, { name: "PUT a file with system metadata", @@ -2848,7 +2803,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2884,7 +2838,7 @@ func TestPublicRouter(t *testing.T) { }, validateS3Object: true, expectedS3ObjectKey: "folder1/system-metadata.txt", - expectedS3ObjectStorageClass: aws.String("Standard"), + expectedS3ObjectStorageClass: "Standard", expectedS3ObjectContentDisposition: aws.String("attachment"), expectedS3ObjectContentEncoding: aws.String("content-encoding"), // Cannot be testing due to limitation on library @@ -2911,7 +2865,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -2969,7 +2922,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3002,8 +2954,8 @@ func TestPublicRouter(t *testing.T) { }, validateS3Object: true, expectedS3ObjectKey: "folder1/test.txt", - expectedS3ObjectStorageClass: aws.String("Standard"), - expectedS3ObjectMetadata: aws.StringMap(map[string]string{"Meta1": "meta1", "M1-Key": "v1"}), + expectedS3ObjectStorageClass: "Standard", + expectedS3ObjectMetadata: map[string]string{"meta1": "meta1", "m1-key": "v1"}, }, { name: "PUT in a path should fail because no input", @@ -3024,7 +2976,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3073,7 +3024,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3142,7 +3092,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3182,8 +3131,8 @@ func TestPublicRouter(t *testing.T) { `, validateS3Object: true, expectedS3ObjectKey: "folder1/test.txt", - expectedS3ObjectStorageClass: aws.String("Standard"), - expectedS3ObjectMetadata: aws.StringMap(map[string]string{"Meta1": "meta1", "M1-Key": "v1"}), + expectedS3ObjectStorageClass: "Standard", + expectedS3ObjectMetadata: map[string]string{"meta1": "meta1", "m1-key": "v1"}, }, { name: "PUT with a custom target template and status code should be ok", @@ -3204,7 +3153,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3253,8 +3201,8 @@ func TestPublicRouter(t *testing.T) { `, validateS3Object: true, expectedS3ObjectKey: "folder1/test.txt", - expectedS3ObjectStorageClass: aws.String("Standard"), - expectedS3ObjectMetadata: aws.StringMap(map[string]string{"Meta1": "meta1", "M1-Key": "v1"}), + expectedS3ObjectStorageClass: "Standard", + expectedS3ObjectMetadata: map[string]string{"meta1": "meta1", "m1-key": "v1"}, }, { name: "GET a file with success with space in path", @@ -3275,7 +3223,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3297,7 +3244,7 @@ func TestPublicRouter(t *testing.T) { }, }, { - name: "GET a file with success with custom headers (general helpers)", + name: "GET a file with success with custom headers _general helpers_", args: args{ cfg: &config.Config{ Server: svrCfg, @@ -3315,7 +3262,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3325,7 +3271,7 @@ func TestPublicRouter(t *testing.T) { Enabled: true, Config: &config.GetActionConfigConfig{ StreamedFileHeaders: map[string]string{ - "Fake": "{{ index .StreamFile.Metadata \"M1-Key\" }}", + "Fake": "{{ index .StreamFile.Metadata \"m1-key\" }}", }, }, }, @@ -3345,7 +3291,7 @@ func TestPublicRouter(t *testing.T) { }, }, { - name: "GET a file with success with custom headers (target helpers)", + name: "GET a file with success with custom headers _target helpers_", args: args{ cfg: &config.Config{ Server: svrCfg, @@ -3363,7 +3309,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3379,7 +3324,7 @@ func TestPublicRouter(t *testing.T) { Enabled: true, Config: &config.GetActionConfigConfig{ StreamedFileHeaders: map[string]string{ - "Fake": "{{ index .StreamFile.Metadata \"M1-Key\" }}", + "Fake": "{{ index .StreamFile.Metadata \"m1-key\" }}", }, }, }, @@ -3426,7 +3371,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3473,7 +3417,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3529,7 +3472,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3602,7 +3544,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3650,7 +3591,6 @@ func TestPublicRouter(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -3815,19 +3755,20 @@ func TestPublicRouter(t *testing.T) { assert.Equal(t, tt.expectedCode, w.Code) if tt.validateS3Object { - r, err := s3serverClient.HeadObject(&s3.HeadObjectInput{ + r, err := s3serverClient.HeadObject(context.TODO(), &s3.HeadObjectInput{ Bucket: &bucket, Key: &tt.expectedS3ObjectKey, }) assert.NoError(t, err) - // TODO Migrate to SDK V2 to test ACL assert.Equal(t, tt.expectedS3ObjectMetadata, r.Metadata, "S3 object metadata") - assert.Equal(t, tt.expectedS3ObjectStorageClass, r.StorageClass, "S3 object storage class") + assert.Equal(t, tt.expectedS3ObjectStorageClass, string(r.StorageClass), "S3 object storage class") assert.Equal(t, tt.expectedS3ObjectCacheControl, r.CacheControl, "S3 object cache control") assert.Equal(t, tt.expectedS3ObjectContentDisposition, r.ContentDisposition, "S3 object content disposition") assert.Equal(t, tt.expectedS3ObjectContentEncoding, r.ContentEncoding, "S3 object content encoding") assert.Equal(t, tt.expectedS3ObjectContentLanguage, r.ContentLanguage, "S3 object content language") - assert.Equal(t, tt.expectedS3ObjectExpires, r.Expires, "S3 object expires") + if r.Expires != nil { + assert.Equal(t, tt.expectedS3ObjectExpires, r.Expires.Format(time.RFC1123), "S3 object expires") + } } }) } @@ -3868,7 +3809,6 @@ func TestTracing(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -4080,7 +4020,6 @@ func TestOIDCAuthentication(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -4109,7 +4048,6 @@ func TestOIDCAuthentication(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount-multiple-provider/"}, @@ -4138,7 +4076,6 @@ func TestOIDCAuthentication(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount-opa-server/"}, @@ -4169,7 +4106,6 @@ func TestOIDCAuthentication(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount-wrong-opa-server-url/"}, @@ -4200,7 +4136,6 @@ func TestOIDCAuthentication(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount-with-group/"}, @@ -4961,7 +4896,6 @@ func TestCORS(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, @@ -5356,7 +5290,6 @@ func TestIndexLargeBucket(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, S3ListMaxKeys: 1000, }, Mount: &config.MountConfig{ @@ -5467,8 +5400,7 @@ func TestListLargeBucketAndSmallMaxKeys(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, - S3ListMaxKeys: int64(maxKeys), + S3ListMaxKeys: int32(maxKeys), }, Mount: &config.MountConfig{ Path: []string{"/"}, @@ -5571,8 +5503,7 @@ func TestListLargeBucketAndMaxKeysGreaterThanS3MaxKeys(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, - S3ListMaxKeys: int64(maxKeys), + S3ListMaxKeys: int32(maxKeys), }, Mount: &config.MountConfig{ Path: []string{"/"}, @@ -5674,7 +5605,6 @@ func TestFolderWithSubFolders(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/"}, @@ -5782,7 +5712,6 @@ func TestTrailingSlashRedirect(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, } type args struct { cfg *config.Config diff --git a/pkg/s3-proxy/server/tls_integration_test.go b/pkg/s3-proxy/server/tls_integration_test.go index 493f6ec6..1d2e09d4 100644 --- a/pkg/s3-proxy/server/tls_integration_test.go +++ b/pkg/s3-proxy/server/tls_integration_test.go @@ -94,7 +94,6 @@ func TestTLSServer(t *testing.T) { AccessKey: &config.CredentialConfig{Value: accessKey}, SecretKey: &config.CredentialConfig{Value: secretAccessKey}, }, - DisableSSL: true, }, Mount: &config.MountConfig{ Path: []string{"/mount/"}, diff --git a/pkg/s3-proxy/server/utils_integration_test.go b/pkg/s3-proxy/server/utils_integration_test.go index 43275fed..be4f29a8 100644 --- a/pkg/s3-proxy/server/utils_integration_test.go +++ b/pkg/s3-proxy/server/utils_integration_test.go @@ -3,14 +3,15 @@ package server import ( + "context" "fmt" "net/http/httptest" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/johannesboyne/gofakes3" "github.com/johannesboyne/gofakes3/backend/s3mem" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/config" @@ -105,28 +106,37 @@ var testsDefaultGeneralTemplateConfig = &config.TemplateConfig{ // Generate metrics instance var metricsCtx = metrics.NewClient() -func setupFakeS3(accessKey, secretAccessKey, region, bucket string) (*s3.S3, *httptest.Server, error) { +func setupFakeS3(accessKey, secretAccessKey, region, bucket string) (*s3.Client, *httptest.Server, error) { backend := s3mem.New() faker := gofakes3.New(backend) ts := httptest.NewServer(faker.Server()) - // configure S3 client - s3Config := &aws.Config{ - Credentials: credentials.NewStaticCredentials(accessKey, secretAccessKey, ""), - Endpoint: aws.String(ts.URL), - Region: aws.String(region), - DisableSSL: aws.Bool(true), - S3ForcePathStyle: aws.Bool(true), + cfg, err := awscfg.LoadDefaultConfig( + context.TODO(), + awscfg.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: ts.URL, + SigningRegion: region, + HostnameImmutable: true, + }, nil + })), + awscfg.WithRegion(region), + awscfg.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(accessKey, secretAccessKey, ""), + ), + ) + if err != nil { + return nil, nil, err } - newSession := session.New(s3Config) - s3Client := s3.New(newSession) + s3Client := s3.NewFromConfig(cfg) cparams := &s3.CreateBucketInput{ Bucket: aws.String(bucket), } // Create a new bucket using the CreateBucket call. - _, err := s3Client.CreateBucket(cparams) + _, err = s3Client.CreateBucket(context.TODO(), cparams) if err != nil { return nil, nil, err } @@ -155,13 +165,21 @@ func setupFakeS3(accessKey, secretAccessKey, region, bucket string) (*s3.S3, *ht // Upload files for k, v := range files { - _, err = s3Client.PutObject(&s3.PutObjectInput{ + var contentType *string + if strings.HasSuffix(k, ".txt") { + contentType = aws.String("text/plain; charset=utf-8") + } else if strings.HasSuffix(k, ".html") { + contentType = aws.String("text/html; charset=utf-8") + } + + _, err = s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ Body: strings.NewReader(v), Bucket: aws.String(bucket), Key: aws.String(k), - Metadata: aws.StringMap(map[string]string{ + Metadata: map[string]string{ "m1-key": "v1", - }), + }, + ContentType: contentType, }) if err != nil { return nil, nil, err @@ -169,13 +187,11 @@ func setupFakeS3(accessKey, secretAccessKey, region, bucket string) (*s3.S3, *ht } // Add file with content-type - _, err = s3Client.PutObject(&s3.PutObjectInput{ - Body: strings.NewReader("test"), - Bucket: aws.String(bucket), - Key: aws.String("content-type/file.txt"), - Metadata: map[string]*string{ - "Content-Type": aws.String("text/plain"), - }, + _, err = s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("test"), + Bucket: aws.String(bucket), + Key: aws.String("content-type/file.txt"), + ContentType: aws.String("text/plain; charset=utf-8"), }) if err != nil { return nil, nil, err