From 78ae1eddf337a395cebb2f22fa6f49bcd4b24e85 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 11 Feb 2025 10:35:25 +0300 Subject: [PATCH 1/5] multipart: Drop set-only struct fields These fields have never been read since they was added in 47b560ebfc0871573bfbc119354fa67b4329b9d9. Signed-off-by: Leonard Lyubich --- api/layer/multipart_upload.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index 0dff2c9e..48c1a25c 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -150,7 +150,6 @@ type ( PartNumber int64 // in nanoseconds CreatedAt int64 - FilePath string } uploadPartAsSlotParams struct { @@ -1433,8 +1432,6 @@ func (n *layer) getSlotAttributes(obj object.Object) (*slotAttributes, error) { attributes.PartNumber, err = strconv.ParseInt(attr.Value(), 10, 64) case headerS3MultipartCreated: attributes.CreatedAt, err = strconv.ParseInt(attr.Value(), 10, 64) - case object.AttributeFilePath: - attributes.FilePath = attr.Value() default: continue } From 814f45ba3567f9f0ca13c19b72c26e6238072b6f Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 11 Feb 2025 10:43:25 +0300 Subject: [PATCH 2/5] multipart: Inline a once called method No longer need additional type also. Signed-off-by: Leonard Lyubich --- api/layer/multipart_upload.go | 50 +++++++++++------------------------ 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index 48c1a25c..bb5ffdcf 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -146,12 +146,6 @@ type ( Created time.Time } - slotAttributes struct { - PartNumber int64 - // in nanoseconds - CreatedAt int64 - } - uploadPartAsSlotParams struct { bktInfo *data.BucketInfo multipartInfo *data.MultipartInfo @@ -1420,30 +1414,6 @@ func (n *layer) uploadPartAsSlot(ctx context.Context, params uploadPartAsSlotPar return &objInfo, nil } -func (n *layer) getSlotAttributes(obj object.Object) (*slotAttributes, error) { - var ( - attributes slotAttributes - err error - ) - - for _, attr := range obj.Attributes() { - switch attr.Key() { - case headerS3MultipartNumber: - attributes.PartNumber, err = strconv.ParseInt(attr.Value(), 10, 64) - case headerS3MultipartCreated: - attributes.CreatedAt, err = strconv.ParseInt(attr.Value(), 10, 64) - default: - continue - } - - if err != nil { - return nil, fmt.Errorf("parse header: %w", err) - } - } - - return &attributes, nil -} - func (n *layer) getFirstArbitraryPart(ctx context.Context, uploadID string, bucketInfo *data.BucketInfo) (int64, error) { var filters object.SearchFilters filters.AddFilter(headerS3MultipartUpload, uploadID, object.MatchStringEqual) @@ -1471,16 +1441,26 @@ func (n *layer) getFirstArbitraryPart(ctx context.Context, uploadID string, buck if err != nil { return 0, fmt.Errorf("object head: %w", err) } + var nextPartNumber int64 + for _, attr := range head.Attributes() { + switch attr.Key() { + case headerS3MultipartNumber: + nextPartNumber, err = strconv.ParseInt(attr.Value(), 10, 64) + case headerS3MultipartCreated: + _, err = strconv.ParseInt(attr.Value(), 10, 64) + default: + continue + } - attributes, err := n.getSlotAttributes(*head) - if err != nil { - return 0, fmt.Errorf("get slot attributes: %w", err) + if err != nil { + return 0, fmt.Errorf("parse header: %w", err) + } } if partNumber == 0 { - partNumber = attributes.PartNumber + partNumber = nextPartNumber } else { - partNumber = min(partNumber, attributes.PartNumber) + partNumber = min(partNumber, nextPartNumber) } } From d1c8d95d5c96d711e2d1d50a6630ab2a2278cb3a Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 11 Feb 2025 10:59:39 +0300 Subject: [PATCH 3/5] api/layer: Drop unused comment Neither TODO nor understanding why it might be needed. Signed-off-by: Leonard Lyubich --- api/layer/object.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/layer/object.go b/api/layer/object.go index c9b50df5..635350cc 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -483,10 +483,6 @@ func (n *layer) searchObjects(ctx context.Context, bkt *data.BucketInfo, prmSear return nil, fmt.Errorf("couldn't head object: %w", err) } - // if head.Type() == object.TypeTombstone || head.Type() == object.TypeLink || head.Type() == object.TypeLock { - // continue - // } - // The object is a part of split chain, it doesn't exist for user. if head.HasParent() { continue From e2bbda306da46687a9ff15c6f7d449e8a901947e Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 11 Feb 2025 11:45:17 +0300 Subject: [PATCH 4/5] WIP SearchV2 Signed-off-by: Evgenii Baidakov --- api/layer/layer.go | 9 +++++- api/layer/multipart_upload.go | 52 ++++++++++++----------------------- api/layer/neofs.go | 3 ++ api/layer/tagging.go | 27 +++++++++--------- 4 files changed, 42 insertions(+), 49 deletions(-) diff --git a/api/layer/layer.go b/api/layer/layer.go index 500e0b6b..aeb440cc 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -19,6 +19,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption" "github.com/nspcc-dev/neofs-s3-gw/api/s3errors" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" + "github.com/nspcc-dev/neofs-sdk-go/bearer" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/eacl" @@ -350,11 +351,17 @@ func (n *layer) OwnerPublicKey(ctx context.Context) (*keys.PublicKey, error) { } func (n *layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth, bktOwner user.ID) { + // TODO: drop method + prm.BearerToken = bearerTokenFromContext(ctx, bktOwner) +} + +func bearerTokenFromContext(ctx context.Context, bktOwner user.ID) *bearer.Token { if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil { if bktOwner.Equals(bd.Gate.BearerToken.ResolveIssuer()) { - prm.BearerToken = bd.Gate.BearerToken + return bd.Gate.BearerToken } } + return nil } // GetBucketInfo returns bucket info by name. diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index bb5ffdcf..c438f68c 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -22,6 +22,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption" "github.com/nspcc-dev/neofs-s3-gw/api/s3errors" + "github.com/nspcc-dev/neofs-sdk-go/client" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "github.com/nspcc-dev/neofs-sdk-go/user" @@ -1416,52 +1417,33 @@ func (n *layer) uploadPartAsSlot(ctx context.Context, params uploadPartAsSlotPar func (n *layer) getFirstArbitraryPart(ctx context.Context, uploadID string, bucketInfo *data.BucketInfo) (int64, error) { var filters object.SearchFilters + filters.AddFilter(headerS3MultipartNumber, "0", object.MatchNumGE) // only primary attribute is sorted filters.AddFilter(headerS3MultipartUpload, uploadID, object.MatchStringEqual) - var prmSearch = PrmObjectSearch{ - Container: bucketInfo.CID, - Filters: filters, + var opts client.SearchObjectsOptions + opts.SetCount(1) + if bt := bearerTokenFromContext(ctx, bucketInfo.Owner); bt != nil { + opts.WithBearerToken(*bt) } - n.prepareAuthParameters(ctx, &prmSearch.PrmAuth, bucketInfo.Owner) - - oids, err := n.neoFS.SearchObjects(ctx, prmSearch) + res, err := n.neoFS.SearchObjectsV2(ctx, bucketInfo.CID, filters, []string{ + headerS3MultipartNumber, + headerS3MultipartCreated, + }, opts) if err != nil { return 0, fmt.Errorf("search objects: %w", err) } - if len(oids) == 0 { + if len(res) == 0 { return 0, nil } - var partNumber int64 - - for _, id := range oids { - head, err := n.objectHead(ctx, bucketInfo, id) - if err != nil { - return 0, fmt.Errorf("object head: %w", err) - } - var nextPartNumber int64 - for _, attr := range head.Attributes() { - switch attr.Key() { - case headerS3MultipartNumber: - nextPartNumber, err = strconv.ParseInt(attr.Value(), 10, 64) - case headerS3MultipartCreated: - _, err = strconv.ParseInt(attr.Value(), 10, 64) - default: - continue - } - - if err != nil { - return 0, fmt.Errorf("parse header: %w", err) - } - } - - if partNumber == 0 { - partNumber = nextPartNumber - } else { - partNumber = min(partNumber, nextPartNumber) - } + partNumber, err := strconv.ParseInt(res[0].Attributes[0], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse header: %w", err) + } + if _, err = strconv.ParseInt(res[0].Attributes[1], 10, 64); err != nil { + return 0, fmt.Errorf("parse header: %w", err) } return partNumber, nil diff --git a/api/layer/neofs.go b/api/layer/neofs.go index b4a44fae..9ac0f92a 100644 --- a/api/layer/neofs.go +++ b/api/layer/neofs.go @@ -9,6 +9,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-sdk-go/bearer" + "github.com/nspcc-dev/neofs-sdk-go/client" "github.com/nspcc-dev/neofs-sdk-go/container" "github.com/nspcc-dev/neofs-sdk-go/container/acl" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" @@ -278,4 +279,6 @@ type NeoFS interface { // SearchObjects searches objects with corresponding filters. SearchObjects(ctx context.Context, prm PrmObjectSearch) ([]oid.ID, error) + + SearchObjectsV2(context.Context, cid.ID, object.SearchFilters, []string, client.SearchObjectsOptions) ([]client.SearchResultItem, error) } diff --git a/api/layer/tagging.go b/api/layer/tagging.go index b63fea92..864e473e 100644 --- a/api/layer/tagging.go +++ b/api/layer/tagging.go @@ -10,6 +10,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/s3errors" + "github.com/nspcc-dev/neofs-sdk-go/client" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" @@ -139,20 +140,20 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *GetObjectTaggingParams) } func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error { - prmSearch := PrmObjectSearch{ - Container: p.BktInfo.CID, - Filters: make(object.SearchFilters, 0, 3), + fs := make(object.SearchFilters, 0, 4) + fs.AddFilter(object.AttributeFilePath, p.ObjectName, object.MatchStringEqual) + fs.AddFilter(attributeTagsMetaObject, "true", object.MatchStringEqual) + fs.AddTypeFilter(object.MatchStringEqual, object.TypeRegular) + if p.VersionID != "" { + fs.AddFilter(AttributeObjectVersion, p.VersionID, object.MatchStringEqual) } - n.prepareAuthParameters(ctx, &prmSearch.PrmAuth, p.BktInfo.Owner) - prmSearch.Filters.AddFilter(object.AttributeFilePath, p.ObjectName, object.MatchStringEqual) - prmSearch.Filters.AddFilter(attributeTagsMetaObject, "true", object.MatchStringEqual) - prmSearch.Filters.AddTypeFilter(object.MatchStringEqual, object.TypeRegular) - if p.VersionID != "" { - prmSearch.Filters.AddFilter(AttributeObjectVersion, p.VersionID, object.MatchStringEqual) + var opts client.SearchObjectsOptions + if bt := bearerTokenFromContext(ctx, p.BktInfo.Owner); bt != nil { + opts.WithBearerToken(*bt) } - ids, err := n.neoFS.SearchObjects(ctx, prmSearch) + res, err := n.neoFS.SearchObjectsV2(ctx, p.BktInfo.CID, fs, nil, opts) if err != nil { if errorsStd.Is(err, apistatus.ErrObjectAccessDenied) { return s3errors.GetAPIError(s3errors.ErrAccessDenied) @@ -161,12 +162,12 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *ObjectVersion) error return fmt.Errorf("search object version: %w", err) } - if len(ids) == 0 { + if len(res) == 0 { return nil } - for _, id := range ids { - if err = n.objectDelete(ctx, p.BktInfo, id); err != nil { + for i := range res { + if err = n.objectDelete(ctx, p.BktInfo, res[i].ID); err != nil { return fmt.Errorf("couldn't delete object: %w", err) } } From 9f08b34880dbaa935e2288a612b14bf07997e92f Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Wed, 12 Feb 2025 13:19:23 +0400 Subject: [PATCH 5/5] WIP SearchV2 Signed-off-by: Evgenii Baidakov --- api/data/info.go | 12 +++ api/handler/object_list.go | 13 +-- api/layer/object.go | 213 +++++++++++++++++++++++++++---------- api/layer/util.go | 2 +- internal/neofs/neofs.go | 24 +++++ 5 files changed, 200 insertions(+), 64 deletions(-) diff --git a/api/data/info.go b/api/data/info.go index b38da040..dd3b536d 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -65,6 +65,18 @@ type ( Headers map[string]string } + ObjectListResponseContent struct { + IsDir bool + + ID oid.ID + DecryptedSize int64 + Size int64 + Owner user.ID + HashSum string + Created time.Time + Name string + } + // NotificationInfo store info to send s3 notification. NotificationInfo struct { Name string diff --git a/api/handler/object_list.go b/api/handler/object_list.go index 52bc9bf6..fc238f6c 100644 --- a/api/handler/object_list.go +++ b/api/handler/object_list.go @@ -207,11 +207,11 @@ func fillPrefixes(src []string, encode string) []CommonPrefix { return dst } -func fillContentsWithOwner(src []*data.ObjectInfo, encode string) ([]Object, error) { +func fillContentsWithOwner(src []data.ObjectListResponseContent, encode string) ([]Object, error) { return fillContents(src, encode, true) } -func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) ([]Object, error) { +func fillContents(src []data.ObjectListResponseContent, encode string, fetchOwner bool) ([]Object, error) { var dst []Object for _, obj := range src { res := Object{ @@ -221,13 +221,8 @@ func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) ([]Obj ETag: obj.HashSum, } - if size, ok := obj.Headers[layer.AttributeDecryptedSize]; ok { - sz, err := strconv.ParseInt(size, 10, 64) - if err != nil { - return nil, fmt.Errorf("parse decrypted size %s: %w", size, err) - } - - res.Size = sz + if obj.DecryptedSize > 0 { + res.Size = obj.DecryptedSize } if fetchOwner { diff --git a/api/layer/object.go b/api/layer/object.go index 635350cc..92f9c6b9 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -22,6 +22,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api/cache" "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/api/s3errors" + "github.com/nspcc-dev/neofs-sdk-go/client" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" "github.com/nspcc-dev/neofs-sdk-go/object" @@ -72,6 +73,14 @@ type ( Marker string ContinuationToken string } + + prefixSearchResult struct { + ID oid.ID + FilePath string + CreationEpoch uint64 + CreationTimestamp int64 + IsDeleteMarker bool + } ) const ( @@ -435,25 +444,96 @@ func (n *layer) searchAllVersionsInNeoFS(ctx context.Context, bkt *data.BucketIn // searchAllVersionsInNeoFS returns all version of object by its objectName. // // Returns ErrNodeNotFound if zero objects found. -func (n *layer) searchAllVersionsInNeoFSByPrefix(ctx context.Context, bkt *data.BucketInfo, owner user.ID, prefix string, onlyUnversioned bool) ([]*object.Object, error) { - prmSearch := PrmObjectSearch{ - Container: bkt.CID, - Filters: make(object.SearchFilters, 0, 4), +func (n *layer) searchAllVersionsInNeoFSByPrefix(ctx context.Context, bkt *data.BucketInfo, owner user.ID, prefix string, onlyUnversioned bool) ([]prefixSearchResult, error) { + var ( + filters = make(object.SearchFilters, 0, 4) + returningAttributes = []string{ + object.FilterCreationEpoch, + object.AttributeFilePath, + object.AttributeTimestamp, + attrS3DeleteMarker, + } + + opts client.SearchObjectsOptions + ) + + if bt := bearerTokenFromContext(ctx, owner); bt != nil { + opts.WithBearerToken(*bt) } - n.prepareAuthParameters(ctx, &prmSearch.PrmAuth, owner) - prmSearch.Filters.AddTypeFilter(object.MatchStringEqual, object.TypeRegular) - prmSearch.Filters.AddFilter(attributeTagsMetaObject, "", object.MatchNotPresent) + filters.AddFilter(object.FilterCreationEpoch, "0", object.MatchNumGT) + filters.AddTypeFilter(object.MatchStringEqual, object.TypeRegular) if len(prefix) > 0 { - prmSearch.Filters.AddFilter(object.AttributeFilePath, prefix, object.MatchCommonPrefix) + filters.AddFilter(object.AttributeFilePath, prefix, object.MatchCommonPrefix) } + filters.AddFilter(attributeTagsMetaObject, "", object.MatchNotPresent) + if onlyUnversioned { - prmSearch.Filters.AddFilter(attrS3VersioningState, data.VersioningUnversioned, object.MatchNotPresent) + filters.AddFilter(attrS3VersioningState, data.VersioningUnversioned, object.MatchNotPresent) } - return n.searchObjects(ctx, bkt, prmSearch) + searchResultItems, err := n.neoFS.SearchObjectsV2(ctx, bkt.CID, filters, returningAttributes, opts) + if err != nil { + if errors.Is(err, apistatus.ErrObjectAccessDenied) { + return nil, s3errors.GetAPIError(s3errors.ErrAccessDenied) + } + + return nil, fmt.Errorf("search objects: %w", err) + } + + if len(searchResultItems) == 0 { + return nil, ErrNodeNotFound + } + + var searchResults = make([]prefixSearchResult, 0, len(searchResultItems)) + + for _, item := range searchResultItems { + if len(item.Attributes) != len(returningAttributes) { + return nil, fmt.Errorf("invalid attribute count returned, expected %d, got %d", len(returningAttributes), len(item.Attributes)) + } + + var psr = prefixSearchResult{ + ID: item.ID, + FilePath: item.Attributes[1], + } + + if item.Attributes[0] != "" { + psr.CreationEpoch, err = strconv.ParseUint(item.Attributes[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid creation epoch %s: %w", item.Attributes[0], err) + } + } + + if item.Attributes[2] != "" { + psr.CreationTimestamp, err = strconv.ParseInt(item.Attributes[2], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid creation timestamp %s: %w", item.Attributes[2], err) + } + } + + psr.IsDeleteMarker = item.Attributes[3] != "" + + searchResults = append(searchResults, psr) + } + + sortFunc := func(a, b prefixSearchResult) int { + if c := cmp.Compare(b.CreationEpoch, a.CreationEpoch); c != 0 { // reverse order. + return c + } + + if c := cmp.Compare(b.CreationTimestamp, a.CreationTimestamp); c != 0 { // reverse order. + return c + } + + // It is a temporary decision. We can't figure out what object was first and what the second right now. + return bytes.Compare(b.ID[:], a.ID[:]) // reverse order. + } + + slices.SortFunc(searchResults, sortFunc) + + return searchResults, nil } func (n *layer) searchObjects(ctx context.Context, bkt *data.BucketInfo, prmSearch PrmObjectSearch) ([]*object.Object, error) { @@ -548,8 +628,8 @@ func sortObjectsFuncByFilePath(a, b *object.Object) int { return cmp.Compare(aPath, bPath) } -func (n *layer) searchLatestVersionsByPrefix(ctx context.Context, bkt *data.BucketInfo, owner user.ID, prefix string, onlyUnversioned bool) ([]*object.Object, error) { - heads, err := n.searchAllVersionsInNeoFSByPrefix(ctx, bkt, owner, prefix, onlyUnversioned) +func (n *layer) searchLatestVersionsByPrefix(ctx context.Context, bkt *data.BucketInfo, owner user.ID, prefix string, onlyUnversioned bool) ([]prefixSearchResult, error) { + searchResults, err := n.searchAllVersionsInNeoFSByPrefix(ctx, bkt, owner, prefix, onlyUnversioned) if err != nil { if errors.Is(err, apistatus.ErrObjectAccessDenied) { return nil, s3errors.GetAPIError(s3errors.ErrAccessDenied) @@ -558,20 +638,12 @@ func (n *layer) searchLatestVersionsByPrefix(ctx context.Context, bkt *data.Buck return nil, fmt.Errorf("get all versions by prefix: %w", err) } - var uniq = make(map[string]*object.Object, len(heads)) - - for _, head := range heads { - var filePath string - for _, attr := range head.Attributes() { - if attr.Key() == object.AttributeFilePath { - filePath = attr.Value() - break - } - } + var uniq = make(map[string]prefixSearchResult, len(searchResults)) + for _, result := range searchResults { // take only first object, because it is the freshest one. - if _, ok := uniq[filePath]; !ok { - uniq[filePath] = head + if _, ok := uniq[result.FilePath]; !ok { + uniq[result.FilePath] = result } } @@ -762,7 +834,7 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis return &result, nil } -func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams) (objects []*data.ObjectInfo, next *data.ObjectInfo, err error) { +func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams) (objects []data.ObjectListResponseContent, next *data.ObjectListResponseContent, err error) { if p.MaxKeys == 0 { return nil, nil, nil } @@ -771,10 +843,10 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams) cacheKey := cache.CreateObjectsListCacheKey(p.Bucket.CID, p.Prefix, true) nodeVersions := n.cache.GetList(owner, cacheKey) - var rawHeads []*object.Object + var latestVersions []prefixSearchResult if nodeVersions == nil { - rawHeads, err = n.searchLatestVersionsByPrefix(ctx, p.Bucket, p.Bucket.Owner, p.Prefix, false) + latestVersions, err = n.searchLatestVersionsByPrefix(ctx, p.Bucket, p.Bucket.Owner, p.Prefix, false) if err != nil { if errors.Is(err, ErrNodeNotFound) { return nil, nil, nil @@ -784,32 +856,71 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p allObjectParams) } } - if len(rawHeads) == 0 { + if len(latestVersions) == 0 { return nil, nil, nil } - slices.SortFunc(rawHeads, sortObjectsFuncByFilePath) + sortFunc := func(a, b prefixSearchResult) int { + return cmp.Compare(a.FilePath, b.FilePath) + } + + slices.SortFunc(latestVersions, sortFunc) existed := make(map[string]struct{}, len(nodeVersions)) // to squash the same directories - for _, head := range rawHeads { - if shouldSkip(head, p, existed) { + for _, ver := range latestVersions { + if shouldSkip(ver, p, existed) { continue } - var oi *data.ObjectInfo - if oi = tryDirectoryFromObject(p.Bucket, p.Prefix, p.Delimiter, *head); oi == nil { - oi = objectInfoFromMeta(p.Bucket, head) + var oi *data.ObjectListResponseContent + + if oi = tryDirectoryFromObject(p.Prefix, p.Delimiter, ver.FilePath); oi == nil { + head, err := n.objectHead(ctx, p.Bucket, ver.ID) + if err != nil { + if isErrObjectAlreadyRemoved(err) { + continue + } + + return nil, nil, fmt.Errorf("head details: %w", err) + } + + var ( + attributeDecryptedSize int64 + ) + + for _, attr := range head.Attributes() { + if attr.Key() == AttributeDecryptedSize { + if attributeDecryptedSize, err = strconv.ParseInt(attr.Value(), 10, 64); err != nil { + return nil, nil, fmt.Errorf("parse decrypted size %s: %w", attr.Value(), err) + } + + break + } + } + + payloadChecksum, _ := head.PayloadChecksum() + + oi = &data.ObjectListResponseContent{ + ID: ver.ID, + Owner: p.Bucket.Owner, + Created: time.Unix(ver.CreationTimestamp, 0), + Name: ver.FilePath, + + DecryptedSize: attributeDecryptedSize, + Size: int64(head.PayloadSize()), + HashSum: hex.EncodeToString(payloadChecksum.Value()), + } } - objects = append(objects, oi) + objects = append(objects, *oi) } - slices.SortFunc(objects, func(a, b *data.ObjectInfo) int { + slices.SortFunc(objects, func(a, b data.ObjectListResponseContent) int { return cmp.Compare(a.Name, b.Name) }) if len(objects) > p.MaxKeys { - next = objects[p.MaxKeys] + next = &objects[p.MaxKeys] objects = objects[:p.MaxKeys] } @@ -906,13 +1017,13 @@ func IsSystemHeader(key string) bool { return ok || strings.HasPrefix(key, api.NeoFSSystemMetadataPrefix) } -func shouldSkip(head *object.Object, p allObjectParams, existed map[string]struct{}) bool { - if isDeleteMarkerObject(*head) { +func shouldSkip(result prefixSearchResult, p allObjectParams, existed map[string]struct{}) bool { + if result.IsDeleteMarker { return true } - filePath := filePathFromAttributes(*head) - if dirName := tryDirectoryName(filePath, p.Prefix, p.Delimiter); len(dirName) != 0 { + filePath := result.FilePath + if dirName := tryDirectoryName(result.FilePath, p.Prefix, p.Delimiter); len(dirName) != 0 { filePath = dirName } if _, ok := existed[filePath]; ok { @@ -925,7 +1036,7 @@ func shouldSkip(head *object.Object, p allObjectParams, existed map[string]struc if p.ContinuationToken != "" { if _, ok := existed[continuationToken]; !ok { - if p.ContinuationToken != head.GetID().EncodeToString() { + if p.ContinuationToken != result.ID.EncodeToString() { return true } existed[continuationToken] = struct{}{} @@ -936,7 +1047,7 @@ func shouldSkip(head *object.Object, p allObjectParams, existed map[string]struc return false } -func triageObjects(allObjects []*data.ObjectInfo) (prefixes []string, objects []*data.ObjectInfo) { +func triageObjects(allObjects []data.ObjectListResponseContent) (prefixes []string, objects []data.ObjectListResponseContent) { for _, ov := range allObjects { if ov.IsDir { prefixes = append(prefixes, ov.Name) @@ -998,21 +1109,15 @@ func tryDirectory(bktInfo *data.BucketInfo, node *data.NodeVersion, prefix, deli } } -func tryDirectoryFromObject(bktInfo *data.BucketInfo, prefix, delimiter string, head object.Object) *data.ObjectInfo { - nv := convert(head) - - dirName := tryDirectoryName(nv.FilePath, prefix, delimiter) +func tryDirectoryFromObject(prefix, delimiter string, filePath string) *data.ObjectListResponseContent { + dirName := tryDirectoryName(filePath, prefix, delimiter) if len(dirName) == 0 { return nil } - return &data.ObjectInfo{ - ID: nv.OID, // to use it as continuation token - CID: bktInfo.CID, - IsDir: true, - IsDeleteMarker: nv.IsDeleteMarker(), - Bucket: bktInfo.Name, - Name: dirName, + return &data.ObjectListResponseContent{ + IsDir: true, + Name: dirName, } } diff --git a/api/layer/util.go b/api/layer/util.go index 1209d6b9..4cb13b4a 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -20,7 +20,7 @@ type ( // ListObjectsInfo contains common fields of data for ListObjectsV1 and ListObjectsV2. ListObjectsInfo struct { Prefixes []string - Objects []*data.ObjectInfo + Objects []data.ObjectListResponseContent IsTruncated bool } diff --git a/internal/neofs/neofs.go b/internal/neofs/neofs.go index 30dcde42..a7897c49 100644 --- a/internal/neofs/neofs.go +++ b/internal/neofs/neofs.go @@ -752,3 +752,27 @@ func (x *NeoFS) SearchObjects(ctx context.Context, prm layer.PrmObjectSearch) ([ return oids, nil } + +func (x *NeoFS) SearchObjectsV2(ctx context.Context, cid cid.ID, filters object.SearchFilters, attributes []string, opts client.SearchObjectsOptions) ([]client.SearchResultItem, error) { + var ( + resultItems []client.SearchResultItem + items []client.SearchResultItem + cursor string + err error + ) + + for { + items, cursor, err = x.pool.SearchObjects(ctx, cid, filters, attributes, cursor, x.signer(ctx), opts) + if err != nil { + return nil, fmt.Errorf("search objects via connection pool: %w", err) + } + + resultItems = append(resultItems, items...) + + if cursor == "" { + break + } + } + + return resultItems, nil +}