Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New search #1078

Closed
wants to merge 5 commits into from
Closed

New search #1078

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
78ae1ed
multipart: Drop set-only struct fields
cthulhu-rider Feb 11, 2025
814f45b
multipart: Inline a once called method
cthulhu-rider Feb 11, 2025
d1c8d95
api/layer: Drop unused comment
cthulhu-rider Feb 11, 2025
e2bbda3
WIP SearchV2
cthulhu-rider Feb 11, 2025
9f08b34
WIP SearchV2
smallhive Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions api/data/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,18 @@ type (
Headers map[string]string
}

ObjectListResponseContent struct {
IsDir bool

ID oid.ID
DecryptedSize int64
Size int64
Owner user.ID
HashSum string
Created time.Time
Name string
}

// NotificationInfo store info to send s3 notification.
NotificationInfo struct {
Name string
Expand Down
13 changes: 4 additions & 9 deletions api/handler/object_list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,11 +207,11 @@ func fillPrefixes(src []string, encode string) []CommonPrefix {
return dst
}

func fillContentsWithOwner(src []*data.ObjectInfo, encode string) ([]Object, error) {
func fillContentsWithOwner(src []data.ObjectListResponseContent, encode string) ([]Object, error) {
return fillContents(src, encode, true)
}

func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) ([]Object, error) {
func fillContents(src []data.ObjectListResponseContent, encode string, fetchOwner bool) ([]Object, error) {
var dst []Object
for _, obj := range src {
res := Object{
Expand All @@ -221,13 +221,8 @@ func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) ([]Obj
ETag: obj.HashSum,
}

if size, ok := obj.Headers[layer.AttributeDecryptedSize]; ok {
sz, err := strconv.ParseInt(size, 10, 64)
if err != nil {
return nil, fmt.Errorf("parse decrypted size %s: %w", size, err)
}

res.Size = sz
if obj.DecryptedSize > 0 {
res.Size = obj.DecryptedSize
}

if fetchOwner {
Expand Down
9 changes: 8 additions & 1 deletion api/layer/layer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
"github.com/nspcc-dev/neofs-s3-gw/api/s3errors"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/nspcc-dev/neofs-sdk-go/eacl"
Expand Down Expand Up @@ -350,11 +351,17 @@ func (n *layer) OwnerPublicKey(ctx context.Context) (*keys.PublicKey, error) {
}

func (n *layer) prepareAuthParameters(ctx context.Context, prm *PrmAuth, bktOwner user.ID) {
// TODO: drop method
prm.BearerToken = bearerTokenFromContext(ctx, bktOwner)
}

func bearerTokenFromContext(ctx context.Context, bktOwner user.ID) *bearer.Token {
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
if bktOwner.Equals(bd.Gate.BearerToken.ResolveIssuer()) {
prm.BearerToken = bd.Gate.BearerToken
return bd.Gate.BearerToken
}
}
return nil
}

// GetBucketInfo returns bucket info by name.
Expand Down
75 changes: 17 additions & 58 deletions api/layer/multipart_upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/layer/encryption"
"github.com/nspcc-dev/neofs-s3-gw/api/s3errors"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
Expand Down Expand Up @@ -146,13 +147,6 @@ type (
Created time.Time
}

slotAttributes struct {
PartNumber int64
// in nanoseconds
CreatedAt int64
FilePath string
}

uploadPartAsSlotParams struct {
bktInfo *data.BucketInfo
multipartInfo *data.MultipartInfo
Expand Down Expand Up @@ -1421,70 +1415,35 @@ func (n *layer) uploadPartAsSlot(ctx context.Context, params uploadPartAsSlotPar
return &objInfo, nil
}

func (n *layer) getSlotAttributes(obj object.Object) (*slotAttributes, error) {
var (
attributes slotAttributes
err error
)

for _, attr := range obj.Attributes() {
switch attr.Key() {
case headerS3MultipartNumber:
attributes.PartNumber, err = strconv.ParseInt(attr.Value(), 10, 64)
case headerS3MultipartCreated:
attributes.CreatedAt, err = strconv.ParseInt(attr.Value(), 10, 64)
case object.AttributeFilePath:
attributes.FilePath = attr.Value()
default:
continue
}

if err != nil {
return nil, fmt.Errorf("parse header: %w", err)
}
}

return &attributes, nil
}

func (n *layer) getFirstArbitraryPart(ctx context.Context, uploadID string, bucketInfo *data.BucketInfo) (int64, error) {
var filters object.SearchFilters
filters.AddFilter(headerS3MultipartNumber, "0", object.MatchNumGE) // only primary attribute is sorted
filters.AddFilter(headerS3MultipartUpload, uploadID, object.MatchStringEqual)

var prmSearch = PrmObjectSearch{
Container: bucketInfo.CID,
Filters: filters,
var opts client.SearchObjectsOptions
opts.SetCount(1)
if bt := bearerTokenFromContext(ctx, bucketInfo.Owner); bt != nil {
opts.WithBearerToken(*bt)
}

n.prepareAuthParameters(ctx, &prmSearch.PrmAuth, bucketInfo.Owner)

oids, err := n.neoFS.SearchObjects(ctx, prmSearch)
res, err := n.neoFS.SearchObjectsV2(ctx, bucketInfo.CID, filters, []string{
headerS3MultipartNumber,
headerS3MultipartCreated,
}, opts)
if err != nil {
return 0, fmt.Errorf("search objects: %w", err)
}

if len(oids) == 0 {
if len(res) == 0 {
return 0, nil
}

var partNumber int64

for _, id := range oids {
head, err := n.objectHead(ctx, bucketInfo, id)
if err != nil {
return 0, fmt.Errorf("object head: %w", err)
}

attributes, err := n.getSlotAttributes(*head)
if err != nil {
return 0, fmt.Errorf("get slot attributes: %w", err)
}

if partNumber == 0 {
partNumber = attributes.PartNumber
} else {
partNumber = min(partNumber, attributes.PartNumber)
}
partNumber, err := strconv.ParseInt(res[0].Attributes[0], 10, 64)
if err != nil {
return 0, fmt.Errorf("parse header: %w", err)
}
if _, err = strconv.ParseInt(res[0].Attributes[1], 10, 64); err != nil {
return 0, fmt.Errorf("parse header: %w", err)
}

return partNumber, nil
Expand Down
3 changes: 3 additions & 0 deletions api/layer/neofs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-sdk-go/bearer"
"github.com/nspcc-dev/neofs-sdk-go/client"
"github.com/nspcc-dev/neofs-sdk-go/container"
"github.com/nspcc-dev/neofs-sdk-go/container/acl"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
Expand Down Expand Up @@ -278,4 +279,6 @@ type NeoFS interface {

// SearchObjects searches objects with corresponding filters.
SearchObjects(ctx context.Context, prm PrmObjectSearch) ([]oid.ID, error)

SearchObjectsV2(context.Context, cid.ID, object.SearchFilters, []string, client.SearchObjectsOptions) ([]client.SearchResultItem, error)
}
Loading
Loading