Switch to v2 object split scheme (#957)

Author: roman-khimov

.github/workflows/s3-tests.yml

@@ -53,47 +53,47 @@ jobs:
         uses: dsaltares/fetch-gh-release-asset@1.1.1
         with:
           repo: 'nspcc-dev/neofs-node'
-          version: 'tags/v0.41.1'
+          version: 'tags/v0.42.0'
           file: 'neofs-cli-linux-amd64'
           target: 's3-tests/neofs-cli'
 
       - name: Download latest stable neofs-adm
         uses: dsaltares/fetch-gh-release-asset@1.1.1
         with:
           repo: 'nspcc-dev/neofs-node'
-          version: 'tags/v0.41.1'
+          version: 'tags/v0.42.0'
           file: 'neofs-adm-linux-amd64'
           target: 's3-tests/neofs-adm'
 
       - name: Download latest stable neofs-ir
         uses: dsaltares/fetch-gh-release-asset@1.1.1
         with:
           repo: 'nspcc-dev/neofs-node'
-          version: 'tags/v0.41.1'
+          version: 'tags/v0.42.0'
           file: 'neofs-ir-linux-amd64'
           target: 's3-tests/neofs-ir'
 
       - name: Download latest stable neofs-lens
         uses: dsaltares/fetch-gh-release-asset@1.1.1
         with:
           repo: 'nspcc-dev/neofs-node'
-          version: 'tags/v0.41.1'
+          version: 'tags/v0.42.0'
           file: 'neofs-lens-linux-amd64'
           target: 's3-tests/neofs-lens'
 
       - name: Download latest stable neofs-node
         uses: dsaltares/fetch-gh-release-asset@1.1.1
         with:
           repo: 'nspcc-dev/neofs-node'
-          version: 'tags/v0.41.1'
+          version: 'tags/v0.42.0'
           file: 'neofs-node-linux-amd64'
           target: 's3-tests/neofs-node'
 
       - name: Download latest stable neo-go
         uses: dsaltares/fetch-gh-release-asset@1.1.1
         with:
           repo: 'nspcc-dev/neo-go'
-          version: 'tags/v0.105.1'
+          version: 'tags/v0.106.0'
           file: 'neo-go-linux-amd64'
           target: 's3-tests/neo-go'
 

api/data/info.go

@@ -40,14 +40,15 @@ type (
 		IsDir          bool
 		IsDeleteMarker bool
 
-		Bucket      string
-		Name        string
-		Size        int64
-		ContentType string
-		Created     time.Time
-		HashSum     string
-		Owner       user.ID
-		Headers     map[string]string
+		Bucket         string
+		Name           string
+		Size           int64
+		ContentType    string
+		Created        time.Time
+		HashSum        string
+		Owner          user.ID
+		OwnerPublicKey keys.PublicKey
+		Headers        map[string]string
 	}
 
 	// NotificationInfo store info to send s3 notification.

api/data/tree.go

@@ -1,9 +1,12 @@
 package data
 
 import (
+	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
+	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"github.com/nspcc-dev/neofs-sdk-go/user"
@@ -72,10 +75,42 @@ type MultipartInfo struct {
 	Key          string
 	UploadID     string
 	Owner        user.ID
+	OwnerPubKey  keys.PublicKey
 	Created      time.Time
 	Meta         map[string]string
 	CopiesNumber uint32
-	SplitID      string
+}
+
+// LinkObjectPayload contains part info of the complex object.
+// This data will be used for linking object construction.
+type LinkObjectPayload struct {
+	OID  oid.ID
+	Size uint32
+}
+
+// Marshal converts LinkObjectPayload to string.
+func (e *LinkObjectPayload) Marshal() string {
+	return fmt.Sprintf("%s:%d", e.OID.String(), e.Size)
+}
+
+// Unmarshal converts string to LinkObjectPayload.
+func (e *LinkObjectPayload) Unmarshal(value string) error {
+	parts := strings.Split(value, ":")
+	if len(parts) != 2 {
+		return fmt.Errorf("invalid format: %s", value)
+	}
+
+	if err := e.OID.DecodeString(parts[0]); err != nil {
+		return fmt.Errorf("invalid id: %w", err)
+	}
+
+	size, err := strconv.ParseUint(parts[1], 10, 32)
+	if err != nil {
+		return fmt.Errorf("invalid size: %w", err)
+	}
+
+	e.Size = uint32(size)
+	return nil
 }
 
 // PartInfo is upload information about part.
@@ -95,8 +130,8 @@ type PartInfo struct {
 	MultipartHash []byte
 	// HomoHash contains internal state of the [hash.Hash] to calculate whole object homomorphic payload hash.
 	HomoHash []byte
-	// Elements contain [oid.ID] object list for the current part.
-	Elements []oid.ID
+	// Elements contain [oid.ID] and size for each element for the current part.
+	Elements []LinkObjectPayload
 }
 
 // ToHeaderString form short part representation to use in S3-Completed-Parts header.

api/handler/multipart_upload.go

@@ -7,7 +7,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/nspcc-dev/neofs-s3-gw/api"
 	"github.com/nspcc-dev/neofs-s3-gw/api/data"
 	"github.com/nspcc-dev/neofs-s3-gw/api/layer"
@@ -101,17 +100,14 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	uploadID := uuid.New()
 	additional := []zap.Field{
-		zap.String("uploadID", uploadID.String()),
 		zap.String("Key", reqInfo.ObjectName),
 	}
 
 	p := &layer.CreateMultipartParams{
 		Info: &layer.UploadInfoParams{
-			UploadID: uploadID.String(),
-			Bkt:      bktInfo,
-			Key:      reqInfo.ObjectName,
+			Bkt: bktInfo,
+			Key: reqInfo.ObjectName,
 		},
 		Data: &layer.UploadData{},
 	}
@@ -154,7 +150,8 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if err = h.obj.CreateMultipartUpload(r.Context(), p); err != nil {
+	uploadID, err := h.obj.CreateMultipartUpload(r.Context(), p)
+	if err != nil {
 		h.logAndSendError(w, "could create multipart upload", reqInfo, err, additional...)
 		return
 	}
@@ -166,9 +163,10 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
 	resp := InitiateMultipartUploadResponse{
 		Bucket:   reqInfo.BucketName,
 		Key:      reqInfo.ObjectName,
-		UploadID: uploadID.String(),
+		UploadID: uploadID,
 	}
 
+	additional = append(additional, zap.String("uploadID", uploadID))
 	if err = api.EncodeToResponse(w, resp); err != nil {
 		h.logAndSendError(w, "could not encode InitiateMultipartUploadResponse to response", reqInfo, err, additional...)
 		return
@@ -648,12 +646,12 @@ func encodeListMultipartUploadsToResponse(info *layer.ListMultipartUploadsInfo,
 		m := MultipartUpload{
 			Initiated: u.Created.UTC().Format(time.RFC3339),
 			Initiator: Initiator{
-				ID:          u.Owner.String(),
+				ID:          u.OwnerPubKey.StringCompressed(),
 				DisplayName: u.Owner.String(),
 			},
 			Key: u.Key,
 			Owner: Owner{
-				ID:          u.Owner.String(),
+				ID:          u.OwnerPubKey.StringCompressed(),
 				DisplayName: u.Owner.String(),
 			},
 			UploadID: u.UploadID,
@@ -671,15 +669,15 @@ func encodeListPartsToResponse(info *layer.ListPartsInfo, params *layer.ListPart
 		XMLName: xml.Name{},
 		Bucket:  params.Info.Bkt.Name,
 		Initiator: Initiator{
-			ID:          info.Owner.String(),
+			ID:          info.OwnerPubKey.StringCompressed(),
 			DisplayName: info.Owner.String(),
 		},
 		IsTruncated:          info.IsTruncated,
 		Key:                  params.Info.Key,
 		MaxParts:             params.MaxParts,
 		NextPartNumberMarker: info.NextPartNumberMarker,
 		Owner: Owner{
-			ID:          info.Owner.String(),
+			ID:          info.OwnerPubKey.StringCompressed(),
 			DisplayName: info.Owner.String(),
 		},
 		PartNumberMarker: params.PartNumberMarker,

api/handler/object_list.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"fmt"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -33,12 +34,18 @@ func (h *handler) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = api.EncodeToResponse(w, encodeV1(params, list)); err != nil {
+	encoded, err := encodeV1(params, list)
+	if err != nil {
+		h.logAndSendError(w, "encode V1", reqInfo, err)
+		return
+	}
+
+	if err = api.EncodeToResponse(w, encoded); err != nil {
 		h.logAndSendError(w, "something went wrong", reqInfo, err)
 	}
 }
 
-func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *ListObjectsV1Response {
+func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) (*ListObjectsV1Response, error) {
 	res := &ListObjectsV1Response{
 		Name:         p.BktInfo.Name,
 		EncodingType: p.Encode,
@@ -52,9 +59,14 @@ func encodeV1(p *layer.ListObjectsParamsV1, list *layer.ListObjectsInfoV1) *List
 
 	res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
 
-	res.Contents = fillContentsWithOwner(list.Objects, p.Encode)
+	content, err := fillContentsWithOwner(list.Objects, p.Encode)
+	if err != nil {
+		return nil, fmt.Errorf("fill contents with owner: %w", err)
+	}
+
+	res.Contents = content
 
-	return res
+	return res, nil
 }
 
 // ListObjectsV2Handler handles objects listing requests for API version 2.
@@ -77,12 +89,18 @@ func (h *handler) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = api.EncodeToResponse(w, encodeV2(params, list)); err != nil {
+	encoded, err := encodeV2(params, list)
+	if err != nil {
+		h.logAndSendError(w, "encode V2", reqInfo, err)
+		return
+	}
+
+	if err = api.EncodeToResponse(w, encoded); err != nil {
 		h.logAndSendError(w, "something went wrong", reqInfo, err)
 	}
 }
 
-func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *ListObjectsV2Response {
+func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) (*ListObjectsV2Response, error) {
 	res := &ListObjectsV2Response{
 		Name:                  p.BktInfo.Name,
 		EncodingType:          p.Encode,
@@ -98,9 +116,14 @@ func encodeV2(p *layer.ListObjectsParamsV2, list *layer.ListObjectsInfoV2) *List
 
 	res.CommonPrefixes = fillPrefixes(list.Prefixes, p.Encode)
 
-	res.Contents = fillContents(list.Objects, p.Encode, p.FetchOwner)
+	content, err := fillContents(list.Objects, p.Encode, p.FetchOwner)
+	if err != nil {
+		return nil, fmt.Errorf("fill content: %w", err)
+	}
 
-	return res
+	res.Contents = content
+
+	return res, nil
 }
 
 func parseListObjectsArgsV1(reqInfo *api.ReqInfo) (*layer.ListObjectsParamsV1, error) {
@@ -184,11 +207,11 @@ func fillPrefixes(src []string, encode string) []CommonPrefix {
 	return dst
 }
 
-func fillContentsWithOwner(src []*data.ObjectInfo, encode string) []Object {
+func fillContentsWithOwner(src []*data.ObjectInfo, encode string) ([]Object, error) {
 	return fillContents(src, encode, true)
 }
 
-func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Object {
+func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) ([]Object, error) {
 	var dst []Object
 	for _, obj := range src {
 		res := Object{
@@ -198,6 +221,15 @@ func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Obje
 			ETag:         obj.HashSum,
 		}
 
+		if size, ok := obj.Headers[layer.AttributeDecryptedSize]; ok {
+			sz, err := strconv.ParseInt(size, 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("parse decrypted size %s: %w", size, err)
+			}
+
+			res.Size = sz
+		}
+
 		if fetchOwner {
 			res.Owner = &Owner{
 				ID:          obj.Owner.String(),
@@ -207,7 +239,7 @@ func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Obje
 
 		dst = append(dst, res)
 	}
-	return dst
+	return dst, nil
 }
 
 func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
@@ -277,7 +309,7 @@ func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, buck
 			Key:          ver.ObjectInfo.Name,
 			LastModified: ver.ObjectInfo.Created.UTC().Format(time.RFC3339),
 			Owner: Owner{
-				ID:          ver.ObjectInfo.Owner.String(),
+				ID:          ver.ObjectInfo.OwnerPublicKey.StringCompressed(),
 				DisplayName: ver.ObjectInfo.Owner.String(),
 			},
 			Size:      ver.ObjectInfo.Size,

api/layer/layer.go

@@ -226,7 +226,7 @@ type (
 
 		DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject
 
-		CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) error
+		CreateMultipartUpload(ctx context.Context, p *CreateMultipartParams) (string, error)
 		CompleteMultipartUpload(ctx context.Context, p *CompleteMultipartParams) (*UploadData, *data.ExtendedObjectInfo, error)
 		UploadPart(ctx context.Context, p *UploadPartParams) (string, error)
 		UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.ObjectInfo, error)