feat(gateway): support for order=, dups= parameters from IPIP-412 (#370)

Co-authored-by: Marcin Rataj <lidel@lidel.org>

Author: hacdias

CHANGELOG.md

@@ -16,13 +16,30 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+* ✨ The gateway now supports the optional `order` and `dups` CAR parameters
+  from [IPIP-412](https://github.com/ipfs/specs/pull/412).
+  * The `BlocksBackend` only implements `order=dfs` (Depth-First Search)
+    ordering, which was already the default behavior.
+  * If a request specifies no `dups`, response with `dups=n` is returned, which
+    was already the default behavior.
+  * If a request explicitly specifies a CAR `order` other than `dfs`, it will
+    result in an error.
+  * The only change to the default behavior on CAR responses is that we follow
+    IPIP-412 and make `order=dfs;dups=n` explicit in the returned
+    `Content-Type` HTTP header.
+
 ### Changed
 
-* 🛠 The `ipns` package has been refactored. You should no longer use the direct Protobuf
-  version of the IPNS Record. Instead, we have a shiny new `ipns.Record` type that wraps
-  all the required functionality to work the best as possible with IPNS v2 Records. Please
-  check the [documentation](https://pkg.go.dev/github.com/ipfs/boxo/ipns) for more information,
-  and follow [ipfs/specs#376](https://github.com/ipfs/specs/issues/376) for related IPIP.
+* 🛠 The `ipns` package has been refactored.
+  * You should no longer use the direct Protobuf version of the IPNS Record.
+    Instead, we have a shiny new `ipns.Record` type that wraps all the required
+    functionality to work the best as possible with IPNS v2 Records. Please
+    check the [documentation](https://pkg.go.dev/github.com/ipfs/boxo/ipns) for
+    more information, and follow
+    [ipfs/specs#376](https://github.com/ipfs/specs/issues/376) for related
+    IPIP.
+  * There is no change to IPNS Records produced by `boxo/ipns`, it still
+    produces both V1 and V2 signatures by default, it is still backward-compatible.
 
 ### Removed
 

gateway/blocks_backend.go

@@ -245,7 +245,12 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car
 
 	r, w := io.Pipe()
 	go func() {
-		cw, err := storage.NewWritable(w, []cid.Cid{pathMetadata.LastSegment.Cid()}, car.WriteAsCarV1(true))
+		cw, err := storage.NewWritable(
+			w,
+			[]cid.Cid{pathMetadata.LastSegment.Cid()},
+			car.WriteAsCarV1(true),
+			car.AllowDuplicatePuts(params.Duplicates.Bool()),
+		)
 		if err != nil {
 			// io.PipeWriter.CloseWithError always returns nil.
 			_ = w.CloseWithError(err)
@@ -312,7 +317,7 @@ func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarP
 				Ctx:                            ctx,
 				LinkSystem:                     *lsys,
 				LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser,
-				LinkVisitOnlyOnce:              true, // This is safe for the "all" selector
+				LinkVisitOnlyOnce:              !params.Duplicates.Bool(),
 			},
 		}
 

gateway/gateway.go

@@ -121,8 +121,10 @@ func (i ImmutablePath) IsValid() error {
 var _ path.Path = (*ImmutablePath)(nil)
 
 type CarParams struct {
-	Range *DagByteRange
-	Scope DagScope
+	Range      *DagByteRange
+	Scope      DagScope
+	Order      DagOrder
+	Duplicates DuplicateBlocksPolicy
 }
 
 // DagByteRange describes a range request within a UnixFS file. "From" and
@@ -189,6 +191,50 @@ const (
 	DagScopeBlock  DagScope = "block"
 )
 
+type DagOrder string
+
+const (
+	DagOrderUnspecified DagOrder = ""
+	DagOrderUnknown     DagOrder = "unk"
+	DagOrderDFS         DagOrder = "dfs"
+)
+
+// DuplicateBlocksPolicy represents the content type parameter 'dups' (IPIP-412)
+type DuplicateBlocksPolicy int
+
+const (
+	DuplicateBlocksUnspecified DuplicateBlocksPolicy = iota // 0 - implicit default
+	DuplicateBlocksIncluded                                 // 1 - explicitly include duplicates
+	DuplicateBlocksExcluded                                 // 2 - explicitly NOT include duplicates
+)
+
+// NewDuplicateBlocksPolicy returns DuplicateBlocksPolicy based on the content type parameter 'dups' (IPIP-412)
+func NewDuplicateBlocksPolicy(dupsValue string) DuplicateBlocksPolicy {
+	switch dupsValue {
+	case "y":
+		return DuplicateBlocksIncluded
+	case "n":
+		return DuplicateBlocksExcluded
+	}
+	return DuplicateBlocksUnspecified
+}
+
+func (d DuplicateBlocksPolicy) Bool() bool {
+	// duplicates should be returned only when explicitly requested,
+	// so any other state than DuplicateBlocksIncluded should return false
+	return d == DuplicateBlocksIncluded
+}
+
+func (d DuplicateBlocksPolicy) String() string {
+	switch d {
+	case DuplicateBlocksIncluded:
+		return "y"
+	case DuplicateBlocksExcluded:
+		return "n"
+	}
+	return ""
+}
+
 type ContentPathMetadata struct {
 	PathSegmentRoots []cid.Cid
 	LastSegment      path.Resolved

gateway/handler.go

@@ -637,28 +637,9 @@ const (
 
 // return explicit response format if specified in request as query parameter or via Accept HTTP header
 func customResponseFormat(r *http.Request) (mediaType string, params map[string]string, err error) {
-	// Translate query param to a content type, if present.
-	if formatParam := r.URL.Query().Get("format"); formatParam != "" {
-		switch formatParam {
-		case "raw":
-			return rawResponseFormat, nil, nil
-		case "car":
-			return carResponseFormat, nil, nil
-		case "tar":
-			return tarResponseFormat, nil, nil
-		case "json":
-			return jsonResponseFormat, nil, nil
-		case "cbor":
-			return cborResponseFormat, nil, nil
-		case "dag-json":
-			return dagJsonResponseFormat, nil, nil
-		case "dag-cbor":
-			return dagCborResponseFormat, nil, nil
-		case "ipns-record":
-			return ipnsRecordResponseFormat, nil, nil
-		}
-	}
-
+	// First, inspect Accept header, as it may not only include content type, but also optional parameters.
+	// such as CAR version or additional ones from IPIP-412.
+	//
 	// Browsers and other user agents will send Accept header with generic types like:
 	// Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8
 	// We only care about explicit, vendor-specific content-types and respond to the first match (in order).
@@ -681,6 +662,28 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
 		}
 	}
 
+	// If no Accept header, translate query param to a content type, if present.
+	if formatParam := r.URL.Query().Get("format"); formatParam != "" {
+		switch formatParam {
+		case "raw":
+			return rawResponseFormat, nil, nil
+		case "car":
+			return carResponseFormat, nil, nil
+		case "tar":
+			return tarResponseFormat, nil, nil
+		case "json":
+			return jsonResponseFormat, nil, nil
+		case "cbor":
+			return cborResponseFormat, nil, nil
+		case "dag-json":
+			return dagJsonResponseFormat, nil, nil
+		case "dag-cbor":
+			return dagCborResponseFormat, nil, nil
+		case "ipns-record":
+			return ipnsRecordResponseFormat, nil, nil
+		}
+	}
+
 	// If none of special-cased content types is found, return empty string
 	// to indicate default, implicit UnixFS response should be prepared
 	return "", nil, nil

gateway/handler_car.go

@@ -30,16 +30,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	switch rq.responseParams["version"] {
-	case "": // noop, client does not care about version
-	case "1": // noop, we support this
-	default:
-		err := fmt.Errorf("unsupported CAR version: only version=1 is supported")
-		i.webError(w, r, err, http.StatusBadRequest)
-		return false
-	}
-
-	params, err := getCarParams(r)
+	params, err := buildCarParams(r, rq.responseParams)
 	if err != nil {
 		i.webError(w, r, err, http.StatusBadRequest)
 		return false
@@ -90,7 +81,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
 	// sub-DAGs and IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769
 	w.Header().Set("Accept-Ranges", "none")
 
-	w.Header().Set("Content-Type", carResponseFormat+"; version=1")
+	w.Header().Set("Content-Type", buildContentTypeFromCarParams(params))
 	w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
 
 	_, copyErr := io.Copy(w, carFile)
@@ -113,7 +104,15 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R
 	return true
 }
 
-func getCarParams(r *http.Request) (CarParams, error) {
+// buildCarParams returns CarParams based on the request, any optional parameters
+// passed in URL, Accept header and the implicit defaults specific to boxo
+// implementation, such as block order and duplicates status.
+//
+// If any of the optional content type parameters (e.g., CAR order or
+// duplicates) are unspecified or empty, the function will automatically infer
+// default values.
+func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarParams, error) {
+	// URL query parameters
 	queryParams := r.URL.Query()
 	rangeStr, hasRange := queryParams.Get(carRangeBytesKey), queryParams.Has(carRangeBytesKey)
 	scopeStr, hasScope := queryParams.Get(carTerminalElementTypeKey), queryParams.Has(carTerminalElementTypeKey)
@@ -122,7 +121,7 @@ func getCarParams(r *http.Request) (CarParams, error) {
 	if hasRange {
 		rng, err := NewDagByteRange(rangeStr)
 		if err != nil {
-			err = fmt.Errorf("invalid entity-bytes: %w", err)
+			err = fmt.Errorf("invalid application/vnd.ipld.car entity-bytes URL parameter: %w", err)
 			return CarParams{}, err
 		}
 		params.Range = &rng
@@ -133,16 +132,78 @@ func getCarParams(r *http.Request) (CarParams, error) {
 		case DagScopeEntity, DagScopeAll, DagScopeBlock:
 			params.Scope = s
 		default:
-			err := fmt.Errorf("unsupported dag-scope %s", scopeStr)
+			err := fmt.Errorf("unsupported application/vnd.ipld.car dag-scope URL parameter: %q", scopeStr)
 			return CarParams{}, err
 		}
 	} else {
 		params.Scope = DagScopeAll
 	}
 
+	// application/vnd.ipld.car content type parameters from Accept header
+
+	// version of CAR format
+	switch contentTypeParams["version"] {
+	case "": // noop, client does not care about version
+	case "1": // noop, we support this
+	default:
+		return CarParams{}, fmt.Errorf("unsupported application/vnd.ipld.car version: only version=1 is supported")
+	}
+
+	// optional order from IPIP-412
+	if order := DagOrder(contentTypeParams["order"]); order != DagOrderUnspecified {
+		switch order {
+		case DagOrderUnknown, DagOrderDFS:
+			params.Order = order
+		default:
+			return CarParams{}, fmt.Errorf("unsupported application/vnd.ipld.car content type order parameter: %q", order)
+		}
+	} else {
+		// when order is not specified, we use DFS as the implicit default
+		// as this has always been the default behavior and we should not break
+		// legacy clients
+		params.Order = DagOrderDFS
+	}
+
+	// optional dups from IPIP-412
+	if dups := NewDuplicateBlocksPolicy(contentTypeParams["dups"]); dups != DuplicateBlocksUnspecified {
+		switch dups {
+		case DuplicateBlocksExcluded, DuplicateBlocksIncluded:
+			params.Duplicates = dups
+		default:
+			return CarParams{}, fmt.Errorf("unsupported application/vnd.ipld.car content type dups parameter:  %q", dups)
+		}
+	} else {
+		// when duplicate block preference is not specified, we set it to
+		// false, as this has always been the default behavior, we should
+		// not break legacy clients, and responses to requests made via ?format=car
+		// should benefit from block deduplication
+		params.Duplicates = DuplicateBlocksExcluded
+
+	}
+
 	return params, nil
 }
 
+// buildContentTypeFromCarParams returns a string for Content-Type header.
+// It does not change any values, CarParams are respected as-is.
+func buildContentTypeFromCarParams(params CarParams) string {
+	h := strings.Builder{}
+	h.WriteString(carResponseFormat)
+	h.WriteString("; version=1")
+
+	if params.Order != DagOrderUnspecified {
+		h.WriteString("; order=")
+		h.WriteString(string(params.Order))
+	}
+
+	if params.Duplicates != DuplicateBlocksUnspecified {
+		h.WriteString("; dups=")
+		h.WriteString(params.Duplicates.String())
+	}
+
+	return h.String()
+}
+
 func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error) {
 	imPathStr := imPath.String()
 	if !strings.HasPrefix(imPathStr, "/ipfs/") {
@@ -167,14 +228,25 @@ func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error)
 func getCarEtag(imPath ImmutablePath, params CarParams, rootCid cid.Cid) string {
 	data := imPath.String()
 	if params.Scope != DagScopeAll {
-		data += "." + string(params.Scope)
+		data += string(params.Scope)
+	}
+
+	// 'order' from IPIP-412 impact Etag only if set to something else
+	// than DFS (which is the implicit default)
+	if params.Order != DagOrderDFS {
+		data += string(params.Order)
+	}
+
+	// 'dups' from IPIP-412 impact Etag only if 'y'
+	if dups := params.Duplicates.String(); dups == "y" {
+		data += dups
 	}
 
 	if params.Range != nil {
 		if params.Range.From != 0 || params.Range.To != nil {
-			data += "." + strconv.FormatInt(params.Range.From, 10)
+			data += strconv.FormatInt(params.Range.From, 10)
 			if params.Range.To != nil {
-				data += "." + strconv.FormatInt(*params.Range.To, 10)
+				data += strconv.FormatInt(*params.Range.To, 10)
 			}
 		}
 	}