Merge pull request #1471 from shipwright-io/add/source-timestamp

Add source timestamp field for source result

Author: openshift-merge-bot[bot]

.github/workflows/ci.yml

@@ -1,13 +1,13 @@
 name: Unit, Integration, and E2E Tests
-on: 
+on:
   pull_request:
     branches:
     - main
   push:
     paths-ignore:
     - 'README.md'
     - 'docs/**'
-    branches: 
+    branches:
     - main
 
 jobs:
@@ -114,8 +114,9 @@ jobs:
           # host.docker.internal does not work in a GitHub action
           docker exec kind-control-plane bash -c "echo '172.17.0.1 host.docker.internal' >>/etc/hosts"
 
-          # Build and load the Git image
+          # Build and load the Git and Bundle image
           export GIT_CONTAINER_IMAGE="$(KO_DOCKER_REPO=kind.local ko publish ./cmd/git)"
+          export BUNDLE_CONTAINER_IMAGE="$(KO_DOCKER_REPO=kind.local ko publish ./cmd/bundle)"
 
           make test-integration
 

cmd/bundle/main.go

@@ -9,21 +9,25 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strconv"
 
 	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/spf13/pflag"
 
 	"github.com/shipwright-io/build/pkg/bundle"
 	"github.com/shipwright-io/build/pkg/image"
 )
 
 type settings struct {
-	help                  bool
-	image                 string
-	prune                 bool
-	target                string
-	secretPath            string
-	resultFileImageDigest string
+	help                      bool
+	image                     string
+	prune                     bool
+	target                    string
+	secretPath                string
+	resultFileImageDigest     string
+	resultFileSourceTimestamp string
 }
 
 var flagValues settings
@@ -36,6 +40,7 @@ func init() {
 	pflag.StringVar(&flagValues.image, "image", "", "Location of the bundle image (mandatory)")
 	pflag.StringVar(&flagValues.target, "target", "/workspace/source", "The target directory to place the code")
 	pflag.StringVar(&flagValues.resultFileImageDigest, "result-file-image-digest", "", "A file to write the image digest")
+	pflag.StringVar(&flagValues.resultFileSourceTimestamp, "result-file-source-timestamp", "", "A file to write the source timestamp")
 
 	pflag.StringVar(&flagValues.secretPath, "secret-path", "", "A directory that contains access credentials (optional)")
 	pflag.BoolVar(&flagValues.prune, "prune", false, "Delete bundle image from registry after it was pulled")
@@ -72,10 +77,20 @@ func Do(ctx context.Context) error {
 	}
 
 	log.Printf("Pulling image %q", ref)
-	img, err := bundle.PullAndUnpack(
-		ref,
-		flagValues.target,
-		options...)
+	desc, err := remote.Get(ref, options...)
+	if err != nil {
+		return err
+	}
+
+	img, err := desc.Image()
+	if err != nil {
+		return err
+	}
+
+	rc := mutate.Extract(img)
+	defer rc.Close()
+
+	unpackDetails, err := bundle.Unpack(rc, flagValues.target)
 	if err != nil {
 		return err
 	}
@@ -93,6 +108,17 @@ func Do(ctx context.Context) error {
 		}
 	}
 
+	if flagValues.resultFileSourceTimestamp != "" {
+		if unpackDetails.MostRecentFileTimestamp != nil {
+			if err = os.WriteFile(flagValues.resultFileSourceTimestamp, []byte(strconv.FormatInt(unpackDetails.MostRecentFileTimestamp.Unix(), 10)), 0644); err != nil {
+				return err
+			}
+
+		} else {
+			log.Printf("Unable to determine source timestamp of content in %s\n", flagValues.target)
+		}
+	}
+
 	if flagValues.prune {
 		// Some container registry implementations, i.e. library/registry:2 will fail to
 		// delete the image when there is no image digest given. Use image digest from the

cmd/bundle/main_test.go

@@ -9,25 +9,31 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http/httptest"
+	"net/url"
 	"os"
 	"path/filepath"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	. "github.com/shipwright-io/build/cmd/bundle"
-	"github.com/shipwright-io/build/pkg/image"
 
 	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/registry"
 	containerreg "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"k8s.io/apimachinery/pkg/util/rand"
+
+	"github.com/shipwright-io/build/pkg/bundle"
+	"github.com/shipwright-io/build/pkg/image"
 )
 
 var _ = Describe("Bundle Loader", func() {
 	const exampleImage = "ghcr.io/shipwright-io/sample-go/source-bundle:latest"
 
-	var run = func(args ...string) error {
+	run := func(args ...string) error {
 		// discard log output
 		log.SetOutput(io.Discard)
 
@@ -40,7 +46,7 @@ var _ = Describe("Bundle Loader", func() {
 		return Do(context.Background())
 	}
 
-	var withTempDir = func(f func(target string)) {
+	withTempDir := func(f func(target string)) {
 		path, err := os.MkdirTemp(os.TempDir(), "bundle")
 		Expect(err).ToNot(HaveOccurred())
 		defer os.RemoveAll(path)
@@ -56,6 +62,24 @@ var _ = Describe("Bundle Loader", func() {
 		f(file.Name())
 	}
 
+	withTempRegistry := func(f func(endpoint string)) {
+		logLogger := log.Logger{}
+		logLogger.SetOutput(GinkgoWriter)
+
+		s := httptest.NewServer(
+			registry.New(
+				registry.Logger(&logLogger),
+				registry.WithReferrersSupport(true),
+			),
+		)
+		defer s.Close()
+
+		u, err := url.Parse(s.URL)
+		Expect(err).ToNot(HaveOccurred())
+
+		f(u.Host)
+	}
+
 	filecontent := func(path string) string {
 		data, err := os.ReadFile(path)
 		Expect(err).ToNot(HaveOccurred())
@@ -188,14 +212,16 @@ var _ = Describe("Bundle Loader", func() {
 		})
 
 		AfterEach(func() {
-			ref, err := name.ParseReference(testImage)
-			Expect(err).ToNot(HaveOccurred())
+			if testImage != "" {
+				ref, err := name.ParseReference(testImage)
+				Expect(err).ToNot(HaveOccurred())
 
-			options, auth, err := image.GetOptions(context.TODO(), ref, true, dockerConfigFile, "test-agent")
-			Expect(err).ToNot(HaveOccurred())
+				options, auth, err := image.GetOptions(context.TODO(), ref, true, dockerConfigFile, "test-agent")
+				Expect(err).ToNot(HaveOccurred())
 
-			// Delete test image (best effort)
-			_ = image.Delete(ref, options, *auth)
+				// Delete test image (best effort)
+				_ = image.Delete(ref, options, *auth)
+			}
 		})
 
 		It("should pull and unpack an image from a private registry", func() {
@@ -232,4 +258,62 @@ var _ = Describe("Bundle Loader", func() {
 			})
 		})
 	})
+
+	Context("Result file checks", func() {
+		tmpFile := func(dir string, name string, data []byte, timestamp time.Time) {
+			var path = filepath.Join(dir, name)
+
+			Expect(os.WriteFile(
+				path,
+				data,
+				os.FileMode(0644),
+			)).To(Succeed())
+
+			Expect(os.Chtimes(
+				path,
+				timestamp,
+				timestamp,
+			)).To(Succeed())
+		}
+
+		// Creates a controlled reference image with one file called "file" with modification
+		// timestamp of Friday, February 13, 2009 11:31:30 PM (unix timestamp 1234567890)
+		withReferenceImage := func(f func(dig name.Digest)) {
+			withTempRegistry(func(endpoint string) {
+				withTempDir(func(target string) {
+					timestamp := time.Unix(1234567890, 0)
+
+					ref, err := name.ParseReference(fmt.Sprintf("%s/namespace/image:tag", endpoint))
+					Expect(err).ToNot(HaveOccurred())
+					Expect(ref).ToNot(BeNil())
+
+					tmpFile(target, "file", []byte("foobar"), timestamp)
+
+					dig, err := bundle.PackAndPush(ref, target)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(dig).ToNot(BeNil())
+
+					f(dig)
+				})
+			})
+		}
+
+		It("should store source timestamp in result file", func() {
+			withTempDir(func(target string) {
+				withTempDir(func(result string) {
+					withReferenceImage(func(dig name.Digest) {
+						resultSourceTimestamp := filepath.Join(result, "source-timestamp")
+
+						Expect(run(
+							"--image", dig.String(),
+							"--target", target,
+							"--result-file-source-timestamp", resultSourceTimestamp,
+						)).To(Succeed())
+
+						Expect(filecontent(resultSourceTimestamp)).To(Equal("1234567890"))
+					})
+				})
+			})
+		})
+	})
 })

cmd/git/main.go

@@ -46,20 +46,21 @@ func (e ExitError) Error() string {
 }
 
 type settings struct {
-	help                   bool
-	url                    string
-	revision               string
-	depth                  uint
-	target                 string
-	resultFileCommitSha    string
-	resultFileCommitAuthor string
-	resultFileBranchName   string
-	secretPath             string
-	skipValidation         bool
-	gitURLRewrite          bool
-	resultFileErrorMessage string
-	resultFileErrorReason  string
-	verbose                bool
+	help                      bool
+	url                       string
+	revision                  string
+	depth                     uint
+	target                    string
+	resultFileCommitSha       string
+	resultFileCommitAuthor    string
+	resultFileBranchName      string
+	resultFileSourceTimestamp string
+	secretPath                string
+	skipValidation            bool
+	gitURLRewrite             bool
+	resultFileErrorMessage    string
+	resultFileErrorReason     string
+	verbose                   bool
 }
 
 var flagValues settings
@@ -81,6 +82,7 @@ func init() {
 	pflag.StringVar(&flagValues.target, "target", "", "The target directory of the clone operation")
 	pflag.StringVar(&flagValues.resultFileCommitSha, "result-file-commit-sha", "", "A file to write the commit sha to.")
 	pflag.StringVar(&flagValues.resultFileCommitAuthor, "result-file-commit-author", "", "A file to write the commit author to.")
+	pflag.StringVar(&flagValues.resultFileSourceTimestamp, "result-file-source-timestamp", "", "A file to write the source timestamp to.")
 	pflag.StringVar(&flagValues.resultFileBranchName, "result-file-branch-name", "", "A file to write the branch name to.")
 	pflag.StringVar(&flagValues.secretPath, "secret-path", "", "A directory that contains a secret. Either username and password for basic authentication. Or a SSH private key and optionally a known hosts file. Optional.")
 
@@ -180,6 +182,17 @@ func runGitClone(ctx context.Context) error {
 		}
 	}
 
+	if flagValues.resultFileSourceTimestamp != "" {
+		output, err := git(ctx, "-C", flagValues.target, "show", "--no-patch", "--format=%ct")
+		if err != nil {
+			return err
+		}
+
+		if err = os.WriteFile(flagValues.resultFileSourceTimestamp, []byte(output), 0644); err != nil {
+			return err
+		}
+	}
+
 	if strings.TrimSpace(flagValues.revision) == "" && strings.TrimSpace(flagValues.resultFileBranchName) != "" {
 		output, err := git(ctx, "-C", flagValues.target, "rev-parse", "--abbrev-ref", "HEAD")
 		if err != nil {

cmd/git/main_test.go

@@ -471,6 +471,21 @@ var _ = Describe("Git Resource", func() {
 				})
 			})
 		})
+
+		It("should store source-timestamp into file specified in --result-file-source-timestamp flag", func() {
+			withTempFile("source-timestamp", func(filename string) {
+				withTempDir(func(target string) {
+					Expect(run(withArgs(
+						"--url", exampleRepo,
+						"--target", target,
+						"--revision", "v0.1.0",
+						"--result-file-source-timestamp", filename,
+					))).ToNot(HaveOccurred())
+
+					Expect(filecontent(filename)).To(Equal("1619426578"))
+				})
+			})
+		})
 	})
 
 	Context("Some tests mutate or depend on git configurations. They must run sequentially to avoid race-conditions.", Ordered, func() {

deploy/crds/shipwright.io_buildruns.yaml

@@ -6334,6 +6334,13 @@ spec:
                     name:
                       description: Name is the name of source
                       type: string
+                    timestamp:
+                      description: Timestamp holds the timestamp of the source, which
+                        depends on the actual source type and could range from being
+                        the commit timestamp or the fileystem timestamp of the most
+                        recent source file in the working directory
+                      format: date-time
+                      type: string
                   required:
                   - name
                   type: object
@@ -12552,6 +12559,13 @@ spec:
                         description: Digest hold the image digest result
                         type: string
                     type: object
+                  timestamp:
+                    description: Timestamp holds the timestamp of the source, which
+                      depends on the actual source type and could range from being
+                      the commit timestamp or the fileystem timestamp of the most
+                      recent source file in the working directory
+                    format: date-time
+                    type: string
                 type: object
               startTime:
                 description: StartTime is the time the build is actually started.

pkg/apis/build/v1alpha1/buildrun_types.go

@@ -117,6 +117,14 @@ type SourceResult struct {
 	//
 	// +optional
 	Bundle *BundleSourceResult `json:"bundle,omitempty"`
+
+	// Timestamp holds the timestamp of the source, which
+	// depends on the actual source type and could range from
+	// being the commit timestamp or the fileystem timestamp
+	// of the most recent source file in the working directory
+	//
+	// +optional
+	Timestamp *metav1.Time `json:"timestamp,omitempty"`
 }
 
 // BundleSourceResult holds the results emitted from the bundle source