Add tests for ansible

README.md

@@ -1,10 +1,80 @@
-# structural-tests
-Securesign project structural and acceptance tests.
+# Structural tests
+Securesign project structural and acceptance tests. Based on
+* Securesign releases: https://github.com/securesign/releases
+* Securesign operator: https://github.com/securesign/secure-sign-operator
+* Securesign Ansible collection: https://github.com/securesign/artifact-signer-ansible
 
+## Automation
+Current automation is done via Github actions here: https://github.com/securesign/releases/actions/workflows/structural.yml
 
-## Repository list
-Pull list of repositories from Pyxis API
+## Manual testing
+It is necessary to point to the release [snapshot](https://github.com/securesign/releases/blob/main/1.1.0/stable/snapshot.json) file. All other components
+for the tests are taken from that file, as shown below:
+
+    "operator": {
+        "snapshot_name": "operator-v1-1-4x2vj",
+        "rhtas-operator-image": "quay.io/securesign/rhtas-operator-v1-1@sha256:3a61aca9fa8ed6580a367bc08a45cc27fc7f50ff24e786ffde9ec3d9c549b00b",
+        "rhtas-operator-bundle-image": "quay.io/securesign/rhtas-operator-bundle-v1-1@sha256:6db817ed76948417f358d402e737df7b320f82462ad164b002ded15e560a0fdf"
+    },
+
+    "artifact-signer-ansible": {
+        "collection": {
+            "url": "https://github.com/securesign/artifact-signer-ansible/actions/runs/11705765669/artifacts/2152648141",
+            "sha256": "4da3d330f9e82a65d93b242e0cc14b5912d4bf65d0eac31fe1d226e4c6ae11f5"
+        }
+    }
+
+### Parameters
+* ``SNAPSHOT`` - points to the ``snapshot.json`` file, can be local or on a server (github).
+* ``TEST_GITHUB_TOKEN`` - token used to access  ``releases`` project on github.
+* ``ANSIBLE`` - ansible collection zip file used instead of the one defined in ``snapshot.json`` file. Can also be local.
+* ``REPOSITORIES`` - file with images published in ``registry.redhat.io``, default ``testdata/repositories.json``. For how to get or update this file, 
+  check [Repository List](#repository-list) chapter.
+
+### Examples
+Run tests based on a github file:
+
+    SNAPSHOT=https://raw.githubusercontent.com/securesign/releases/refs/heads/feat/release-1.1.1/1.1.1/stable/snapshot.json \
+    TEST_GITHUB_TOKEN=ghp_Ae \
+    go test -v ./test/... --ginkgo.v
+
+Run the same tests on a local (cloned) file:
+
+    SNAPSHOT=../releases/1.1.1/stable/snapshot.json \
+    go test -v ./test/... --ginkgo.v
+
+Force different ansible collection instead of the one defined in ``snapshot.json`` file. This may be useful, when checking ansible collection not yet published:
+
+    SNAPSHOT=../releases/1.1.1/stable/snapshot.json \
+    ANSIBLE=https://api.github.com/repos/securesign/artifact-signer-ansible/actions/artifacts/2442056100/zip \
+    go test -v ./test/... --ginkgo.v
+
+To run just individual test use ``--ginkgo.fokus-file`` parameter:
+
+    SNAPSHOT=../releases/1.1.1/stable/snapshot.json \
+    go test -v ./test/... --ginkgo.v --ginkgo.focus-file "ansible"
+
+## Repository List
+The [repositories.json](testdata/repositories.json) file is used to check of all images are published correctly. To pull the list of repositories from Pyxis API:
 
 ```bash
 curl --negotiate -u : -b .cookiejar.txt -c .cookiejar.txt 'https://pyxis.engineering.redhat.com/v1/product-listings/id/6604180e80e2fa3e4947d1d5/repositories?filter=release_categories%3Din%3D%28%22Generally%20Available%22%29&include=data.repository,data._id,data.published' | jq > testdata/repositories.json
 ```
+
+## Ansible Artifacts
+Published Ansible collections are also stored as an zip [artifacts](https://github.com/securesign/artifact-signer-ansible/actions/workflows/collection-build.yaml).
+To download list of available artifacts:
+
+    curl -L \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer ghp_Ae" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/securesign/artifact-signer-ansible/actions/artifacts
+
+Downloading one artifact:
+
+    curl -L -O \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer ghp_Ae" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/securesign/artifact-signer-ansible/actions/artifacts/2442056100/zip

go.mod

@@ -6,6 +6,8 @@ require (
 	github.com/docker/docker v27.4.1+incompatible
 	github.com/onsi/ginkgo/v2 v2.22.2
 	github.com/onsi/gomega v1.36.2
+	github.com/pkg/errors v0.9.1
+	gopkg.in/yaml.v3 v3.0.1
 	sigs.k8s.io/controller-runtime v0.19.4
 )
 
@@ -28,7 +30,6 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
 	go.opentelemetry.io/otel v1.33.0 // indirect
@@ -44,6 +45,5 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
 	google.golang.org/grpc v1.66.0 // indirect
 	google.golang.org/protobuf v1.36.1 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.1 // indirect
 )

test/acceptance/ansible_images_test.go

@@ -0,0 +1,129 @@
+package acceptance
+
+import (
+	"fmt"
+	"log"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/securesign/structural-tests/test/support"
+)
+
+var _ = Describe("Trusted Artifact Signer Ansible", Ordered, func() {
+
+	var (
+		snapshotData support.SnapshotData
+		repositories *support.RepositoryList
+
+		ansibleFileContent []byte
+
+		ansibleTasImages   support.AnsibleMap
+		ansibleOtherImages support.AnsibleMap
+	)
+
+	It("get and parse snapshot file", func() {
+		var err error
+		snapshotData, err = support.ParseSnapshotData()
+		support.LogMap(fmt.Sprintf("Snapshot images (%d):", len(snapshotData.Images)), snapshotData.Images)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(snapshotData.Images).NotTo(BeEmpty(), "No images were detected in snapshot file")
+
+		repositories, err = support.LoadRepositoryList()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repositories.Data).NotTo(BeEmpty(), "No images were detected in repositories file")
+	})
+
+	It("load ansible definition file", func() {
+		var err error
+		ansibleCollectionURL := support.GetEnv(support.EnvAnsibleImagesFile)
+		if ansibleCollectionURL == "" {
+			support.LogAvailableAnsibleArtifacts()
+			// standard way - use ansible definition file path from releases snapshot.json file
+			snapshotAnsibleURL := snapshotData.Others[support.AnsibleCollectionKey]
+			log.Printf("Using %s URL from snapshot.json file\n", snapshotAnsibleURL)
+			Expect(snapshotAnsibleURL).NotTo(BeEmpty())
+			ansibleCollectionURL, err = support.MapAnsibleZipFileURL(snapshotAnsibleURL)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ansibleCollectionURL).NotTo(BeEmpty())
+		}
+		ansibleFileContent, err = support.LoadAnsibleCollectionSnapshotFile(ansibleCollectionURL, support.AnsibleCollectionSnapshotFile)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ansibleFileContent).NotTo(BeEmpty(), "Ansible definition file seems to be empty")
+	})
+
+	It("get and parse ansible images definition file", func() {
+		ansibleAllImages, err := support.MapAnsibleImages(ansibleFileContent)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ansibleAllImages).NotTo(BeEmpty())
+		ansibleTasImages, ansibleOtherImages = support.SplitMap(ansibleAllImages, support.AnsibleTasImageKeys())
+		Expect(ansibleTasImages).NotTo(BeEmpty())
+		Expect(ansibleOtherImages).NotTo(BeEmpty())
+		support.LogMap(fmt.Sprintf("Ansible TAS images (%d):", len(ansibleTasImages)), ansibleTasImages)
+		support.LogMap(fmt.Sprintf("Ansible other images (%d):", len(ansibleOtherImages)), ansibleOtherImages)
+	})
+
+	It("ansible TAS images are listed in registry.redhat.io", func() {
+		var errs []error
+		for _, ansibleImage := range ansibleTasImages {
+			if repositories.FindByImage(ansibleImage) == nil {
+				errs = append(errs, fmt.Errorf("%w: %s", ErrNotFoundInRegistry, ansibleImage))
+			}
+		}
+		Expect(errs).To(BeEmpty())
+	})
+
+	It("ansible TAS images are all valid", func() {
+		Expect(support.GetMapKeys(ansibleTasImages)).To(ContainElements(support.AnsibleTasImageKeys()))
+		Expect(len(ansibleTasImages)).To(BeNumerically("==", len(support.AnsibleTasImageKeys())))
+		Expect(ansibleTasImages).To(HaveEach(MatchRegexp(support.TasImageDefinitionRegexp)))
+	})
+
+	It("ansible other images are all valid", func() {
+		Expect(support.GetMapKeys(ansibleOtherImages)).To(ContainElements(support.AnsibleOtherImageKeys()))
+		Expect(len(ansibleOtherImages)).To(BeNumerically("==", len(support.AnsibleOtherImageKeys())))
+		Expect(ansibleOtherImages).To(HaveEach(MatchRegexp(support.OtherImageDefinitionRegexp)))
+	})
+
+	It("all ansible TAS image hashes are also defined in releases snapshot", func() {
+		mapped := make(map[string]string)
+		for _, imageKey := range support.AnsibleTasImageKeys() {
+
+			// skip, while ansible uses older tuf image
+			if imageKey == "tas_single_node_tuf_image" {
+				log.Printf("Ansible uses differet TUF image - skipping")
+				log.Printf("  Ansible:  %s", ansibleTasImages[imageKey])
+				log.Printf("  Snapshot: %s", snapshotData.Images[support.ConvertAnsibleImageKey(imageKey)])
+				continue
+			}
+
+			aSha := support.ExtractHash(ansibleTasImages[imageKey])
+			if _, keyExist := snapshotData.Images[support.ConvertAnsibleImageKey(imageKey)]; !keyExist {
+				mapped[imageKey] = "MISSING"
+				continue
+			}
+			sSha := support.ExtractHash(snapshotData.Images[support.ConvertAnsibleImageKey(imageKey)])
+			if aSha == sSha {
+				mapped[imageKey] = "match"
+			} else {
+				mapped[imageKey] = "DIFFERENT HASHES"
+			}
+		}
+		Expect(mapped).To(HaveEach("match"), "Ansible images are missing or have different hashes in snapshot file")
+	})
+
+	It("image hashes are all unique", func() {
+		aImageHashes := support.ExtractHashes(support.GetMapValues(ansibleTasImages))
+		hashesCounts := make(map[string]int)
+		for _, hash := range aImageHashes {
+			_, exist := hashesCounts[hash]
+			if exist {
+				hashesCounts[hash]++
+			} else {
+				hashesCounts[hash] = 1
+			}
+		}
+		Expect(hashesCounts).To(HaveEach(1))
+		Expect(ansibleTasImages).To(HaveLen(len(hashesCounts)))
+	})
+
+})

test/acceptance/client_server_test.go

@@ -30,16 +30,16 @@ const (
 var _ = Describe("Client server", Ordered, func() {
 
 	var clientServerImage string
-	var snapshotImages support.SnapshotMap
+	var snapshotData support.SnapshotData
 	var tmpDir string
 
 	Describe("client-server image", func() {
 		It("snapshot.json", func() {
 			var err error
-			snapshotImages, err = support.ParseSnapshotImages()
+			snapshotData, err = support.ParseSnapshotData()
 			Expect(err).NotTo(HaveOccurred())
 
-			clientServerImage = snapshotImages["client-server-image"]
+			clientServerImage = snapshotData.Images["client-server-image"]
 			Expect(clientServerImage).NotTo(BeEmpty())
 		})
 
@@ -68,13 +68,13 @@ var _ = Describe("Client server", Ordered, func() {
 					It("init", func() {
 						switch cli {
 						case "createtree", "updatetree":
-							image = snapshotImages[cli+"-image"]
+							image = snapshotData.Images[cli+"-image"]
 						case "tuftool":
-							image = snapshotImages["tuf-tool-image"]
+							image = snapshotData.Images["tuf-tool-image"]
 						case "rekor-cli":
-							image = snapshotImages["rekor-cli-image"]
+							image = snapshotData.Images["rekor-cli-image"]
 						default:
-							image = snapshotImages[cli+"-cli-image"]
+							image = snapshotData.Images[cli+"-cli-image"]
 						}
 					})
 

test/acceptance/fbc_images_test.go

@@ -25,16 +25,16 @@ var _ = Describe("File-based catalog images", Ordered, func() {
 	var ocps []TableEntry
 	var bundleImage string
 
-	snapshotImages, err := support.ParseSnapshotImages()
+	snapshotData, err := support.ParseSnapshotData()
 	Expect(err).NotTo(HaveOccurred())
-	for key, snapshotImage := range snapshotImages {
+	for key, snapshotImage := range snapshotData.Images {
 		if strings.Index(key, "fbc-") == 0 {
 			ocps = append(ocps, Entry(key, key, snapshotImage))
 		}
 	}
 	Expect(ocps).NotTo(BeEmpty())
 
-	bundleImage = snapshotImages[support.OperatorBundleImageKey]
+	bundleImage = snapshotData.Images[support.OperatorBundleImageKey]
 
 	DescribeTableSubtree("ocp",
 		func(key, fbcImage string) {
@@ -77,9 +77,9 @@ var _ = Describe("File-based catalog images", Ordered, func() {
 			})
 
 			It("extract bundle-image from snapshot.json", func() {
-				snapshotImages, err := support.ParseSnapshotImages()
+				snapshotData, err := support.ParseSnapshotData()
 				Expect(err).NotTo(HaveOccurred())
-				Expect(snapshotImages).NotTo(BeEmpty())
+				Expect(snapshotData.Images).NotTo(BeEmpty())
 
 			})
 

test/acceptance/operator_images_test.go

@@ -16,27 +16,27 @@ var ErrNotFoundInRegistry = errors.New("not found in registry")
 var _ = Describe("Trusted Artifact Signer Operator", Ordered, func() {
 
 	var (
-		snapshotImages      support.SnapshotMap
+		snapshotData        support.SnapshotData
 		repositories        *support.RepositoryList
 		operatorTasImages   support.OperatorMap
 		operatorOtherImages support.OperatorMap
 		operator            string
 	)
 
-	It("get and parse snapshot.json file", func() {
+	It("get and parse snapshot file", func() {
 		var err error
-		snapshotImages, err = support.ParseSnapshotImages()
+		snapshotData, err = support.ParseSnapshotData()
 		Expect(err).NotTo(HaveOccurred())
-		Expect(snapshotImages).NotTo(BeEmpty())
+		Expect(snapshotData.Images).NotTo(BeEmpty(), "No images were detected in snapshot file")
 
 		repositories, err = support.LoadRepositoryList()
 		Expect(err).NotTo(HaveOccurred())
-		Expect(snapshotImages).NotTo(BeEmpty())
+		Expect(repositories.Data).NotTo(BeEmpty(), "No images were detected in repositories file")
 	})
 
 	It("get operator image", func() {
-		operator = snapshotImages[support.OperatorImageKey]
-		Expect(operator).NotTo(BeEmpty())
+		operator = snapshotData.Images[support.OperatorImageKey]
+		Expect(operator).NotTo(BeEmpty(), "Operator image not detected in snapshot file")
 		log.Printf("Using %s\n", operator)
 	})
 
@@ -64,27 +64,31 @@ var _ = Describe("Trusted Artifact Signer Operator", Ordered, func() {
 	It("operator TAS images are all valid", func() {
 		Expect(support.GetMapKeys(operatorTasImages)).To(ContainElements(support.MandatoryTasOperatorImageKeys()))
 		Expect(len(operatorTasImages)).To(BeNumerically("==", len(support.MandatoryTasOperatorImageKeys())))
-		Expect(operatorTasImages).To(HaveEach(MatchRegexp(support.OperatorTasImageDefinitionRegexp)))
+		Expect(operatorTasImages).To(HaveEach(MatchRegexp(support.TasImageDefinitionRegexp)))
 	})
 
 	It("operator other images are all valid", func() {
 		Expect(support.GetMapKeys(operatorOtherImages)).To(ContainElements(support.OtherOperatorImageKeys()))
 		Expect(len(operatorOtherImages)).To(BeNumerically("==", len(support.OtherOperatorImageKeys())))
-		Expect(operatorOtherImages).To(HaveEach(MatchRegexp(support.OtherOperatorImageDefinitionRegexp)))
+		Expect(operatorOtherImages).To(HaveEach(MatchRegexp(support.OtherImageDefinitionRegexp)))
 	})
 
 	It("all image hashes are also defined in releases snapshot", func() {
 		mapped := make(map[string]string)
 		for _, imageKey := range support.MandatoryTasOperatorImageKeys() {
 			oSha := support.ExtractHash(operatorTasImages[imageKey])
-			sSha := support.ExtractHash(snapshotImages[imageKey])
+			if _, keyExist := snapshotData.Images[imageKey]; !keyExist {
+				mapped[imageKey] = "MISSING"
+				continue
+			}
+			sSha := support.ExtractHash(snapshotData.Images[imageKey])
 			if oSha == sSha {
 				mapped[imageKey] = "match"
 			} else {
 				mapped[imageKey] = "DIFFERENT HASHES"
 			}
 		}
-		Expect(mapped).To(HaveEach("match"))
+		Expect(mapped).To(HaveEach("match"), "Operator images are missing or have different hashes in snapshot file")
 	})
 
 	It("image hashes are all unique", func() {
@@ -103,11 +107,11 @@ var _ = Describe("Trusted Artifact Signer Operator", Ordered, func() {
 	})
 
 	It("operator-bundle use the right operator", func() {
-		fileContent, err := support.RunImage(snapshotImages[support.OperatorBundleImageKey], []string{"cat", support.OperatorBundleClusterServiceVersionFile})
+		fileContent, err := support.RunImage(snapshotData.Images[support.OperatorBundleImageKey], []string{"cat", support.OperatorBundleClusterServiceVersionFile})
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fileContent).NotTo(BeEmpty())
 
-		operatorHash := support.ExtractHash(snapshotImages[support.OperatorImageKey])
+		operatorHash := support.ExtractHash(snapshotData.Images[support.OperatorImageKey])
 		re := regexp.MustCompile(`(\w+:\s*[\w./-]+operator[\w-]*@sha256:` + operatorHash + `)`)
 		matches := re.FindAllString(fileContent, -1)
 		Expect(matches).NotTo(BeEmpty())