Skip to content

Commit 2e55bec

Browse files
PriyaModaliPriyaModali
authored
Implement custom tester functionality in Skaffold (#5451)
* Adding new test runner for custom tester. * Adding new custom tester config to Skaffold comfig. * Adding usage example for newly added custon tester. * Extracting structure tests run logic into a seperate method. * Moving structure test's dependency file extraction logic to NewTester() to reduce code redundancy. * Revert "Moving structure test's dependency file extraction logic to NewTester() to reduce code redundancy." This reverts commit 5e1d859. * Moving structure test's dependency file extraction logic to NewTester() to reduce code redundancy. * Making the custom command a required field.[D * Adding new error codes for custom tester. * Updating custom test example. * Removing errors file. * Updating custom test example. * Adding generated schema files. * Added custom test example to `integration/run_test.go`. * Added custom test example in `integration/examples directory`. * Updating the apiVersion in example skaffold.yaml. * Updating the apiVersion in example skaffold.yaml. * Removing the custom-test form examples folder as the newly added custom test fields are not available in the current version of the skaffold config. * Updates per review feedback. * Added schema validation & updated example. * Adding tests for error scenarios. * Updated the test exit code. * Special casing error tests for Windows platform. * Updating the container name for custom test example. * Updated custom-test example container pods to continue running. * Added tests for TestDependencies. * Adding tests for custom test schema validation. * Updated unit tests per review feedback. * Updated unit tests to use mock by overriding util.RunCmd method. * Updating windows specific unit test. * Added another instance of custom command to the integration test. * Updated custom test dependencies. * Updating custom test example with command dependencies.
1 parent 7a04427 commit 2e55bec

File tree

15 files changed

+794
-8
lines changed

15 files changed

+794
-8
lines changed

‎docs/content/en/schemas/v2beta13.json

+79-2
Original file line numberDiff line numberDiff line change
@@ -922,6 +922,74 @@
922922
"description": "*beta* tags images with a configurable template string.",
923923
"x-intellij-html-description": "<em>beta</em> tags images with a configurable template string."
924924
},
925+
"CustomTest": {
926+
"required": [
927+
"command"
928+
],
929+
"properties": {
930+
"command": {
931+
"type": "string",
932+
"description": "custom command to be executed. If the command exits with a non-zero return code, the test will be considered to have failed.",
933+
"x-intellij-html-description": "custom command to be executed. If the command exits with a non-zero return code, the test will be considered to have failed."
934+
},
935+
"dependencies": {
936+
"$ref": "#/definitions/CustomTestDependencies",
937+
"description": "additional test-specific file dependencies; changes to these files will re-run this test.",
938+
"x-intellij-html-description": "additional test-specific file dependencies; changes to these files will re-run this test."
939+
},
940+
"timeoutSeconds": {
941+
"type": "integer",
942+
"description": "sets the wait time for skaffold for the command to complete. If unset or 0, Skaffold will wait until the command completes.",
943+
"x-intellij-html-description": "sets the wait time for skaffold for the command to complete. If unset or 0, Skaffold will wait until the command completes."
944+
}
945+
},
946+
"preferredOrder": [
947+
"command",
948+
"timeoutSeconds",
949+
"dependencies"
950+
],
951+
"additionalProperties": false,
952+
"description": "describes the custom test command provided by the user. Custom tests are run after an image build whenever build or test dependencies are changed.",
953+
"x-intellij-html-description": "describes the custom test command provided by the user. Custom tests are run after an image build whenever build or test dependencies are changed."
954+
},
955+
"CustomTestDependencies": {
956+
"properties": {
957+
"command": {
958+
"type": "string",
959+
"description": "represents a command that skaffold executes to obtain dependencies. The output of this command *must* be a valid JSON array.",
960+
"x-intellij-html-description": "represents a command that skaffold executes to obtain dependencies. The output of this command <em>must</em> be a valid JSON array."
961+
},
962+
"ignore": {
963+
"items": {
964+
"type": "string"
965+
},
966+
"type": "array",
967+
"description": "specifies the paths that should be ignored by skaffold's file watcher. If a file exists in both `paths` and in `ignore`, it will be ignored, and will be excluded from both retest and file synchronization. Will only work in conjunction with `paths`.",
968+
"x-intellij-html-description": "specifies the paths that should be ignored by skaffold's file watcher. If a file exists in both <code>paths</code> and in <code>ignore</code>, it will be ignored, and will be excluded from both retest and file synchronization. Will only work in conjunction with <code>paths</code>.",
969+
"default": "[]"
970+
},
971+
"paths": {
972+
"items": {
973+
"type": "string"
974+
},
975+
"type": "array",
976+
"description": "should be set to the file dependencies for this command, so that the skaffold file watcher knows when to retest and perform file synchronization.",
977+
"x-intellij-html-description": "should be set to the file dependencies for this command, so that the skaffold file watcher knows when to retest and perform file synchronization.",
978+
"default": "[]",
979+
"examples": [
980+
"[\"src/test/**\"]"
981+
]
982+
}
983+
},
984+
"preferredOrder": [
985+
"command",
986+
"paths",
987+
"ignore"
988+
],
989+
"additionalProperties": false,
990+
"description": "used to specify dependencies for custom test command. `paths` should be specified for file watching to work as expected.",
991+
"x-intellij-html-description": "used to specify dependencies for custom test command. <code>paths</code> should be specified for file watching to work as expected."
992+
},
925993
"DateTimeTagger": {
926994
"properties": {
927995
"format": {
@@ -2910,6 +2978,14 @@
29102978
"image"
29112979
],
29122980
"properties": {
2981+
"custom": {
2982+
"items": {
2983+
"$ref": "#/definitions/CustomTest"
2984+
},
2985+
"type": "array",
2986+
"description": "the set of custom tests to run after an artifact is built.",
2987+
"x-intellij-html-description": "the set of custom tests to run after an artifact is built."
2988+
},
29132989
"image": {
29142990
"type": "string",
29152991
"description": "artifact on which to run those tests.",
@@ -2933,11 +3009,12 @@
29333009
},
29343010
"preferredOrder": [
29353011
"image",
3012+
"custom",
29363013
"structureTests"
29373014
],
29383015
"additionalProperties": false,
2939-
"description": "a list of structure tests to run on images that Skaffold builds.",
2940-
"x-intellij-html-description": "a list of structure tests to run on images that Skaffold builds."
3016+
"description": "a list of tests to run on images that Skaffold builds.",
3017+
"x-intellij-html-description": "a list of tests to run on images that Skaffold builds."
29413018
}
29423019
}
29433020
}

‎integration/examples/custom-tests/Dockerfile

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM golang:1.15 as builder
2+
COPY main.go .
3+
# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
4+
ARG SKAFFOLD_GO_GCFLAGS
5+
RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o /app main.go
6+
7+
FROM alpine:3
8+
# Define GOTRACEBACK to mark this container as using the Go language runtime
9+
# for `skaffold debug` (https://skaffold.dev/docs/workflows/debug/).
10+
ENV GOTRACEBACK=single
11+
CMD ["./app"]
12+
COPY --from=builder /app .

‎integration/examples/custom-tests/README.md

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
### Example: Running custom tests on built images
2+
3+
This example shows how to run _custom tests_ on newly built images in the skaffold dev loop.
4+
5+
Custom tests are associated with single image artifacts. When test dependencies change, no build will happen but tests would get re-run. Tests are configured in the `skaffold.yaml` in the `test` stanza, e.g.
6+
7+
```yaml
8+
test:
9+
- image: skaffold-example
10+
custom:
11+
- command: <command>
12+
timeoutSeconds: <timeout in seconds>
13+
dependencies: <dependencies for this command>
14+
paths: <file dependencies>
15+
- <paths glob>
16+
```
17+
18+
As tests take time, you might prefer to configure tests using [profiles](https://skaffold.dev/docs/https://skaffold.dev/docs/environment/profiles/) so that they can be automatically enabled or disabled, e.g.
19+
If the `command` exits with a non-zero return code then the test will have failed, and deployment will not continue.
20+
21+
```yaml
22+
profiles:
23+
- name: test
24+
test:
25+
- image: skaffold-example
26+
custom:
27+
- command: <command>
28+
timeoutSeconds: <timeout in seconds>
29+
dependencies: <dependencies for this command>
30+
paths: <file dependencies>
31+
- <paths glob>
32+
```

‎integration/examples/custom-tests/k8s-pod.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: custom-test
5+
spec:
6+
containers:
7+
- name: custom-test
8+
image: custom-test-example

‎integration/examples/custom-tests/main.go

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
Copyright 2021 The Skaffold Authors
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
Unless required by applicable law or agreed to in writing, software
8+
distributed under the License is distributed on an "AS IS" BASIS,
9+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
See the License for the specific language governing permissions and
11+
limitations under the License.
12+
*/
13+
14+
package main
15+
16+
import (
17+
"fmt"
18+
"math/rand"
19+
"time"
20+
)
21+
22+
func MinInt(a, b int) int {
23+
if a < b {
24+
return a
25+
}
26+
return b
27+
}
28+
29+
func main() {
30+
rand.Seed(time.Now().UnixNano())
31+
for {
32+
x := rand.Intn(100)
33+
y := rand.Intn(100)
34+
35+
min := MinInt(x, y)
36+
fmt.Println("Min of ", x, " and ", y, " is: ", min)
37+
time.Sleep(time.Second * 1)
38+
}
39+
}

‎integration/examples/custom-tests/main_test.go

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
Copyright 2021 The Skaffold Authors
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
Unless required by applicable law or agreed to in writing, software
8+
distributed under the License is distributed on an "AS IS" BASIS,
9+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
See the License for the specific language governing permissions and
11+
limitations under the License.
12+
*/
13+
14+
package main
15+
16+
import (
17+
"fmt"
18+
"testing"
19+
)
20+
21+
func TestMinIntBasic(tb *testing.T) {
22+
fmt.Println("Running Basic test.")
23+
min := MinInt(5, -5)
24+
if min != -5 {
25+
tb.Errorf("MinInt(5, -5) returned %d; expecting -5", min)
26+
}
27+
}
28+
29+
func TestMinIntTableDriven(tdt *testing.T) {
30+
var tests = []struct {
31+
x, y int
32+
want int
33+
}{
34+
{0, 0, 0},
35+
{1, 0, 0},
36+
{0, 1, 0},
37+
{0, -1, -1},
38+
{-1, 0, -1},
39+
{-2, -5, -5},
40+
{-5, -2, -5},
41+
}
42+
43+
fmt.Println("Running Table driven test.")
44+
for _, t := range tests {
45+
testname := fmt.Sprintf("TestMinInt(): %d,%d", t.x, t.y)
46+
tdt.Run(testname, func(tdt *testing.T) {
47+
min := MinInt(t.x, t.y)
48+
if min != t.want {
49+
tdt.Errorf("MinInt(%d, %d) returned %d; expecting %d", t.x, t.y, min, t.want)
50+
}
51+
})
52+
}
53+
}

‎integration/examples/custom-tests/skaffold.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
apiVersion: skaffold/v2beta13
2+
kind: Config
3+
build:
4+
artifacts:
5+
- image: custom-test-example
6+
test:
7+
- image: custom-test-example
8+
custom:
9+
- command: ./test.sh
10+
timeoutSeconds: 60
11+
dependencies:
12+
paths:
13+
- "*_test.go"
14+
- "test.sh"
15+
- command: echo Hello world!!
16+
dependencies:
17+
command: echo [\"main_test.go\"]
18+
deploy:
19+
kubectl:
20+
manifests:
21+
- k8s-*

‎integration/examples/custom-tests/test.sh

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
# Copyright 2021 The Skaffold Authors
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -e
18+
19+
echo "go custom test $@"
20+
21+
go test .

‎integration/run_test.go

+5
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ func TestRun(t *testing.T) {
5151
dir: "examples/structure-tests",
5252
pods: []string{"getting-started"},
5353
},
54+
{
55+
description: "custom-tests",
56+
dir: "examples/custom-tests",
57+
pods: []string{"custom-test"},
58+
},
5459
{
5560
description: "microservices",
5661
dir: "examples/microservices",

‎pkg/skaffold/schema/latest/config.go

+34-1
Original file line numberDiff line numberDiff line change
@@ -468,12 +468,15 @@ type ResourceRequirement struct {
468468
ResourceStorage string `yaml:"resourceStorage,omitempty"`
469469
}
470470

471-
// TestCase is a list of structure tests to run on images that Skaffold builds.
471+
// TestCase is a list of tests to run on images that Skaffold builds.
472472
type TestCase struct {
473473
// ImageName is the artifact on which to run those tests.
474474
// For example: `gcr.io/k8s-skaffold/example`.
475475
ImageName string `yaml:"image" yamltags:"required"`
476476

477+
// CustomTests lists the set of custom tests to run after an artifact is built.
478+
CustomTests []CustomTest `yaml:"custom,omitempty"`
479+
477480
// StructureTests lists the [Container Structure Tests](https://github.com/GoogleContainerTools/container-structure-test)
478481
// to run on that artifact.
479482
// For example: `["./test/*"]`.
@@ -1016,6 +1019,36 @@ type CustomDependencies struct {
10161019
Ignore []string `yaml:"ignore,omitempty"`
10171020
}
10181021

1022+
// CustomTest describes the custom test command provided by the user.
1023+
// Custom tests are run after an image build whenever build or test dependencies are changed.
1024+
type CustomTest struct {
1025+
// Command is the custom command to be executed. If the command exits with a non-zero return
1026+
// code, the test will be considered to have failed.
1027+
Command string `yaml:"command" yamltags:"required"`
1028+
1029+
// TimeoutSeconds sets the wait time for skaffold for the command to complete.
1030+
// If unset or 0, Skaffold will wait until the command completes.
1031+
TimeoutSeconds int `yaml:"timeoutSeconds,omitempty"`
1032+
1033+
// Dependencies are additional test-specific file dependencies; changes to these files will re-run this test.
1034+
Dependencies *CustomTestDependencies `yaml:"dependencies,omitempty"`
1035+
}
1036+
1037+
// CustomTestDependencies is used to specify dependencies for custom test command.
1038+
// `paths` should be specified for file watching to work as expected.
1039+
type CustomTestDependencies struct {
1040+
// Command represents a command that skaffold executes to obtain dependencies. The output of this command *must* be a valid JSON array.
1041+
Command string `yaml:"command,omitempty" yamltags:"oneOf=dependency"`
1042+
1043+
// Paths should be set to the file dependencies for this command, so that the skaffold file watcher knows when to retest and perform file synchronization.
1044+
// For example: `["src/test/**"]`
1045+
Paths []string `yaml:"paths,omitempty" yamltags:"oneOf=dependency" skaffold:"filepath"`
1046+
1047+
// Ignore specifies the paths that should be ignored by skaffold's file watcher. If a file exists in both `paths` and in `ignore`, it will be ignored, and will be excluded from both retest and file synchronization.
1048+
// Will only work in conjunction with `paths`.
1049+
Ignore []string `yaml:"ignore,omitempty" skaffold:"filepath"`
1050+
}
1051+
10191052
// DockerfileDependency *beta* is used to specify a custom build artifact that is built from a Dockerfile. This allows skaffold to determine dependencies from the Dockerfile.
10201053
type DockerfileDependency struct {
10211054
// Path locates the Dockerfile relative to workspace.

0 commit comments

Comments
 (0)