Skip to content

Commit fc1d61c

Browse files
committed
[SWT-NNNN] Exit tests
One of the first enhancement requests we received for swift-testing was the ability to test for precondition failures and other critical failures that terminate the current process when they occur. This feature is also frequently requested for XCTest. With swift-testing, we have the opportunity to build such a feature in an ergonomic way. Read the full proposal [here](https://github.com/apple/swift-testing/blob/jgrynspan/exit-tests-proposal/Documentation/Proposals/NNNN-exit-tests.md).
1 parent c488e8f commit fc1d61c

File tree

10 files changed

+856
-27
lines changed

10 files changed

+856
-27
lines changed

Documentation/Proposals/NNNN-exit-tests.md

+744
Large diffs are not rendered by default.

Sources/Testing/ExitTests/ExitCondition.swift

+66-6
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,42 @@ private import _TestingInternals
1717
/// test is expected to pass or fail by passing them to
1818
/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or
1919
/// ``require(exitsWith:observing:_:sourceLocation:performing:)``.
20-
@_spi(Experimental)
20+
///
21+
/// ## Topics
22+
///
23+
/// ### Successful exit conditions
24+
///
25+
/// - ``success``
26+
///
27+
/// ### Failing exit conditions
28+
///
29+
/// - ``failure``
30+
/// - ``exitCode(_:)``
31+
/// - ``signal(_:)``
32+
///
33+
/// ### Comparing exit conditions
34+
///
35+
/// - ``/Swift/Optional/==(_:_:)``
36+
/// - ``/Swift/Optional/!=(_:_:)``
37+
/// - ``/Swift/Optional/===(_:_:)``
38+
/// - ``/Swift/Optional/!==(_:_:)``
39+
///
40+
/// @Metadata {
41+
/// @Available(Swift, introduced: 6.2)
42+
/// }
2143
#if SWT_NO_PROCESS_SPAWNING
2244
@available(*, unavailable, message: "Exit tests are not available on this platform.")
2345
#endif
2446
public enum ExitCondition: Sendable {
2547
/// The process terminated successfully with status `EXIT_SUCCESS`.
48+
///
49+
/// The C programming language defines two [standard exit codes](https://en.cppreference.com/w/c/program/EXIT_status),
50+
/// `EXIT_SUCCESS` and `EXIT_FAILURE` as well as `0` (as a synonym for
51+
/// `EXIT_SUCCESS`.)
52+
///
53+
/// @Metadata {
54+
/// @Available(Swift, introduced: 6.2)
55+
/// }
2656
public static var success: Self {
2757
// Strictly speaking, the C standard treats 0 as a successful exit code and
2858
// potentially distinct from EXIT_SUCCESS. To my knowledge, no modern
@@ -33,6 +63,10 @@ public enum ExitCondition: Sendable {
3363

3464
/// The process terminated abnormally with any status other than
3565
/// `EXIT_SUCCESS` or with any signal.
66+
///
67+
/// @Metadata {
68+
/// @Available(Swift, introduced: 6.2)
69+
/// }
3670
case failure
3771

3872
/// The process terminated with the given exit code.
@@ -56,6 +90,10 @@ public enum ExitCondition: Sendable {
5690
/// the process is yielded to the parent process. Linux and other POSIX-like
5791
/// systems may only reliably report the low unsigned 8 bits (0–255) of
5892
/// the exit code.
93+
///
94+
/// @Metadata {
95+
/// @Available(Swift, introduced: 6.2)
96+
/// }
5997
case exitCode(_ exitCode: CInt)
6098

6199
/// The process terminated with the given signal.
@@ -73,12 +111,18 @@ public enum ExitCondition: Sendable {
73111
/// | FreeBSD | [`<signal.h>`](https://man.freebsd.org/cgi/man.cgi?signal(3)) |
74112
/// | OpenBSD | [`<signal.h>`](https://man.openbsd.org/signal.3) |
75113
/// | Windows | [`<signal.h>`](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) |
114+
///
115+
/// @Metadata {
116+
/// @Available(Swift, introduced: 6.2)
117+
/// }
76118
case signal(_ signal: CInt)
77119
}
78120

79121
// MARK: - Equatable
80122

81-
@_spi(Experimental)
123+
/// @Metadata {
124+
/// @Available(Swift, introduced: 6.2)
125+
/// }
82126
#if SWT_NO_PROCESS_SPAWNING
83127
@available(*, unavailable, message: "Exit tests are not available on this platform.")
84128
#endif
@@ -109,7 +153,11 @@ extension Optional<ExitCondition> {
109153
/// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable).
110154
///
111155
/// For any values `a` and `b`, `a == b` implies that `a != b` is `false`.
112-
public static func ==(lhs: Self, rhs: Self) -> Bool {
156+
///
157+
/// @Metadata {
158+
/// @Available(Swift, introduced: 6.2)
159+
/// }
160+
public static func ==(lhs: ExitCondition?, rhs: ExitCondition?) -> Bool {
113161
#if !SWT_NO_PROCESS_SPAWNING
114162
return switch (lhs, rhs) {
115163
case let (.failure, .exitCode(exitCode)), let (.exitCode(exitCode), .failure):
@@ -151,7 +199,11 @@ extension Optional<ExitCondition> {
151199
/// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable).
152200
///
153201
/// For any values `a` and `b`, `a == b` implies that `a != b` is `false`.
154-
public static func !=(lhs: Self, rhs: Self) -> Bool {
202+
///
203+
/// @Metadata {
204+
/// @Available(Swift, introduced: 6.2)
205+
/// }
206+
public static func !=(lhs: ExitCondition?, rhs: ExitCondition?) -> Bool {
155207
#if !SWT_NO_PROCESS_SPAWNING
156208
!(lhs == rhs)
157209
#else
@@ -185,7 +237,11 @@ extension Optional<ExitCondition> {
185237
/// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable).
186238
///
187239
/// For any values `a` and `b`, `a === b` implies that `a !== b` is `false`.
188-
public static func ===(lhs: Self, rhs: Self) -> Bool {
240+
///
241+
/// @Metadata {
242+
/// @Available(Swift, introduced: 6.2)
243+
/// }
244+
public static func ===(lhs: ExitCondition?, rhs: ExitCondition?) -> Bool {
189245
return switch (lhs, rhs) {
190246
case (.none, .none):
191247
true
@@ -226,7 +282,11 @@ extension Optional<ExitCondition> {
226282
/// or [`Hashable`](https://developer.apple.com/documentation/swift/hashable).
227283
///
228284
/// For any values `a` and `b`, `a === b` implies that `a !== b` is `false`.
229-
public static func !==(lhs: Self, rhs: Self) -> Bool {
285+
///
286+
/// @Metadata {
287+
/// @Available(Swift, introduced: 6.2)
288+
/// }
289+
public static func !==(lhs: ExitCondition?, rhs: ExitCondition?) -> Bool {
230290
#if !SWT_NO_PROCESS_SPAWNING
231291
!(lhs === rhs)
232292
#else

Sources/Testing/ExitTests/ExitTest.swift

+9-11
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ private import _TestingInternals
3636
/// an instance of this type. To create an exit test, use the
3737
/// ``expect(exitsWith:_:sourceLocation:performing:)`` or
3838
/// ``require(exitsWith:_:sourceLocation:performing:)`` macro.
39-
@_spi(Experimental) @_spi(ForToolsIntegrationOnly)
39+
@_spi(ForToolsIntegrationOnly)
4040
#if SWT_NO_EXIT_TESTS
4141
@available(*, unavailable, message: "Exit tests are not available on this platform.")
4242
#endif
@@ -47,7 +47,6 @@ public typealias ExitTest = __ExitTest
4747
/// - Warning: This type is used to implement the `#expect(exitsWith:)` macro.
4848
/// Do not use it directly. Tools can use the SPI ``ExitTest`` typealias if
4949
/// needed.
50-
@_spi(Experimental)
5150
#if SWT_NO_EXIT_TESTS
5251
@available(*, unavailable, message: "Exit tests are not available on this platform.")
5352
#endif
@@ -130,7 +129,7 @@ public struct __ExitTest: Sendable, ~Copyable {
130129
#if !SWT_NO_EXIT_TESTS
131130
// MARK: - Invocation
132131

133-
@_spi(Experimental) @_spi(ForToolsIntegrationOnly)
132+
@_spi(ForToolsIntegrationOnly)
134133
extension ExitTest {
135134
/// Disable crash reporting, crash logging, or core dumps for the current
136135
/// process.
@@ -180,8 +179,7 @@ extension ExitTest {
180179
/// This function invokes the closure originally passed to
181180
/// `#expect(exitsWith:)` _in the current process_. That closure is expected
182181
/// to terminate the process; if it does not, the testing library will
183-
/// terminate the process in a way that causes the corresponding expectation
184-
/// to fail.
182+
/// terminate the process as if its `main()` function returned naturally.
185183
public consuming func callAsFunction() async -> Never {
186184
Self._disableCrashReporting()
187185

@@ -231,7 +229,7 @@ extension ExitTest: TestContent {
231229
typealias TestContentAccessorHint = ID
232230
}
233231

234-
@_spi(Experimental) @_spi(ForToolsIntegrationOnly)
232+
@_spi(ForToolsIntegrationOnly)
235233
extension ExitTest {
236234
/// Find the exit test function at the given source location.
237235
///
@@ -358,7 +356,7 @@ extension ABI {
358356
fileprivate typealias BackChannelVersion = v1
359357
}
360358

361-
@_spi(Experimental) @_spi(ForToolsIntegrationOnly)
359+
@_spi(ForToolsIntegrationOnly)
362360
extension ExitTest {
363361
/// A handler that is invoked when an exit test starts.
364362
///
@@ -394,7 +392,7 @@ extension ExitTest {
394392
/// events should be written, or `nil` if the file handle could not be
395393
/// resolved.
396394
private static let _backChannelForEntryPoint: FileHandle? = {
397-
guard let backChannelEnvironmentVariable = Environment.variable(named: "SWT_EXPERIMENTAL_BACKCHANNEL") else {
395+
guard let backChannelEnvironmentVariable = Environment.variable(named: "SWT_BACKCHANNEL") else {
398396
return nil
399397
}
400398

@@ -427,7 +425,7 @@ extension ExitTest {
427425
static func findInEnvironmentForEntryPoint() -> Self? {
428426
// Find the ID of the exit test to run, if any, in the environment block.
429427
var id: __ExitTest.ID?
430-
if var idString = Environment.variable(named: "SWT_EXPERIMENTAL_EXIT_TEST_ID") {
428+
if var idString = Environment.variable(named: "SWT_EXIT_TEST_ID") {
431429
id = try? idString.withUTF8 { idBuffer in
432430
try JSON.decode(__ExitTest.ID.self, from: UnsafeRawBufferPointer(idBuffer))
433431
}
@@ -560,7 +558,7 @@ extension ExitTest {
560558
// Insert a specific variable that tells the child process which exit test
561559
// to run.
562560
try JSON.withEncoding(of: exitTest.id) { json in
563-
childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self)
561+
childEnvironment["SWT_EXIT_TEST_ID"] = String(decoding: json, as: UTF8.self)
564562
}
565563

566564
typealias ResultUpdater = @Sendable (inout ExitTestArtifacts) -> Void
@@ -606,7 +604,7 @@ extension ExitTest {
606604
#warning("Platform-specific implementation missing: back-channel pipe unavailable")
607605
#endif
608606
if let backChannelEnvironmentVariable {
609-
childEnvironment["SWT_EXPERIMENTAL_BACKCHANNEL"] = backChannelEnvironmentVariable
607+
childEnvironment["SWT_BACKCHANNEL"] = backChannelEnvironmentVariable
610608
}
611609

612610
// Spawn the child process.

Sources/Testing/ExitTests/ExitTestArtifacts.swift

+16-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@
1616
/// instances of this type.
1717
///
1818
/// - Warning: The name of this type is still unstable and subject to change.
19-
@_spi(Experimental)
19+
///
20+
/// @Metadata {
21+
/// @Available(Swift, introduced: 6.2)
22+
/// }
2023
#if SWT_NO_EXIT_TESTS
2124
@available(*, unavailable, message: "Exit tests are not available on this platform.")
2225
#endif
@@ -29,6 +32,10 @@ public struct ExitTestArtifacts: Sendable {
2932
/// ``require(exitsWith:observing:_:sourceLocation:performing:)``. You can
3033
/// compare two instances of ``ExitCondition`` with
3134
/// ``/Swift/Optional/==(_:_:)``.
35+
///
36+
/// @Metadata {
37+
/// @Available(Swift, introduced: 6.2)
38+
/// }
3239
public var exitCondition: ExitCondition
3340

3441
/// All bytes written to the standard output stream of the exit test before
@@ -55,6 +62,10 @@ public struct ExitTestArtifacts: Sendable {
5562
///
5663
/// If you did not request standard output content when running an exit test,
5764
/// the value of this property is the empty array.
65+
///
66+
/// @Metadata {
67+
/// @Available(Swift, introduced: 6.2)
68+
/// }
5869
public var standardOutputContent: [UInt8] = []
5970

6071
/// All bytes written to the standard error stream of the exit test before
@@ -81,6 +92,10 @@ public struct ExitTestArtifacts: Sendable {
8192
///
8293
/// If you did not request standard error content when running an exit test,
8394
/// the value of this property is the empty array.
95+
///
96+
/// @Metadata {
97+
/// @Available(Swift, introduced: 6.2)
98+
/// }
8499
public var standardErrorContent: [UInt8] = []
85100

86101
@_spi(ForToolsIntegrationOnly)

Sources/Testing/Expectations/Expectation+Macro.swift

+12-4
Original file line numberDiff line numberDiff line change
@@ -536,12 +536,16 @@ public macro require<R>(
536536
/// ```
537537
///
538538
/// An exit test cannot run within another exit test.
539-
@_spi(Experimental)
539+
///
540+
/// @Metadata {
541+
/// @Available(Swift, introduced: 6.2)
542+
/// }
540543
#if SWT_NO_EXIT_TESTS
541544
@available(*, unavailable, message: "Exit tests are not available on this platform.")
542545
#endif
543546
@discardableResult
544-
@freestanding(expression) public macro expect(
547+
@freestanding(expression)
548+
public macro expect(
545549
exitsWith expectedExitCondition: ExitCondition,
546550
observing observedValues: [any PartialKeyPath<ExitTestArtifacts> & Sendable] = [],
547551
_ comment: @autoclosure () -> Comment? = nil,
@@ -648,12 +652,16 @@ public macro require<R>(
648652
/// ```
649653
///
650654
/// An exit test cannot run within another exit test.
651-
@_spi(Experimental)
655+
///
656+
/// @Metadata {
657+
/// @Available(Swift, introduced: 6.2)
658+
/// }
652659
#if SWT_NO_EXIT_TESTS
653660
@available(*, unavailable, message: "Exit tests are not available on this platform.")
654661
#endif
655662
@discardableResult
656-
@freestanding(expression) public macro require(
663+
@freestanding(expression)
664+
public macro require(
657665
exitsWith expectedExitCondition: ExitCondition,
658666
observing observedValues: [any PartialKeyPath<ExitTestArtifacts> & Sendable] = [],
659667
_ comment: @autoclosure () -> Comment? = nil,

Sources/Testing/Expectations/ExpectationChecking+Macro.swift

-1
Original file line numberDiff line numberDiff line change
@@ -1145,7 +1145,6 @@ public func __checkClosureCall<R>(
11451145
///
11461146
/// - Warning: This function is used to implement the `#expect()` and
11471147
/// `#require()` macros. Do not call it directly.
1148-
@_spi(Experimental)
11491148
public func __checkClosureCall(
11501149
identifiedBy exitTestID: __ExitTest.ID,
11511150
exitsWith expectedExitCondition: ExitCondition,

Sources/Testing/Running/Configuration.swift

+1-2
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,7 @@ public struct Configuration: Sendable {
217217
/// When using the `swift test` command from Swift Package Manager, this
218218
/// property is pre-configured. Otherwise, the default value of this property
219219
/// records an issue indicating that it has not been configured.
220-
@_spi(Experimental)
221-
public var exitTestHandler: ExitTest.Handler = { exitTest in
220+
public var exitTestHandler: ExitTest.Handler = { _ in
222221
throw SystemError(description: "Exit test support has not been implemented by the current testing infrastructure.")
223222
}
224223
#endif

Sources/Testing/Test+Discovery+Legacy.swift

-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ let testContainerTypeNameMagic = "__🟠$test_container__"
3030
/// - Warning: This protocol is used to implement the `#expect(exitsWith:)`
3131
/// macro. Do not use it directly.
3232
@_alwaysEmitConformanceMetadata
33-
@_spi(Experimental)
3433
public protocol __ExitTestContainer {
3534
/// The unique identifier of the exit test.
3635
static var __id: __ExitTest.ID { get }

Sources/Testing/Testing.docc/Expectations.md

+7
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,13 @@ the test when the code doesn't satisfy a requirement, use
7272
- ``require(throws:_:sourceLocation:performing:)-4djuw``
7373
- ``require(_:sourceLocation:performing:throws:)``
7474

75+
### Checking how processes exit
76+
77+
- ``expect(exitsWith:observing:_:sourceLocation:performing:)``
78+
- ``require(exitsWith:observing:_:sourceLocation:performing:)``
79+
- ``ExitCondition``
80+
- ``ExitTestArtifacts``
81+
7582
### Confirming that asynchronous events occur
7683

7784
- <doc:testing-asynchronous-code>

Tests/TestingTests/ExitTestTests.swift

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
99
//
1010

11-
@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing
11+
@testable @_spi(ForToolsIntegrationOnly) import Testing
1212
private import _TestingInternals
1313

1414
#if !SWT_NO_EXIT_TESTS

0 commit comments

Comments
 (0)