[Testing] Enable snapshot tests on Catalyst (#25710)

* Enable snapshot tests on Catalyst

* Added some mac snapshots

* Updated mac test project

* More snapshots

* More snapshots

* Added scripts to disable the notification center running macos UI tests

* More snapshots

* More snapshots

* More snapshots

* More fixes

* Updated snapshot

* More snapshots

* Updated snapshot

* Ignore test on Catalyst

* More changes

* More changes

* More fixes

* Update AppiumCatalystApp.cs

* Added snapshot from new UITest

* more mac screenshots

---------

Co-authored-by: Rui Marinho <me@ruimarinho.net>
Co-authored-by: tj-devel709 <tj.devel709@gmail.com>

Author: 3 people

eng/pipelines/common/ui-tests-steps.yml

@@ -37,6 +37,14 @@ steps:
       continueOnError: true
       timeoutInMinutes: 60
 
+  - ${{ if eq(parameters.platform, 'catalyst')}}:
+        - bash: |
+              chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/disable-notification-center.sh
+              $(System.DefaultWorkingDirectory)/eng/scripts/disable-notification-center.sh
+          displayName: 'Disable Notification Center'
+          continueOnError: true
+          timeoutInMinutes: 60
+          
   - template: provision.yml
     parameters:
       skipAndroidSdks: ${{ ne(parameters.platform, 'android') }}
@@ -143,8 +151,16 @@ steps:
     condition: always()
     displayName: publish artifacts
 
+  - ${{ if eq(parameters.platform, 'catalyst')}}:
+        - bash: |
+              chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/enable-notification-center.sh
+              $(System.DefaultWorkingDirectory)/eng/scripts/enable-notification-center.sh
+          displayName: 'Enable Notification Center'
+          continueOnError: true
+          timeoutInMinutes: 60
+
   - ${{ if and(eq(parameters.platform, 'ios'), ne(parameters.poolName, 'Azure Pipelines'), eq(variables['System.TeamProject'], 'devdiv') ) }}:
       # This must always be placed as the last step in the job
       - template: agent-rebooter/mac.v1.yml@yaml-templates
         parameters:
-          AgentPoolAccessToken: ${{ parameters.agentPoolAccessToken }}
+          AgentPoolAccessToken: ${{ parameters.agentPoolAccessToken }}

eng/scripts/disable-notification-center.sh

@@ -0,0 +1,22 @@
+#!/bin/sh 
+
+export PATH=/usr/bin:/bin:/usr/sbin:/sbin 
+
+currentUser=$( echo "show State:/Users/ConsoleUser" | scutil | awk '/Name :/ { print $3 }' ) 
+
+if [ -z "$currentUser" -o "$currentUser" = "loginwindow" ]; then 
+  echo "no user logged in, cannot proceed"  
+  exit 1  
+fi  
+
+uid=$(id -u "$currentUser") 
+
+runAsUser() {   
+  if [ "$currentUser" != "loginwindow" ]; then 
+    launchctl asuser "$uid" sudo -u "$currentUser" "$@" 
+  else 
+    echo "no user logged in" 
+  fi 
+} 
+
+runAsUser launchctl unload -w /System/Library/LaunchAgents/com.apple.notificationcenterui.plist 

eng/scripts/enable-notification-center.sh

@@ -0,0 +1,21 @@
+#!/bin/sh 
+export PATH=/usr/bin:/bin:/usr/sbin:/sbin 
+
+currentUser=$( echo "show State:/Users/ConsoleUser" | scutil | awk '/Name :/ { print $3 }' ) 
+
+if [ -z "$currentUser" -o "$currentUser" = "loginwindow" ]; then 
+  echo "no user logged in, cannot proceed" 
+  exit 1  
+fi 
+
+uid=$(id -u "$currentUser")  
+
+runAsUser() {   
+  if [ "$currentUser" != "loginwindow" ]; then 
+    launchctl asuser "$uid" sudo -u "$currentUser" "$@" 
+  else 
+    echo "no user logged in" 
+  fi 
+} 
+
+runAsUser launchctl load -w /System/Library/LaunchAgents/com.apple.notificationcenterui.plist 

src/Controls/tests/TestCases.Mac.Tests/Controls.TestCases.Mac.Tests.csproj

@@ -40,7 +40,13 @@
     <ProjectReference Include="$(MauiRootDirectory)..\visual-test-utils\src\VisualTestUtils.MagickNet\VisualTestUtils.MagickNet.csproj" />
     <ProjectReference Include="$(MauiRootDirectory)..\visual-test-utils\src\VisualTestUtils\VisualTestUtils.csproj" />
   </ItemGroup>
-
+  
+  <ItemGroup>
+    <Content Include="snapshots\**">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+  
   <ItemGroup>
     <Compile Include="..\TestCases.Shared.Tests\**\*.cs" Exclude="..\TestCases.Shared.Tests\obj\**;..\TestCases.Shared.Tests\bin\**" LinkBase="Shared" Visible="false" />
   </ItemGroup>

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/BackgroundGradientsShouldRenderCorrectly.png

@@ -19,6 +19,7 @@ protected override void NavigateToGallery()
 		App.NavigateToGallery(ImageButtonGallery);
 	}
 
+#if TEST_FAILS_ON_CATALYST
 	[Test]
 	public void Aspect()
 	{
@@ -37,6 +38,7 @@ public void Aspect()
 		remote.TapStateButton();
 		VerifyScreenshot("ImageButtonUITests_Aspect_State_AspectFit");
 	}
+#endif
 
 	[Test]
 	public void Aspect_AspectFill()

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/BottomTabColorTest.png

@@ -24,6 +24,7 @@ public override void IsEnabled()
 		Assert.Ignore("Image elements do not really have a concept of being \"disabled\".");
 	}
 
+#if TEST_FAILS_ON_CATALYST
 	[Test]
 	public void Source_FontImageSource()
 	{
@@ -36,6 +37,7 @@ public void Source_FontImageSource()
 		remote.TapStateButton();
 		VerifyScreenshot("ImageUITests_Source_FontImageSource_FontAwesome");
 	}
+#endif
 
 	[Test]
 	public async Task IsAnimationPlaying()

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/Bugzilla30166Test.png

@@ -1,3 +1,4 @@
+#if TEST_FAILS_ON_CATALYST
 using NUnit.Framework;
 using UITest.Appium;
 using UITest.Core;
@@ -58,3 +59,4 @@ void WaitForAllElements()
 		}
 	}
 }
+#endif

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/ButtonLayoutResizesWithImagePositionBottom.png

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+#if TEST_FAILS_ON_CATALYST
+using NUnit.Framework;
 using UITest.Appium;
 using UITest.Core;
 
@@ -27,3 +28,4 @@ public void BorderWidthAffectsTheImageSizing()
 		}
 	}
 }
+#endif

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/ButtonLayoutResizesWithImagePositionLeft.png

@@ -1,3 +1,4 @@
+#if TEST_FAILS_ON_CATALYST
 using NUnit.Framework;
 using UITest.Appium;
 using UITest.Core;
@@ -103,3 +104,4 @@ public void VerifyButtonPage8()
 		}
 	}
 }
+#endif

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/ButtonLayoutResizesWithImagePositionRight.png

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+#if TEST_FAILS_ON_CATALYST
+using NUnit.Framework;
 using UITest.Appium;
 using UITest.Core;
 
@@ -35,3 +36,4 @@ public async Task ButtonLayoutAndSpacingTests()
 		}
 	}
 }
+#endif

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/ButtonLayoutResizesWithImagePositionTop.png

@@ -11,6 +11,7 @@ public Issue22606(TestDevice device) : base(device) { }
 
 		public override string Issue => "Border does not expand on Content size changed";
 
+#if !MACCATALYST
 		[Test]
 		public void BorderBackgroundExpandsOnContentSizeChanged()
 		{
@@ -21,6 +22,7 @@ public void BorderBackgroundExpandsOnContentSizeChanged()
 			App.Tap("SetHeightTo500");
 			VerifyScreenshot("Issue22606_SetHeightTo500");
 		}
+#endif
 
 #if ANDROID || IOS
 		[Test]

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/ButtonPaddingIsAddedWhenNeeded.png

@@ -1,4 +1,5 @@
-using NUnit.Framework;
+#if TEST_FAILS_ON_CATALYST
+using NUnit.Framework;
 using UITest.Appium;
 using UITest.Core;
 
@@ -20,4 +21,5 @@ public void TextInEditorShouldBeCorrectlyPositionedAfterResizing()
 		App.Click("button");
 		VerifyScreenshot("TextsInEditorsAfterScaling");
 	}
-}
+}
+#endif

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/CVHorizontalLinearItemsLayoutItemSpacing.png

@@ -1,3 +1,4 @@
+#if TEST_FAILS_ON_CATALYST
 using NUnit.Framework;
 using UITest.Appium;
 using UITest.Core;
@@ -23,3 +24,4 @@ public void ButtonResizesWhenTitleOrImageChanges()
 		}
 	}
 }
+#endif

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/CarouselViewShouldRenderCorrectly.png

@@ -1,3 +1,4 @@
+#if TEST_FAILS_ON_CATALYST
 using NUnit.Framework;
 using UITest.Appium;
 using UITest.Core;
@@ -23,3 +24,4 @@ public void ButtonTitleFillsSpaceWhenImageChanges()
 		}
 	}
 }
+#endif

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/CollectionViewHorizontalItemSpacing.png

@@ -1,6 +1,5 @@
 using System.Reflection;
 using NUnit.Framework;
-using OpenQA.Selenium.Appium.iOS;
 using UITest.Appium;
 using UITest.Appium.NUnit;
 using UITest.Core;
@@ -178,11 +177,7 @@ but both can happen.
 						break;
 
 					case TestDevice.Mac:
-						// For now, ignore visual tests on Mac Catalyst since the Appium screenshot on Mac (unlike Windows)
-						// is of the entire screen, not just the app. Later when xharness relay support is in place to
-						// send a message to the MAUI app to get the screenshot, we can use that to just screenshot
-						// the app.
-						Assert.Ignore("MacCatalyst isn't supported yet for visual tests");
+						environmentName = "mac";
 						break;
 
 					default:
@@ -198,7 +193,11 @@ but both can happen.
 					Thread.Sleep(350);
 				}
 
+#if MACUITEST
+				byte[] screenshotPngBytes = TakeScreenshot() ?? throw new InvalidOperationException("Failed to get screenshot");
+#else
 				byte[] screenshotPngBytes = App.Screenshot() ?? throw new InvalidOperationException("Failed to get screenshot");
+#endif
 
 				var actualImage = new ImageSnapshot(screenshotPngBytes, ImageSnapshotFormat.PNG);
 
@@ -254,8 +253,27 @@ public override void TestSetup()
 					Thread.Sleep(1000);
 					App.SetOrientationPortrait();
 				}
-
 			}
 		}
+		
+#if MACUITEST
+		byte[] TakeScreenshot()
+		{
+			// Since the Appium screenshot on Mac (unlike Windows) is of the entire screen, not just the app,
+			// we are going to maximize the App before take the screenshot.
+			App.EnterFullScreen();
+
+			// The app might not be ready to take the screenshot.
+			// Wait a little bit to complete the system animation moving the App Window to FullScreen.
+			Thread.Sleep(1000);
+
+			byte[] screenshotPngBytes = App.Screenshot() ?? throw new InvalidOperationException("Failed to get screenshot");
+
+			// TODO: After take the screenshot, restore the App Window to the previous state.
+			App.ExitFullScreen();
+
+			return screenshotPngBytes;
+		}
+#endif
 	}
 }

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/CollectionViewPreSelectionShouldUpdate.png

@@ -0,0 +1,69 @@
+using UITest.Core;
+
+namespace UITest.Appium;
+
+public class AppiumCatalystSpecificActions : ICommandExecutionGroup
+{
+	const string EnterFullScreenCommand = "enterFullScreen";
+	const string ExitFullScreenCommand = "exitFullScreen";
+	
+	readonly AppiumApp _appiumApp;
+
+	readonly List<string> _commands = new()
+	{
+		EnterFullScreenCommand,
+		ExitFullScreenCommand,
+	};
+
+	public AppiumCatalystSpecificActions(AppiumApp appiumApp)
+	{
+		_appiumApp = appiumApp;
+	}
+
+	public bool IsCommandSupported(string commandName)
+	{
+		return _commands.Contains(commandName, StringComparer.OrdinalIgnoreCase);
+	}
+
+	public CommandResponse Execute(string commandName, IDictionary<string, object> parameters)
+	{
+		return commandName switch
+		{
+			EnterFullScreenCommand => EnterFullScreen(parameters),
+			ExitFullScreenCommand => ExitFullScreen(parameters),
+			_ => CommandResponse.FailedEmptyResponse,
+		};
+	}
+	
+	CommandResponse EnterFullScreen(IDictionary<string, object> parameters)
+	{
+		try
+		{
+			_appiumApp.Driver.Manage().Window.FullScreen();
+
+			return CommandResponse.SuccessEmptyResponse;
+		}
+		catch
+		{
+			return CommandResponse.FailedEmptyResponse;
+		}
+	}
+	
+	CommandResponse ExitFullScreen(IDictionary<string, object> parameters)
+	{
+		try
+		{
+			string[] keys = ["XCUIKeyboardKeyEscape"];
+			_appiumApp.Driver.ExecuteScript("macos: keys", new Dictionary<string, object>
+			{
+				{ "keys", keys },
+			});
+
+			return CommandResponse.SuccessEmptyResponse;
+		}
+		catch
+		{
+			return CommandResponse.FailedEmptyResponse;
+		}
+	}
+}

src/Controls/tests/TestCases.Mac.Tests/snapshots/mac/DynamicTabSectionVisibility.png

@@ -13,6 +13,7 @@ public AppiumCatalystApp(Uri remoteAddress, IConfig config)
 			_commandExecutor.AddCommandGroup(new AppiumCatalystMouseActions(this));
 			_commandExecutor.AddCommandGroup(new AppiumCatalystTouchActions(this));
 			_commandExecutor.AddCommandGroup(new AppiumCatalystAlertActions(this));
+			_commandExecutor.AddCommandGroup(new AppiumCatalystSpecificActions(this));
 			_commandExecutor.AddCommandGroup(new AppiumCatalystVirtualKeyboardActions(this));
 		}