Skip to content

Commit c9934b3

Browse files
peterzhuamazonpeterzhuamazon
authored
Add Windows 2019 ServerCore Docker Images (#3859)
Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
1 parent 1ac48c3 commit c9934b3

File tree

2 files changed

+255
-0
lines changed

2 files changed

+255
-0
lines changed

‎docker/ci/config/windows-servercore-setup.ps1

+224
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
# Copyright OpenSearch Contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# The OpenSearch Contributors require contributions made to
5+
# this file be licensed under the Apache-2.0 license or a
6+
# compatible open source license.
7+
8+
# This script is specifically used on Windows Server Core based docker images
9+
10+
# Set TLS to 1.2 so SSL/TLS can be enabled for downloading artifacts
11+
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
12+
13+
# Install Scoop as Administrator User here
14+
iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
15+
16+
# Disable "current" alias directory as it is not preserved after AMI creation
17+
# Use static path in environment variable
18+
scoop config no_junction true
19+
20+
# Install git
21+
scoop install git
22+
git --version
23+
# Path for git windows usr bin
24+
$fileName = 'nohup.exe'
25+
$fileDir = 'C:\\Users\\ContainerAdministrator\\scoop\\apps\\git'
26+
$fileFound = (Get-ChildItem -Path $fileDir -Filter $fileName -Recurse | %{$_.FullName} | select -first 1)
27+
$fileFound
28+
$gitPathFound = $fileFound.replace("$fileName", '')
29+
$gitPathFound
30+
# Add to EnvVar
31+
$userenv = [System.Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::User)
32+
[System.Environment]::SetEnvironmentVariable("PATH", $userenv + ";$gitPathFound", [System.EnvironmentVariableTarget]::User)
33+
# Make sure mem size are set to avoid "Out of memory, malloc failed" issues on Windows
34+
git config --system core.packedGitLimit 128m
35+
git config --system core.packedGitWindowSize 128m
36+
git config --system core.longPaths true
37+
git config --system pack.deltaCacheSize 128m
38+
git config --system pack.packSizeLimit 128m
39+
git config --system pack.windowMemory 128m
40+
git config --system pack.window 0
41+
git config --system pack.threads 1
42+
git config --system core.compression 0
43+
git config --system protocol.version 1
44+
git config --system --list
45+
# Rename system32 find.exe in case it gets conflicted with POSIX find
46+
bash.exe -c "mv -v 'C:\\Windows\\System32\\find.exe' 'C:\\Windows\\System32\\find_windows.exe'"
47+
48+
# Add some sleep due to a potential race condition
49+
Start-Sleep -Seconds 5
50+
51+
# Setup Repos (This has to happen after git is installed or will error out)
52+
scoop bucket add java
53+
scoop bucket add versions
54+
scoop bucket add extras
55+
scoop bucket add github-gh https://github.com/cli/scoop-gh.git
56+
57+
# Install mingw for k-NN specific requirements with renaming
58+
# Try to lock on to 12.2.0-rt_v10-rev1 as the newer versions on scoop pointed to the ucrt version to replace legacy msvcrt
59+
# https://github.com/opensearch-project/k-NN/issues/829#issuecomment-1499846457
60+
scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/dad0cee42bb2c0be7acf9f341fba2a55e415e0f2/bucket/mingw.json
61+
$libName = 'libgfortran-5.dll'
62+
$libNameRequired = 'libgfortran-3.dll'
63+
$libDir = 'C:\\Users\\ContainerAdministrator\\scoop\\apps\\mingw'
64+
$libFound = (Get-ChildItem -Path $libDir -Filter $libName -Recurse | %{$_.FullName} | select -first 1)
65+
$libFound
66+
$libPathFound = $libFound.replace("$libName", '')
67+
$libPathFound
68+
mv -v "$libFound" "$libPathFound\\$libNameRequired"
69+
# Add MINGW_BIN path to User Env Var for k-NN to retrieve libs
70+
[System.Environment]::SetEnvironmentVariable("MINGW_BIN", "$libPathFound", [System.EnvironmentVariableTarget]::User)
71+
72+
# Install zlib for k-NN compilation requirements
73+
scoop install zlib
74+
# Reg PEP
75+
$zlibVersionInfo = (scoop info zlib | out-string -stream | Select-String 'Version.*:')
76+
$zlibVersionNumber = ($zlibVersionInfo -split ':' | select -last 1)
77+
$zlibVersionNumber = $zlibVersionNumber.Trim()
78+
$zlibHome = "C:\\Users\\ContainerAdministrator\\scoop\\apps\\zlib\\$zlibVersionNumber"
79+
$zlibRegFilePath = "$zlibHome\\register.reg"
80+
$zlibRegFilePath
81+
regedit /s $zlibRegFilePath
82+
83+
# Install jdk
84+
$jdkVersionList = "temurin8-jdk JAVA8_HOME", "temurin11-jdk JAVA11_HOME", "temurin17-jdk JAVA17_HOME", "temurin19-jdk JAVA19_HOME", "openjdk20 JAVA20_HOME", "openjdk14 JAVA14_HOME"
85+
Foreach ($jdkVersion in $jdkVersionList)
86+
{
87+
$jdkVersion
88+
$jdkArray = $jdkVersion.Split(" ")
89+
$jdkArray[0]
90+
$jdkArray[1]
91+
scoop install $jdkArray[0]
92+
$JAVA_HOME_TEMP = [System.Environment]::GetEnvironmentVariable("JAVA_HOME", [System.EnvironmentVariableTarget]::User).replace("\", "/")
93+
$JAVA_HOME_TEMP
94+
[System.Environment]::SetEnvironmentVariable($jdkArray[1], "$JAVA_HOME_TEMP", [System.EnvironmentVariableTarget]::User)
95+
java -version
96+
}
97+
# Need to reset to jdk11 for Jenkins Agent to start
98+
scoop reset temurin11-jdk
99+
$JAVA_HOME_TEMP = [System.Environment]::GetEnvironmentVariable("JAVA_HOME", [System.EnvironmentVariableTarget]::User).replace("\", "/")
100+
$JAVA_HOME_TEMP
101+
[System.Environment]::SetEnvironmentVariable('JAVA_HOME', "$JAVA_HOME_TEMP", [System.EnvironmentVariableTarget]::User)
102+
java -version
103+
104+
# Install python and lock onto 3.9.13 now
105+
scoop install https://raw.githubusercontent.com/ScoopInstaller/Versions/13510e2f3fb92b696c203de5a008c0dc62a44220/bucket/python39.json
106+
python --version
107+
# Reg PEP
108+
$versionInfo = (scoop info python39 | out-string -stream | Select-String 'Version.*:')
109+
$versionInfo
110+
$versionNumber = ($versionInfo -split ':' | select -last 1)
111+
$versionNumber
112+
$versionNumber = $versionNumber.Trim()
113+
$versionNumber
114+
$pythonHome = "C:\\Users\\ContainerAdministrator\\scoop\\apps\\python39\\$versionNumber"
115+
$pythonHome
116+
$pythonLibHome = "$pythonHome\\Lib"
117+
$pythonLibHome
118+
$regFilePath = "$pythonHome\\install-pep-514.reg"
119+
$regFilePath
120+
regedit /s $regFilePath
121+
# Windows AMI does not preserve alias directory, copy all the files to an actual directory
122+
New-Item -Path "$pythonHome\\Scripts_temp" -ItemType Directory
123+
Copy-Item -Path "$pythonHome\\Scripts\\*" -Destination "$pythonHome\\Scripts_temp\\"
124+
Remove-Item "$pythonHome\\Scripts" -Force -Recurse
125+
Rename-Item "$pythonHome\\Scripts_temp" "$pythonHome\\Scripts"
126+
# Same as above but different dir
127+
New-Item -Path "$pythonLibHome\\site-packages_temp" -ItemType Directory
128+
Copy-Item -Path "$pythonLibHome\\site-packages\\*" -Destination "$pythonLibHome\\site-packages_temp\\"
129+
Remove-Item "$pythonLibHome\\site-packages" -Force -Recurse
130+
Rename-Item "$pythonLibHome\\site-packages_temp" "$pythonLibHome\\site-packages"
131+
132+
# Install maven
133+
scoop install maven
134+
mvn --version
135+
136+
# Install jq 1.6
137+
scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/96316b49652d5c9672960b39db141962ad4210b0/bucket/jq.json
138+
jq --version
139+
140+
# Install yq 4.34.2
141+
scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/1e9c8b3dfa5ebe7a8c270daf7355aa1db9926e00/bucket/yq.json
142+
yq --version
143+
144+
# Install volta to replace nvm on Windows as Windows is not able to handle symlink after AMI creation
145+
# While Volta is using a fixed location and switch binary version automatically for the Windows Agent
146+
# As of now, volta is locked to version 1.0.8 due to issues installing yarn within docker container
147+
# https://github.com/opensearch-project/opensearch-ci/issues/281#issuecomment-1654424423
148+
scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/49d6f71e5bd7096d49b3286ad02d5d482726b467/bucket/volta.json
149+
volta --version
150+
$nodeVersionList = "10.24.1","14.19.1","14.20.0", "14.20.1", "14.21.3", "16.20.0", "18.16.0"
151+
Foreach ($nodeVersion in $nodeVersionList)
152+
{
153+
$nodeVersion
154+
volta install "node@$nodeVersion"
155+
node -v
156+
}
157+
$ref = "main"
158+
$JSON_BASE = "https://raw.githubusercontent.com/opensearch-project/OpenSearch-Dashboards/$ref/package.json"
159+
$yarnVersion = (curl.exe -s -o- $JSON_BASE | yq.exe -r '.engines.yarn')
160+
$yarnVersion
161+
volta install yarn@$yarnVersion
162+
yarn --version
163+
# Temporarily comment out cypress caching due to performance issues on Windows startup
164+
# The opensearch-dashboards-functional-test repo can still run tests
165+
# as cypress will be installed globally by `npm install` when running integtest.sh
166+
# https://github.com/opensearch-project/opensearch-ci/issues/297
167+
# $cypressVersionList = "5.6.0", "9.5.4", "12.13.0"
168+
# Foreach ($cypressVersion in $cypressVersionList)
169+
# {
170+
# $cypressVersion
171+
# volta install "cypress@$cypressVersion"
172+
# cypress --version
173+
# }
174+
$userenv2 = [System.Environment]::GetEnvironmentVariable("Path", [System.EnvironmentVariableTarget]::User)
175+
$nodePathFixed = "C:\\Users\\ContainerAdministrator\\scoop\\persist\\volta\\appdata\\bin"
176+
[System.Environment]::SetEnvironmentVariable("PATH", $userenv2 + ";$nodePathFixed", [System.EnvironmentVariableTarget]::User)
177+
178+
# Install chromium (internally it is chrome.exe in app directory)
179+
scoop install chromium
180+
$chromiumName = 'chrome.exe'
181+
$chromiumDir = 'C:\\Users\\ContainerAdministrator\\scoop\\apps\\chromium'
182+
$chromiumFound = (Get-ChildItem -Path $chromiumDir -Filter $chromiumName -Recurse | %{$_.FullName} | select -first 1)
183+
$chromiumFound
184+
# Add BROWSER_PATH path to User Env Var for cypress test to retrieve chromium.exe path
185+
[System.Environment]::SetEnvironmentVariable("BROWSER_PATH", "$chromiumFound", [System.EnvironmentVariableTarget]::User)
186+
187+
# Install ruby24
188+
scoop install ruby24
189+
ruby --version
190+
191+
# Install gh
192+
scoop install gh
193+
gh version
194+
195+
# Install dev tools
196+
# Lock to 3.23.3
197+
scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/56eed69c3bf04110e306f77ad45cfc8c1c5bb9bc/bucket/cmake.json
198+
cmake --version
199+
200+
# Install zip
201+
scoop install zip
202+
203+
# Scoop cleanup
204+
scoop cache rm *
205+
206+
# Pip
207+
wget https://bootstrap.pypa.io/get-pip.py -OutFile get-pip.py
208+
python get-pip.py
209+
pip --version
210+
# Install pipenv
211+
pip install pipenv
212+
pipenv --version
213+
# Install awscli
214+
pip install awscli
215+
aws --version
216+
# Cleanup
217+
pip cache remove *
218+
Remove-Item 'get-pip.py' -Force
219+
220+
# Refresh env vars
221+
$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
222+
223+
# Enable Long Path
224+
set-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem LongPathsEnabled -Type DWORD -Value 1 -Force

‎docker/ci/dockerfiles/current/build.windows2019.servercore.x64.dockerfile

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# The OpenSearch Contributors require contributions made to
4+
# this file be licensed under the Apache-2.0 license or a
5+
# compatible open source license.
6+
7+
# This is a docker image specifically for standardize the ci/cd environment
8+
# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards
9+
# Please read the README.md file for all the information before using this dockerfile
10+
11+
# This image can only by built on a Windows2019 server or higher version
12+
13+
ARG ServerCoreRepo=mcr.microsoft.com/windows/servercore
14+
15+
FROM ${ServerCoreRepo}:ltsc2019
16+
17+
USER ContainerAdministrator
18+
19+
COPY config/windows-servercore-setup.ps1 ./
20+
21+
RUN powershell ./windows-servercore-setup.ps1
22+
23+
RUN powershell -Command Remove-Item './windows-servercore-setup.ps1' -Force
24+
25+
RUN bash.exe -c "curl https://ci.opensearch.org > /dev/null 2>&1 || echo add certificates"
26+
27+
RUN bash.exe -c "curl https://artifacts.opensearch.org > /dev/null 2>&1 || echo add certificates"
28+
29+
WORKDIR "C:/Users/ContainerAdministrator"
30+
31+
CMD ["powershell.exe"]

0 commit comments

Comments
 (0)