Skip to content

Commit e6dc4b0

Browse files
KevinFairise2CelianR
KevinFairise2
and
CelianR
authored
Allow installing package from the filesystem (#1456)
Co-authored-by: Célian Raimbault <161456554+CelianR@users.noreply.github.com>
1 parent 8dddad9 commit e6dc4b0

15 files changed

+175
-25
lines changed

‎common/config/environment.go

+5
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ const (
4040
DDAgentVersionParamName = "version"
4141
DDAgentFlavorParamName = "flavor"
4242
DDAgentPipelineID = "pipeline_id"
43+
DDAgentLocalPackage = "localPackage"
4344
DDAgentCommitSHA = "commit_sha"
4445
DDAgentFullImagePathParamName = "fullImagePath"
4546
DDClusterAgentVersionParamName = "clusterAgentVersion"
@@ -109,6 +110,7 @@ type Env interface {
109110
AgentDeploy() bool
110111
AgentVersion() string
111112
AgentFIPS() bool
113+
AgentLocalPackage() string
112114
PipelineID() string
113115
CommitSHA() string
114116
ClusterAgentVersion() string
@@ -271,6 +273,9 @@ func (e *CommonEnvironment) AgentFlavor() string {
271273
return e.AgentConfig.Get(DDAgentFlavorParamName)
272274
}
273275

276+
func (e *CommonEnvironment) AgentLocalPackage() string {
277+
return e.AgentConfig.Get(DDAgentLocalPackage)
278+
}
274279
func (e *CommonEnvironment) PipelineID() string {
275280
return e.AgentConfig.Get(DDAgentPipelineID)
276281
}

‎components/command/filemanager.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,12 @@ func (fm *FileManager) CopyFile(name string, localPath, remotePath pulumi.String
7070

7171
func (fm *FileManager) CopyInlineFile(fileContent pulumi.StringInput, remotePath string, opts ...pulumi.ResourceOption) (pulumi.Resource, error) {
7272
// Write the content into a temporary file and get the path
73-
localFilePath := fileContent.ToStringOutput().ApplyT(func(content string) (string, error) {
73+
tempFile, err := os.CreateTemp("", filepath.Base(remotePath))
74+
if err != nil {
75+
return nil, err
76+
}
7477

75-
tempFile, err := os.CreateTemp("", filepath.Base(remotePath))
78+
_ = fileContent.ToStringOutput().ApplyT(func(content string) (string, error) {
7679
if err != nil {
7780
return "", err
7881
}
@@ -86,7 +89,7 @@ func (fm *FileManager) CopyInlineFile(fileContent pulumi.StringInput, remotePath
8689
return tempFilePath, nil
8790
}).(pulumi.StringInput)
8891

89-
return fm.CopyFile(remotePath, localFilePath, pulumi.String(remotePath), opts...)
92+
return fm.CopyFile(remotePath, pulumi.String(tempFile.Name()), pulumi.String(remotePath), opts...)
9093
}
9194

9295
// CopyRelativeFolder copies recursively a relative folder to a remote folder.

‎components/command/unixOSCommand.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -128,9 +128,12 @@ func (fs unixOSCommand) copyRemoteFile(runner *RemoteRunner, name string, src, d
128128
return fs.PathJoin(runner.OsCommand().GetTemporaryDirectory(), filepath.Base(path))
129129
}).(pulumi.StringOutput)
130130

131-
tempCopyFile, err := remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{
131+
srcAsset := src.ToStringOutput().ApplyT(func(path string) pulumi.AssetOrArchive {
132+
return pulumi.NewFileAsset(path)
133+
}).(pulumi.AssetOrArchiveOutput)
134+
tempCopyFile, err := remote.NewCopyToRemote(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyToRemoteArgs{
132135
Connection: runner.Config().connection,
133-
LocalPath: src,
136+
Source: srcAsset,
134137
RemotePath: tempRemotePath,
135138
Triggers: pulumi.Array{src, tempRemotePath},
136139
}, utils.MergeOptions(runner.PulumiOptions(), opts...)...)

‎components/command/windowsOSCommand.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,13 @@ func (fs windowsOSCommand) copyLocalFile(runner *LocalRunner, name string, src,
109109
}
110110

111111
func (fs windowsOSCommand) copyRemoteFile(runner *RemoteRunner, name string, src, dst pulumi.StringInput, opts ...pulumi.ResourceOption) (pulumi.Resource, error) {
112-
return remote.NewCopyFile(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyFileArgs{
112+
srcAsset := src.ToStringOutput().ApplyT(func(path string) pulumi.AssetOrArchive {
113+
return pulumi.NewFileAsset(path)
114+
}).(pulumi.AssetOrArchiveOutput)
115+
116+
return remote.NewCopyToRemote(runner.Environment().Ctx(), runner.Namer().ResourceName("copy", name), &remote.CopyToRemoteArgs{
113117
Connection: runner.Config().connection,
114-
LocalPath: src,
118+
Source: srcAsset,
115119
RemotePath: dst,
116120
Triggers: pulumi.Array{src, dst},
117121
}, utils.MergeOptions(runner.PulumiOptions(), opts...)...)

‎components/datadog/agent/host.go

+77-3
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ package agent
22

33
import (
44
"fmt"
5+
"io/fs"
6+
"os"
57
"path"
8+
"strings"
69

710
"github.com/DataDog/datadog-agent/pkg/util/option"
811
"github.com/DataDog/test-infra-definitions/common/config"
@@ -12,6 +15,7 @@ import (
1215
"github.com/DataDog/test-infra-definitions/components/command"
1316
"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
1417
perms "github.com/DataDog/test-infra-definitions/components/datadog/agentparams/filepermissions"
18+
tifos "github.com/DataDog/test-infra-definitions/components/os"
1519
remoteComp "github.com/DataDog/test-infra-definitions/components/remote"
1620

1721
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
@@ -69,10 +73,10 @@ func NewHostAgent(e config.Env, host *remoteComp.Host, options ...agentparams.Op
6973
return hostInstallComp, nil
7074
}
7175

72-
func (h *HostAgent) installAgent(env config.Env, params *agentparams.Params, baseOpts ...pulumi.ResourceOption) error {
76+
func (h *HostAgent) installScriptInstallation(env config.Env, params *agentparams.Params, baseOpts ...pulumi.ResourceOption) (command.Command, error) {
7377
installCmdStr, err := h.manager.getInstallCommand(params.Version, params.AdditionalInstallParameters)
7478
if err != nil {
75-
return err
79+
return nil, err
7680
}
7781

7882
installCmd, err := h.Host.OS.Runner().Command(
@@ -81,7 +85,77 @@ func (h *HostAgent) installAgent(env config.Env, params *agentparams.Params, bas
8185
Create: pulumi.Sprintf(installCmdStr, env.AgentAPIKey()),
8286
}, baseOpts...)
8387
if err != nil {
84-
return err
88+
return nil, err
89+
}
90+
return installCmd, nil
91+
}
92+
93+
func (h *HostAgent) directInstallInstallation(env config.Env, params *agentparams.Params, baseOpts ...pulumi.ResourceOption) (command.Command, error) {
94+
var wantedExt string
95+
switch h.Host.OS.Descriptor().Flavor {
96+
case tifos.AmazonLinux, tifos.CentOS, tifos.RedHat, tifos.AmazonLinuxECS, tifos.Fedora, tifos.Suse, tifos.RockyLinux:
97+
wantedExt = ".rpm"
98+
case tifos.Debian, tifos.Ubuntu:
99+
wantedExt = ".deb"
100+
case tifos.WindowsServer:
101+
wantedExt = ".msi"
102+
case tifos.MacosOS, tifos.Unknown:
103+
fallthrough
104+
default:
105+
return nil, fmt.Errorf("unsupported flavor for local packages installation: %s", h.Host.OS.Descriptor().Flavor)
106+
}
107+
108+
pathInfo, err := os.Stat(params.Version.LocalPath)
109+
if err != nil {
110+
return nil, err
111+
}
112+
packagePath := params.Version.LocalPath
113+
matches := []string{}
114+
if pathInfo.IsDir() {
115+
matches, err = fs.Glob(os.DirFS(params.Version.LocalPath), fmt.Sprintf("*%s", wantedExt))
116+
if err != nil {
117+
return nil, err
118+
}
119+
if len(matches) == 0 {
120+
return nil, fmt.Errorf("no package found in %s with extension %s", params.Version.LocalPath, wantedExt)
121+
}
122+
if len(matches) > 1 {
123+
env.Ctx().Log.Warn(fmt.Sprintf("Found multiple packages to install, using the first one: %s", matches[0]), nil)
124+
}
125+
packagePath = path.Join(packagePath, matches[0])
126+
} else {
127+
if strings.HasSuffix(params.Version.LocalPath, wantedExt) {
128+
matches = append(matches, path.Base(params.Version.LocalPath))
129+
} else {
130+
return nil, fmt.Errorf("local package %s does not have the expected extension %s", params.Version.LocalPath, wantedExt)
131+
}
132+
}
133+
packageToInstall := matches[0]
134+
env.Ctx().Log.Info(fmt.Sprintf("Found local package to install %s", packageToInstall), nil)
135+
uploadCmd, err := h.Host.OS.FileManager().CopyFile("copy-agent-package", pulumi.String(packagePath), pulumi.String("./"), baseOpts...)
136+
if err != nil {
137+
return nil, err
138+
}
139+
140+
installCmd, err := h.manager.directInstallCommand(env, packageToInstall, params.Version, []string{}, utils.MergeOptions(baseOpts, utils.PulumiDependsOn(uploadCmd))...)
141+
if err != nil {
142+
return nil, err
143+
}
144+
return installCmd, nil
145+
}
146+
func (h *HostAgent) installAgent(env config.Env, params *agentparams.Params, baseOpts ...pulumi.ResourceOption) error {
147+
var installCmd pulumi.Resource
148+
var err error
149+
if params.Version.LocalPath != "" {
150+
installCmd, err = h.directInstallInstallation(env, params, baseOpts...)
151+
if err != nil {
152+
return err
153+
}
154+
} else {
155+
installCmd, err = h.installScriptInstallation(env, params, baseOpts...)
156+
if err != nil {
157+
return err
158+
}
85159
}
86160

87161
afterInstallOpts := utils.MergeOptions(baseOpts, utils.PulumiDependsOn(installCmd))

‎components/datadog/agent/host_linuxos.go

+5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"strings"
66

7+
"github.com/DataDog/test-infra-definitions/common/config"
78
"github.com/DataDog/test-infra-definitions/components/command"
89
"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
910
"github.com/DataDog/test-infra-definitions/components/os"
@@ -20,6 +21,10 @@ func newLinuxManager(host *remoteComp.Host) agentOSManager {
2021
return &agentLinuxManager{targetOS: host.OS}
2122
}
2223

24+
func (am *agentLinuxManager) directInstallCommand(_ config.Env, packagePath string, _ agentparams.PackageVersion, _ []string, opts ...pulumi.ResourceOption) (command.Command, error) {
25+
return am.targetOS.PackageManager().Ensure("./"+packagePath, nil, "", opts...)
26+
}
27+
2328
func (am *agentLinuxManager) getInstallCommand(version agentparams.PackageVersion, _ []string) (string, error) {
2429
var commandLine string
2530
testEnvVars := []string{}

‎components/datadog/agent/host_os.go

+2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package agent
33
import (
44
"fmt"
55

6+
"github.com/DataDog/test-infra-definitions/common/config"
67
"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
78

89
"github.com/DataDog/test-infra-definitions/components/command"
@@ -14,6 +15,7 @@ import (
1415

1516
// internal interface to be able to provide the different OS-specific commands
1617
type agentOSManager interface {
18+
directInstallCommand(env config.Env, packagePath string, version agentparams.PackageVersion, additionalInstallParameters []string, opts ...pulumi.ResourceOption) (command.Command, error)
1719
getInstallCommand(version agentparams.PackageVersion, additionalInstallParameters []string) (string, error)
1820
getAgentConfigFolder() string
1921
restartAgentServices(transform command.Transformer, opts ...pulumi.ResourceOption) (command.Command, error)

‎components/datadog/agent/host_windowsos.go

+45-15
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313

1414
"golang.org/x/exp/slices"
1515

16+
"github.com/DataDog/test-infra-definitions/common/config"
1617
"github.com/DataDog/test-infra-definitions/components/command"
1718
"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
1819
remoteComp "github.com/DataDog/test-infra-definitions/components/remote"
@@ -31,12 +32,55 @@ func newWindowsManager(host *remoteComp.Host) agentOSManager {
3132
return &agentWindowsManager{host: host}
3233
}
3334

35+
func (am *agentWindowsManager) directInstallCommand(env config.Env, packagePath string, version agentparams.PackageVersion, additionalInstallParameters []string, opts ...pulumi.ResourceOption) (command.Command, error) {
36+
cmd := fmt.Sprintf(`
37+
$ProgressPreference = 'SilentlyContinue';
38+
$ErrorActionPreference = 'Stop';
39+
`)
40+
installCommandStr, err := am.getInstallPackageCommand(packagePath, version, additionalInstallParameters)
41+
if err != nil {
42+
return nil, err
43+
}
44+
cmd += installCommandStr
45+
return am.host.OS.Runner().Command("install-agent", &command.Args{Create: pulumi.Sprintf(cmd, env.AgentAPIKey())}, opts...)
46+
}
47+
3448
func (am *agentWindowsManager) getInstallCommand(version agentparams.PackageVersion, additionalInstallParameters []string) (string, error) {
3549
url, err := getAgentURL(version)
3650
if err != nil {
3751
return "", err
3852
}
3953

54+
cmd := ""
55+
if version.Flavor == agentparams.FIPSFlavor {
56+
cmd = fmt.Sprint(`
57+
Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Lsa\FipsAlgorithmPolicy' -Name 'Enabled' -Value 1 -Type DWORD`)
58+
}
59+
60+
localFilename := `C:\datadog-agent.msi`
61+
cmd += fmt.Sprintf(`
62+
$ProgressPreference = 'SilentlyContinue';
63+
$ErrorActionPreference = 'Stop';
64+
for ($i=0; $i -lt 3; $i++) {
65+
try {
66+
(New-Object Net.WebClient).DownloadFile('%s','%s')
67+
} catch {
68+
if ($i -eq 2) {
69+
throw
70+
}
71+
}
72+
};
73+
`, url, localFilename)
74+
installPackageCommandStr, err := am.getInstallPackageCommand(localFilename, version, additionalInstallParameters)
75+
if err != nil {
76+
return "", err
77+
}
78+
cmd += installPackageCommandStr
79+
80+
return cmd, nil
81+
}
82+
83+
func (am *agentWindowsManager) getInstallPackageCommand(filePath string, version agentparams.PackageVersion, additionalInstallParameters []string) (string, error) {
4084
logFilePath := "C:\\install.log"
4185
logParamIdx := slices.IndexFunc(additionalInstallParameters, func(s string) bool {
4286
return strings.HasPrefix(s, "/log")
@@ -51,30 +95,16 @@ func (am *agentWindowsManager) getInstallCommand(version agentparams.PackageVers
5195
}
5296
logFilePath = paramParts[1]
5397
}
54-
5598
cmd := ""
5699
if version.Flavor == agentparams.FIPSFlavor {
57100
cmd = fmt.Sprintf(`
58101
Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Lsa\FipsAlgorithmPolicy' -Name 'Enabled' -Value 1 -Type DWORD`)
59102
}
60-
61-
localFilename := `C:\datadog-agent.msi`
62103
cmd += fmt.Sprintf(`
63-
$ProgressPreference = 'SilentlyContinue';
64-
$ErrorActionPreference = 'Stop';
65-
for ($i=0; $i -lt 3; $i++) {
66-
try {
67-
(New-Object Net.WebClient).DownloadFile('%s','%s')
68-
} catch {
69-
if ($i -eq 2) {
70-
throw
71-
}
72-
}
73-
};
74104
$exitCode = (Start-Process -Wait msiexec -PassThru -ArgumentList '/qn /i %s APIKEY=%%s %s').ExitCode
75105
Get-Content %s
76106
Exit $exitCode
77-
`, url, localFilename, localFilename, strings.Join(additionalInstallParameters, " "), logFilePath)
107+
`, filePath, strings.Join(additionalInstallParameters, " "), logFilePath)
78108
return cmd, nil
79109
}
80110

‎components/datadog/agentparams/params.go

+12
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@ func NewParams(env config.Env, options ...Option) (*Params, error) {
7676
if env.AgentFIPS() {
7777
defaultFlavor = WithFlavor(FIPSFlavor)
7878
}
79+
fmt.Println("env.AgentLocalPackage(): ", env.AgentLocalPackage())
80+
if env.AgentLocalPackage() != "" {
81+
defaultFlavor = WithLocalPackage(env.AgentLocalPackage())
82+
}
7983

8084
options = append([]Option{defaultFlavor}, options...)
8185
options = append([]Option{WithMajorVersion(env.MajorVersion())}, options...)
@@ -123,6 +127,14 @@ func WithFlavor(flavor string) func(*Params) error {
123127
}
124128
}
125129

130+
// WithLocalPackage use a local package of the Agent
131+
func WithLocalPackage(path string) func(*Params) error {
132+
return func(p *Params) error {
133+
p.Version.LocalPath = path
134+
return nil
135+
}
136+
}
137+
126138
// WithPipeline use a specific version of the Agent by pipeline id
127139
func WithPipeline(pipelineID string) func(*Params) error {
128140
return func(p *Params) error {

‎components/datadog/agentparams/version.go

+1
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,5 @@ type PackageVersion struct {
2626
Channel channel
2727
PipelineID string
2828
Flavor string // Empty means default (base)
29+
LocalPath string // Local path to the agent packages
2930
}

‎tasks/aws/deploy.py

+2
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ def deploy(
3737
agent_config_path: Optional[str] = None,
3838
agent_env: Optional[str] = None,
3939
helm_config: Optional[str] = None,
40+
local_package: Optional[str] = None,
4041
) -> str:
4142
flags = extra_flags if extra_flags else {}
4243

@@ -95,6 +96,7 @@ def deploy(
9596
agent_config_path,
9697
agent_env,
9798
helm_config,
99+
local_package,
98100
)
99101

100102

‎tasks/aws/vm.py

+3
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
"add_known_host": doc.add_known_host,
4949
"agent_flavor": doc.agent_flavor,
5050
"agent_config_path": doc.agent_config_path,
51+
"local_package": doc.local_package,
5152
}
5253
)
5354
def create_vm(
@@ -72,6 +73,7 @@ def create_vm(
7273
add_known_host: Optional[bool] = True,
7374
agent_flavor: Optional[str] = None,
7475
agent_config_path: Optional[str] = None,
76+
local_package: Optional[str] = None,
7577
) -> None:
7678
"""
7779
Create a new virtual machine on aws.
@@ -116,6 +118,7 @@ def create_vm(
116118
deploy_job=deploy_job,
117119
agent_flavor=agent_flavor,
118120
agent_config_path=agent_config_path,
121+
local_package=local_package,
119122
)
120123

121124
if interactive:

0 commit comments

Comments
 (0)