Skip to content

Commit b89f50d

Browse files
tkmcmasterdependabot[bot]
tkmcmaster
and
dependabot[bot]
authored
Update scripting20231128 (#183)
* Bump openssl from 0.10.57 to 0.10.60 (#181) Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.57 to 0.10.60. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](sfackler/rust-openssl@openssl-v0.10.57...openssl-v0.10.60) --- updated-dependencies: - dependency-name: openssl dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Reduced log levels to reduce logging to Splunk * Added the ability to upate the tags on an S3File * Added code so that the delete API will also delete the files from S3 - We're changing the recurring tests to not be automatically deleted after 6 months so we need to have the delete schedule delete the files from s3 * Added code to re-tag files on the last run of a recurring set * Updated the postTest recurring to not tag for deletion - postTest with a recurring schedule will add the tag recurring=true instead of test=true - Changing from recurring to non-recurring or vice-versa will update the tags accordingly - Recurring tests should no longer delete after 6 months * Fixed some issues from the previous shadow variable bug * Cleaned up a ton of shadow variable bugs * Cleaned up more shadow variables * Fixed the last of the shadow variable bugs and added the lint rule * Updated version and dependencies * Updated additional dependencies: typescript, next, storybook, etc * Update 2023-11-28 (#182) * Reduced log levels to reduce logging to Splunk * Added the ability to upate the tags on an S3File * Added code so that the delete API will also delete the files from S3 - We're changing the recurring tests to not be automatically deleted after 6 months so we need to have the delete schedule delete the files from s3 * Added code to re-tag files on the last run of a recurring set * Updated the postTest recurring to not tag for deletion - postTest with a recurring schedule will add the tag recurring=true instead of test=true - Changing from recurring to non-recurring or vice-versa will update the tags accordingly - Recurring tests should no longer delete after 6 months * Fixed some issues from the previous shadow variable bug * Cleaned up a ton of shadow variable bugs * Cleaned up more shadow variables * Fixed the last of the shadow variable bugs and added the lint rule * Updated version and dependencies * Updated additional dependencies: typescript, next, storybook, etc * Fixed the copy s3File to use the new tags rather than always re-using the old tags * Updated the remove test code to return a 500 error if we can't delete any of the files from s3 --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
1 parent b396e79 commit b89f50d

33 files changed

+3189
-3644
lines changed

‎.eslintrc

+2
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@
6868
"camelcase": 1,
6969
"no-irregular-whitespace": 1,
7070
"object-shorthand": 1,
71+
"no-shadow": "off",
72+
"@typescript-eslint/no-shadow": "warn",
7173
"@typescript-eslint/await-thenable": 1,
7274
"quotes": ["warn", "double"]
7375
}

‎Cargo.lock

+4-4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎agent/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@fs/ppaas-agent",
3-
"version": "3.0.2",
3+
"version": "3.0.3",
44
"description": "Agent Service for running pewpew tests",
55
"main": "dist/src/app.js",
66
"scripts": {
@@ -58,6 +58,6 @@
5858
"eslint": "^8.40.0",
5959
"mocha": "^10.2.0",
6060
"nyc": "^15.1.0",
61-
"typescript": "~5.2.0"
61+
"typescript": "~5.3.0"
6262
}
6363
}

‎agent/src/pewpewtest.ts

+14-14
Original file line numberDiff line numberDiff line change
@@ -102,27 +102,27 @@ export function copyTestStatus (ppaasTestStatus: PpaasTestStatus, s3Status: Test
102102
}
103103

104104
/** instanceId doesn't change, so we'll save it after the first call as the instanceId or an Error */
105-
let instanceId: string | Error | undefined;
105+
let globalInstanceId: string | Error | undefined;
106106
async function getInstanceId (): Promise<string> {
107-
if (typeof instanceId === "string") {
108-
log("instanceId string: " + instanceId, LogLevel.DEBUG, instanceId);
109-
return instanceId;
107+
if (typeof globalInstanceId === "string") {
108+
log("instanceId string: " + globalInstanceId, LogLevel.DEBUG, globalInstanceId);
109+
return globalInstanceId;
110110
}
111-
if (instanceId instanceof Error) {
112-
log("instanceId instanceof Error", LogLevel.DEBUG, instanceId);
113-
throw instanceId;
111+
if (globalInstanceId instanceof Error) {
112+
log("instanceId instanceof Error", LogLevel.DEBUG, globalInstanceId);
113+
throw globalInstanceId;
114114
}
115115

116116
try {
117-
log("instanceId getInstanceId", LogLevel.DEBUG, instanceId);
118-
instanceId = await ec2.getInstanceId();
119-
log("instanceId new string: " + instanceId, LogLevel.DEBUG, instanceId);
117+
log("instanceId getInstanceId", LogLevel.DEBUG, globalInstanceId);
118+
globalInstanceId = await ec2.getInstanceId();
119+
log("instanceId new string: " + globalInstanceId, LogLevel.DEBUG, globalInstanceId);
120120
} catch (error) {
121-
instanceId = error;
122-
log("instanceId new Error: " + instanceId, LogLevel.DEBUG, instanceId);
123-
throw instanceId;
121+
globalInstanceId = error;
122+
log("instanceId new Error: " + globalInstanceId, LogLevel.DEBUG, globalInstanceId);
123+
throw globalInstanceId;
124124
}
125-
return instanceId;
125+
return globalInstanceId;
126126
}
127127
if (IS_RUNNING_IN_AWS) {
128128
getInstanceId().catch((error: unknown) => log("Could not retrieve instanceId", LogLevel.ERROR, error));

‎common/integration/s3.spec.ts

+39-27
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,36 @@ export const UNIT_TEST_LOCAL_FILE_LOCATION: string = process.env.UNIT_TEST_LOCAL
4848
export const MAX_POLL_WAIT: number = parseInt(process.env.MAX_POLL_WAIT || "0", 10) || 500;
4949
// const LARGE_FILE_SIZE: number = parseInt(process.env.LARGE_FILE_SIZE || "0", 10) || 500000000;
5050

51-
export const tagKey: string = "unittest";
52-
export const tagValue: string = "true";
53-
export const testTags = new Map<string, string>([[tagKey, tagValue]]);
51+
export const unittestTagKey: string = "unittest";
52+
export const unittestTagValue: string = "true";
53+
export const testTags = new Map<string, string>([[unittestTagKey, unittestTagValue]]);
5454
// These are set by the before after init()
5555
export const defaultTags = new Map<string, string>();
5656
export const fullTestTags = new Map<string, string>([...testTags]);
5757

58+
export function initTags (): string {
59+
let defaultKey: string | undefined;
60+
let defaultValue: string | undefined;
61+
if (ADDITIONAL_TAGS_ON_ALL.size > 0) {
62+
for (const [key, value] of ADDITIONAL_TAGS_ON_ALL) {
63+
if (!defaultKey) {
64+
defaultKey = key;
65+
defaultValue = value;
66+
}
67+
defaultTags.set(key, value);
68+
fullTestTags.set(key, value);
69+
}
70+
} else {
71+
defaultKey = "application";
72+
defaultValue = util.APPLICATION_NAME;
73+
defaultTags.set(defaultKey, defaultValue);
74+
fullTestTags.set(defaultKey, defaultValue);
75+
}
76+
log("tags", LogLevel.DEBUG, { defaultKey, tags: Array.from(testTags.entries()), defaultTags: Array.from(defaultTags.entries()), allTags: Array.from(fullTestTags.entries()) });
77+
expect(defaultKey, "defaultKey").to.not.equal(undefined);
78+
return defaultKey!;
79+
}
80+
5881
export const validateTagMap = (actual: Map<string, string>, expected: Map<string, string>) => {
5982
try {
6083
expect(actual.size, "validateTagMap actual.size").to.equal(expected.size);
@@ -87,29 +110,14 @@ export const validateTagSet = (actual: S3Tag[], expected: Map<string, string>) =
87110
describe("S3Util Integration", () => {
88111
let s3FileKey: string | undefined;
89112
let healthCheckDate: Date | undefined;
90-
let tagKey: string;
91-
let tagValue: string;
113+
let defaultTagKey: string;
92114

93115
before (async () => {
94116
// This test was failing until we reset everything. I don't know why and it bothers me.
95117
s3Config.s3Client = undefined as any;
96118
initS3();
97-
if (ADDITIONAL_TAGS_ON_ALL.size > 0) {
98-
for (const [key, value] of ADDITIONAL_TAGS_ON_ALL) {
99-
if (!tagKey) {
100-
tagKey = key;
101-
tagValue = value;
102-
}
103-
defaultTags.set(key, value);
104-
fullTestTags.set(key, value);
105-
}
106-
} else {
107-
tagKey = "application";
108-
tagValue = util.APPLICATION_NAME;
109-
defaultTags.set(tagKey, tagValue);
110-
fullTestTags.set(tagKey, tagValue);
111-
}
112-
log("tags", LogLevel.DEBUG, { tags: Array.from(testTags.entries()), defaultTags: Array.from(defaultTags.entries()), allTags: Array.from(fullTestTags.entries()) });
119+
defaultTagKey = initTags();
120+
expect(defaultTagKey, "defaultTagKey").to.not.equal(undefined);
113121
// Set the access callback to test that healthchecks will be updated
114122
setAccessCallback((date: Date) => healthCheckDate = date);
115123
try {
@@ -685,7 +693,7 @@ describe("S3Util Integration", () => {
685693
};
686694

687695
// Change tags
688-
expectedTags.set(tagKey, "pewpewagent");
696+
expectedTags.set(defaultTagKey, "pewpewagent");
689697
expectedTags.set("unittest", "false");
690698
expectedTags.set("additionaltag", "additionalvalue");
691699
const tags = new Map(expectedTags);
@@ -794,8 +802,8 @@ describe("S3Util Integration", () => {
794802
it("Copy File should change name", (done: Mocha.Done) => {
795803
if (s3FileKey) {
796804
// Change tags
797-
expectedTags.set(tagKey, "pewpewagent");
798-
expectedTags.set("unittest", "false");
805+
expectedTags.set(defaultTagKey, "pewpewagent");
806+
expectedTags.set(unittestTagKey, "false");
799807
expectedTags.set("additionaltag", "additionalvalue");
800808
const tags = new Map(expectedTags);
801809
expect(expectedTags.size, "expectedTags.size before").to.equal(3); // Make sure there aren't some others we don't know about
@@ -939,7 +947,7 @@ describe("S3Util Integration", () => {
939947
try {
940948
const uploadTags = new Map(testTags);
941949
// Change it so clearing will set it back
942-
uploadTags.set(tagKey, "pewpewagent");
950+
uploadTags.set(defaultTagKey, "pewpewagent");
943951
const url: string = await uploadFile({
944952
filepath: UNIT_TEST_FILEPATH,
945953
s3Folder,
@@ -961,7 +969,7 @@ describe("S3Util Integration", () => {
961969
validateTagSet(tagging.TagSet!, uploadTags);
962970
expectedTags = new Map(uploadTags);
963971
} catch (error) {
964-
log("copyObject beforeEach error", LogLevel.ERROR, error);
972+
log("putObjectTagging beforeEach error", LogLevel.ERROR, error);
965973
throw error;
966974
}
967975
});
@@ -974,13 +982,14 @@ describe("S3Util Integration", () => {
974982
validateTagSet(tagging.TagSet!, expectedTags);
975983
}
976984
} catch (error) {
977-
log("copyObject beforeEach error", LogLevel.ERROR, error);
985+
log("putObjectTagging beforeEach error", LogLevel.ERROR, error);
978986
throw error;
979987
}
980988
});
981989

982990
it("putObjectTagging should put a tag", (done: Mocha.Done) => {
983991
expectedTags.set("additionalTag", "additionalValue");
992+
log("putObjectTagging should put a tag", LogLevel.DEBUG, expectedTags);
984993
const tags = new Map(expectedTags);
985994
putObjectTagging({ key: s3FileKey!, tags }).then((result: PutObjectTaggingCommandOutput) => {
986995
expect(result).to.not.equal(undefined);
@@ -990,6 +999,7 @@ describe("S3Util Integration", () => {
990999

9911000
it("putTags should put a tag", (done: Mocha.Done) => {
9921001
expectedTags.set("additionalTag", "additionalValue");
1002+
log("putTags should put a tag", LogLevel.DEBUG, expectedTags);
9931003
const tags = new Map(expectedTags);
9941004
putTags({ filename, s3Folder, tags }).then(() => {
9951005
done();
@@ -999,6 +1009,7 @@ describe("S3Util Integration", () => {
9991009
it("putObjectTagging should clear tags", (done: Mocha.Done) => {
10001010
const tags = new Map();
10011011
expectedTags = new Map(defaultTags); // default will be set back
1012+
log("putObjectTagging should clear tags", LogLevel.DEBUG, expectedTags);
10021013
putObjectTagging({ key: s3FileKey!, tags }).then((result: PutObjectTaggingCommandOutput) => {
10031014
expect(result).to.not.equal(undefined);
10041015
done();
@@ -1008,6 +1019,7 @@ describe("S3Util Integration", () => {
10081019
it("putTags should clear tags", (done: Mocha.Done) => {
10091020
const tags = new Map();
10101021
expectedTags = new Map(defaultTags); // default will be set back
1022+
log("putTags should clear tags", LogLevel.DEBUG, expectedTags);
10111023
putTags({ filename, s3Folder, tags }).then(() => {
10121024
done();
10131025
}).catch((error) => done(error));

‎common/integration/s3file.spec.ts

+64-2
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,33 @@
11
import * as path from "path";
22
import {
3+
ADDITIONAL_TAGS_ON_ALL,
34
BUCKET_URL,
45
KEYSPACE_PREFIX,
56
deleteObject,
67
getObjectTagging,
78
listFiles
89
} from "../src/util/s3";
10+
import { GetObjectTaggingCommandOutput, _Object as S3Object } from "@aws-sdk/client-s3";
911
import {
1012
LogLevel,
1113
PpaasS3File,
1214
PpaasS3FileOptions,
1315
PpaasTestId,
14-
log
16+
log,
17+
s3
1518
} from "../src/index";
1619
import {
1720
MAX_POLL_WAIT,
1821
UNIT_TEST_FILENAME,
1922
UNIT_TEST_FILEPATH,
2023
UNIT_TEST_LOCAL_FILE_LOCATION,
24+
defaultTags,
2125
fullTestTags,
26+
initTags,
2227
testTags,
2328
validateTagMap,
2429
validateTagSet
2530
} from "./s3.spec";
26-
import { _Object as S3Object } from "@aws-sdk/client-s3";
2731
import { Stats } from "fs";
2832
import { expect } from "chai";
2933
import fs from "fs/promises";
@@ -76,6 +80,12 @@ describe("PpaasS3File Integration", () => {
7680
localDirectory: UNIT_TEST_LOCAL_FILE_LOCATION,
7781
tags: testTags
7882
});
83+
if (ADDITIONAL_TAGS_ON_ALL.size > 0) {
84+
for (const [key, value] of ADDITIONAL_TAGS_ON_ALL) {
85+
defaultTags.set(key, value);
86+
fullTestTags.set(key, value);
87+
}
88+
}
7989
expectedTags = new Map(fullTestTags);
8090
});
8191

@@ -455,4 +465,56 @@ describe("PpaasS3File Integration", () => {
455465
}
456466
});
457467
});
468+
469+
describe("Update Tagging in S3", () => {
470+
const clearedTags = new Map<string, string>();
471+
472+
beforeEach (async () => {
473+
try {
474+
const defaultTagKey = initTags();
475+
const uploadTags = new Map(testTags);
476+
// Change it so we verify clearing won't set it back
477+
uploadTags.set(defaultTagKey, "pewpewagent");
478+
clearedTags.set(defaultTagKey, "pewpewagent");
479+
testPpaasS3FileUpload.tags = uploadTags;
480+
await testPpaasS3FileUpload.upload(true);
481+
log("testPpaasS3FileUpload.upload() succeeded", LogLevel.DEBUG);
482+
s3FileKey = testPpaasS3FileUpload.key;
483+
// As long as we don't throw, it passes
484+
// Need time for eventual consistency to complete
485+
await poll(async (): Promise<boolean | undefined> => {
486+
const objects = await s3.listObjects(s3FileKey!);
487+
return (objects && objects.Contents && objects.Contents.length > 0);
488+
}, MAX_POLL_WAIT, (errMsg: string) => `${errMsg} Could not find the ${s3FileKey} in s3`);
489+
const expectedObject = await s3.getObject(s3FileKey);
490+
expect(expectedObject, "actualObject").to.not.equal(undefined);
491+
expect(expectedObject.TagCount, "TagCount").to.equal(uploadTags.size);
492+
const tagging: GetObjectTaggingCommandOutput = await getObjectTagging(s3FileKey);
493+
expect(tagging.TagSet, "tagging.TagSet").to.not.equal(undefined);
494+
validateTagSet(tagging.TagSet!, uploadTags);
495+
expectedTags = new Map(uploadTags);
496+
} catch (error) {
497+
log("updateTags beforeEach error", LogLevel.ERROR, error);
498+
throw error;
499+
}
500+
});
501+
502+
it("updateTags should put a tag", (done: Mocha.Done) => {
503+
expectedTags.set("additionalTag", "additionalValue");
504+
log("updateTags should put a tag", LogLevel.DEBUG, expectedTags);
505+
testPpaasS3FileUpload.tags = new Map(expectedTags);
506+
testPpaasS3FileUpload.updateTags().then(() => {
507+
done();
508+
}).catch((error) => done(error));
509+
});
510+
511+
it("updateTags should clear tags", (done: Mocha.Done) => {
512+
testPpaasS3FileUpload.tags = new Map();
513+
expectedTags = new Map(clearedTags); // default will be set back
514+
log("updateTags should clear tags", LogLevel.DEBUG, expectedTags);
515+
testPpaasS3FileUpload.updateTags().then(() => {
516+
done();
517+
}).catch((error) => done(error));
518+
});
519+
});
458520
});

‎common/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@fs/ppaas-common",
3-
"version": "3.0.2",
3+
"version": "3.0.3",
44
"description": "Common Code for the PewPewController and PewPewAgent",
55
"main": "dist/src/index.js",
66
"types": "dist/src/index.d.ts",
@@ -56,6 +56,6 @@
5656
"eslint": "^8.40.0",
5757
"mocha": "^10.2.0",
5858
"nyc": "^15.1.0",
59-
"typescript": "~5.2.0"
59+
"typescript": "~5.3.0"
6060
}
6161
}

0 commit comments

Comments
 (0)