Skip to content

Commit ac2953f

Browse files
PhilippMaPhilippMa
committed
feat: add functionality for merging files in backend
* organize conversion code in OOP architecture for better code reuse between handling of legacy and external files
1 parent 7e220ee commit ac2953f

10 files changed

+353
-172
lines changed

‎src/ElectronBackend/main/listeners.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import {
3333
} from '../errorHandling/errorHandling';
3434
import { loadInputAndOutputFromFilePath } from '../input/importFromFile';
3535
import { serializeAttributions } from '../input/parseInputData';
36-
import { convertToOpossum } from '../opossum-file/convertToOpossum';
36+
import { convertToOpossum } from '../opossum-file/opossum-file';
3737
import { writeCsvToFile } from '../output/writeCsvToFile';
3838
import { writeSpdxFile } from '../output/writeSpdxFile';
3939
import { GlobalBackendState, OpossumOutputFile } from '../types/types';

‎src/ElectronBackend/opossum-file/ExternalFileConverter.ts

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// SPDX-FileCopyrightText: Meta Platforms, Inc. and its affiliates
2+
// SPDX-FileCopyrightText: TNG Technology Consulting GmbH <https://www.tngtech.com>
3+
//
4+
// SPDX-License-Identifier: Apache-2.0
5+
import { FileConverter } from './FileConverter';
6+
7+
export abstract class ExternalFileConverter extends FileConverter {
8+
protected override preConvertInputFile(_: string): Promise<string | null> {
9+
return new Promise((resolve) => resolve(null));
10+
}
11+
12+
override async convertFile(
13+
pathToInputFile: string,
14+
pathToOpossumFile: string,
15+
): Promise<void> {
16+
try {
17+
await this.execFile(this.OPOSSUM_FILE_EXECUTABLE, [
18+
'generate',
19+
'-o',
20+
pathToOpossumFile,
21+
this.fileTypeSwitch,
22+
pathToInputFile,
23+
]);
24+
} catch (error) {
25+
throw new Error(
26+
`Conversion of ${this.fileTypeName} file to .opossum file failed`,
27+
);
28+
}
29+
}
30+
}

‎src/ElectronBackend/opossum-file/FileConverter.ts

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// SPDX-FileCopyrightText: Meta Platforms, Inc. and its affiliates
2+
// SPDX-FileCopyrightText: TNG Technology Consulting GmbH <https://www.tngtech.com>
3+
//
4+
// SPDX-License-Identifier: Apache-2.0
5+
import { execFile as execFileCallback } from 'child_process';
6+
import { app } from 'electron';
7+
import fs from 'fs';
8+
import { join } from 'path';
9+
import { promisify } from 'util';
10+
11+
export abstract class FileConverter {
12+
protected abstract readonly fileTypeSwitch: string;
13+
protected abstract readonly fileTypeName: string;
14+
15+
protected readonly execFile = promisify(execFileCallback);
16+
17+
protected readonly OPOSSUM_FILE_EXECUTABLE = join(
18+
app?.getAppPath?.() ?? './',
19+
process.env.NODE_ENV === 'e2e' ? '../..' : '',
20+
'bin/opossum-file',
21+
);
22+
23+
protected abstract preConvertInputFile(
24+
pathToInputFile: string,
25+
): Promise<string | null>;
26+
27+
abstract convertFile(
28+
pathToInputFile: string,
29+
pathToOpossumFile: string,
30+
): Promise<void>;
31+
32+
async mergeFiles(
33+
pathToInputFile: string,
34+
pathToOpossumFile: string,
35+
): Promise<void> {
36+
try {
37+
const pathToPreConvertedInputFile =
38+
await this.preConvertInputFile(pathToInputFile);
39+
40+
await this.execFile(this.OPOSSUM_FILE_EXECUTABLE, [
41+
'generate',
42+
'-o',
43+
pathToOpossumFile,
44+
this.fileTypeSwitch,
45+
pathToPreConvertedInputFile || pathToInputFile,
46+
'--opossum',
47+
pathToOpossumFile,
48+
]);
49+
50+
if (pathToPreConvertedInputFile) {
51+
fs.rmSync(pathToPreConvertedInputFile);
52+
}
53+
} catch (error) {
54+
throw new Error(
55+
`Merging ${this.fileTypeName} file into current file failed`,
56+
);
57+
}
58+
}
59+
}

‎src/ElectronBackend/opossum-file/LegacyFileConverter.ts

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// SPDX-FileCopyrightText: Meta Platforms, Inc. and its affiliates
2+
// SPDX-FileCopyrightText: TNG Technology Consulting GmbH <https://www.tngtech.com>
3+
//
4+
// SPDX-License-Identifier: Apache-2.0
5+
import { app } from 'electron';
6+
import fs from 'fs';
7+
import path from 'path';
8+
import zlib from 'zlib';
9+
10+
import {
11+
jsonFileExtension,
12+
jsonGzipFileExtension,
13+
outputFileEnding,
14+
} from '../../Frontend/shared-constants';
15+
import { writeOpossumFile } from '../../shared/write-file';
16+
import { getFilePathWithAppendix } from '../utils/getFilePathWithAppendix';
17+
import { FileConverter } from './FileConverter';
18+
19+
export class LegacyFileConverter extends FileConverter {
20+
readonly fileTypeName: string = 'Legacy Opossum';
21+
readonly fileTypeSwitch: string = '--opossum';
22+
23+
private tryToGetInputFileFromOutputFile(filePath: string): string {
24+
const outputFilePattern = `(${outputFileEnding})$`;
25+
const outputFileRegex = new RegExp(outputFilePattern);
26+
27+
return fs.existsSync(filePath.replace(outputFileRegex, jsonFileExtension))
28+
? filePath.replace(outputFileRegex, jsonFileExtension)
29+
: fs.existsSync(filePath.replace(outputFileRegex, jsonGzipFileExtension))
30+
? filePath.replace(outputFileRegex, jsonGzipFileExtension)
31+
: filePath;
32+
}
33+
34+
private getInputJson(resourceFilePath: string): string {
35+
let inputJson: string;
36+
if (resourceFilePath.endsWith(jsonGzipFileExtension)) {
37+
const file = fs.readFileSync(resourceFilePath);
38+
inputJson = zlib.gunzipSync(file).toString();
39+
} else {
40+
inputJson = fs.readFileSync(resourceFilePath, {
41+
encoding: 'utf-8',
42+
});
43+
}
44+
45+
return inputJson;
46+
}
47+
48+
private getOutputJson(resourceFilePath: string): string | undefined {
49+
const expectedAssociatedAttributionFilePath = getFilePathWithAppendix(
50+
resourceFilePath,
51+
outputFileEnding,
52+
);
53+
if (fs.existsSync(expectedAssociatedAttributionFilePath)) {
54+
return fs.readFileSync(expectedAssociatedAttributionFilePath, {
55+
encoding: 'utf-8',
56+
});
57+
}
58+
59+
return undefined;
60+
}
61+
62+
override async convertFile(
63+
pathToInputFile: string,
64+
pathToOpossumFile: string,
65+
): Promise<void> {
66+
let pathToInputJson = pathToInputFile;
67+
68+
if (pathToInputFile.endsWith(outputFileEnding)) {
69+
pathToInputJson = this.tryToGetInputFileFromOutputFile(pathToInputFile);
70+
}
71+
72+
await writeOpossumFile({
73+
path: pathToOpossumFile,
74+
input: this.getInputJson(pathToInputJson),
75+
output: this.getOutputJson(pathToInputJson),
76+
});
77+
}
78+
79+
protected override async preConvertInputFile(
80+
pathToInputFile: string,
81+
): Promise<string | null> {
82+
let tempFilePath;
83+
try {
84+
tempFilePath = path.join(app.getPath('temp'), 'temp.opossum');
85+
} catch (error) {
86+
// When executing as part of unit tests, app.getPath('temp') throws an error
87+
tempFilePath = path.join(__dirname, 'temp.opossum');
88+
}
89+
90+
console.log(`Temp file path: ${tempFilePath}`);
91+
92+
await this.convertFile(pathToInputFile, tempFilePath);
93+
94+
return tempFilePath;
95+
}
96+
}

‎src/ElectronBackend/opossum-file/__tests__/convertToOpossum.test.ts

-42
This file was deleted.

‎src/ElectronBackend/opossum-file/__tests__/legacy.json

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"metadata": {
3+
"projectId": "2a58a469-738e-4508-98d3-a27bce6e71f7",
4+
"projectTitle": "Test Title",
5+
"fileCreationDate": "2020-07-23 11:47:13.764544"
6+
},
7+
"resources": {
8+
"index.html": 1
9+
},
10+
"externalAttributions": {},
11+
"resourcesToAttributions": {},
12+
"frequentLicenses": [],
13+
"attributionBreakpoints": [],
14+
"filesWithChildren": [],
15+
"baseUrlsForSources": {},
16+
"externalAttributionSources": {}
17+
}

‎src/ElectronBackend/opossum-file/__tests__/merge_base.opossum

594 Bytes
Binary file not shown.

‎src/ElectronBackend/opossum-file/__tests__/opossum-file.test.ts

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
// SPDX-FileCopyrightText: Meta Platforms, Inc. and its affiliates
2+
// SPDX-FileCopyrightText: TNG Technology Consulting GmbH <https://www.tngtech.com>
3+
//
4+
// SPDX-License-Identifier: Apache-2.0
5+
import fs from 'fs';
6+
import { uniqueId } from 'lodash';
7+
import { tmpdir } from 'os';
8+
import { join } from 'path';
9+
10+
import { FileType } from '../../../shared/shared-types';
11+
import { parseOpossumFile } from '../../input/parseFile';
12+
import { ParsedOpossumInputAndOutput } from '../../types/types';
13+
import { isOpossumFileFormat } from '../../utils/isOpossumFileFormat';
14+
import { convertToOpossum, mergeFiles } from '../opossum-file';
15+
16+
describe('conversion to opossum', () => {
17+
const SCANCODE_TEST_FILE = join(__dirname, 'scancode.json');
18+
const OWASP_TEST_FILE = join(__dirname, 'owasp-dependency-check-report.json');
19+
const LEGACY_OPOSSUM_TEST_FILE = join(__dirname, 'legacy.json');
20+
const OPOSSUM_TEST_FILE = join(__dirname, 'merge_base.opossum');
21+
const OPOSSUM_TEST_FILE_COPY = join(__dirname, 'test_merge.opossum');
22+
23+
it('should convert the ScanCode file into a valid .opossum file', async () => {
24+
const opossumPath = join(tmpdir(), `${uniqueId('opossum_')}.opossum`);
25+
await convertToOpossum(
26+
SCANCODE_TEST_FILE,
27+
opossumPath,
28+
FileType.SCANCODE_JSON,
29+
);
30+
expect(fs.existsSync(opossumPath)).toBe(true);
31+
expect(isOpossumFileFormat(opossumPath)).toBe(true);
32+
33+
const parsingResult = await parseOpossumFile(opossumPath);
34+
expect(parsingResult).toHaveProperty('input');
35+
});
36+
37+
it('should convert the owasp file into a valid .opossum file', async () => {
38+
const opossumPath = join(tmpdir(), `${uniqueId('opossum_')}.opossum`);
39+
await convertToOpossum(OWASP_TEST_FILE, opossumPath, FileType.OWASP_JSON);
40+
expect(fs.existsSync(opossumPath)).toBe(true);
41+
expect(isOpossumFileFormat(opossumPath)).toBe(true);
42+
43+
const parsingResult = await parseOpossumFile(opossumPath);
44+
expect(parsingResult).toHaveProperty('input');
45+
});
46+
47+
it('should merge the legacy opossum file into an existing .opossum file', async () => {
48+
fs.copyFileSync(OPOSSUM_TEST_FILE, OPOSSUM_TEST_FILE_COPY);
49+
await mergeFiles(
50+
LEGACY_OPOSSUM_TEST_FILE,
51+
OPOSSUM_TEST_FILE_COPY,
52+
FileType.LEGACY_OPOSSUM,
53+
);
54+
const parsingResult = await parseOpossumFile(OPOSSUM_TEST_FILE_COPY);
55+
56+
expect(parsingResult).toHaveProperty('input');
57+
expect(
58+
(parsingResult as ParsedOpossumInputAndOutput).input.resources[
59+
'index.tsx'
60+
],
61+
).toBeDefined();
62+
expect(
63+
(parsingResult as ParsedOpossumInputAndOutput).input.resources[
64+
'index.html'
65+
],
66+
).toBeDefined();
67+
});
68+
69+
it('should merge the ScanCode file into an existing .opossum file', async () => {
70+
fs.copyFileSync(OPOSSUM_TEST_FILE, OPOSSUM_TEST_FILE_COPY);
71+
await mergeFiles(
72+
SCANCODE_TEST_FILE,
73+
OPOSSUM_TEST_FILE_COPY,
74+
FileType.SCANCODE_JSON,
75+
);
76+
const parsingResult = await parseOpossumFile(OPOSSUM_TEST_FILE_COPY);
77+
78+
expect(parsingResult).toHaveProperty('input');
79+
expect(
80+
(parsingResult as ParsedOpossumInputAndOutput).input.resources[
81+
'index.tsx'
82+
],
83+
).toBeDefined();
84+
expect(
85+
(parsingResult as ParsedOpossumInputAndOutput).input.resources['src'],
86+
).toBeDefined();
87+
});
88+
89+
it('should merge the owasp file into an existing .opossum file', async () => {
90+
fs.copyFileSync(OPOSSUM_TEST_FILE, OPOSSUM_TEST_FILE_COPY);
91+
await mergeFiles(
92+
OWASP_TEST_FILE,
93+
OPOSSUM_TEST_FILE_COPY,
94+
FileType.OWASP_JSON,
95+
);
96+
const parsingResult = await parseOpossumFile(OPOSSUM_TEST_FILE_COPY);
97+
98+
expect(parsingResult).toHaveProperty('input');
99+
expect(
100+
(parsingResult as ParsedOpossumInputAndOutput).input.resources[
101+
'index.tsx'
102+
],
103+
).toBeDefined();
104+
expect(
105+
(parsingResult as ParsedOpossumInputAndOutput).input.resources['contrib'],
106+
).toBeDefined();
107+
});
108+
});

0 commit comments

Comments
 (0)