Characterization Tests for Convert

.gitignore

@@ -40,3 +40,4 @@ src/lib/data/catalog.js
 src/lib/data/firebase-config.js
 src/lib/data/config.js
 src/lib/data/contents.js
+coverage

.prettierignore

@@ -13,6 +13,7 @@ src/lib/data/firebase-config.js
 static
 example_data
 test_data
+**/characterization/test_apps
 
 # Ignore files for PNPM, NPM and YARN
 pnpm-lock.yaml

convert/Task.ts

@@ -3,6 +3,12 @@ export type FileContent = {
     content: string;
 };
 
+export interface TaskOutDirs {
+    static: string;
+    config: string;
+    firebase: string;
+}
+
 export interface TaskOutput {
     taskName: string;
     files: FileContent[];
@@ -13,7 +19,10 @@ export type Promisable<T> = T | Promise<T>;
 export abstract class Task {
     public triggerFiles!: string[];
 
-    constructor(protected dataDir: string) {}
+    constructor(
+        protected dataDir: string,
+        protected outDirs: TaskOutDirs
+    ) {}
 
     public abstract run(
         verbose: number,

convert/convertAbout.ts

@@ -1,16 +1,16 @@
 import { copyFile } from 'fs';
 import path from 'path';
-import { TaskOutput, Task } from './Task';
+import { TaskOutput, Task, TaskOutDirs } from './Task';
 
 export interface AboutTaskOutput extends TaskOutput {
     taskName: 'ConvertAbout';
 }
 /**
  * Copies about.partial.html to static folder
  */
-export function convertAbout(dataDir: string, verbose: number) {
+export function convertAbout(dataDir: string, outDir: string, verbose: number) {
     const srcFile = path.join(dataDir, 'about.partial.html');
-    const dstFile = path.join('static', 'about.partial.html');
+    const dstFile = path.join(outDir, 'about.partial.html');
     copyFile(srcFile, dstFile, function (err: any) {
         if (err) throw err;
         if (verbose) console.log(`copied ${srcFile} to ${dstFile}`);
@@ -19,11 +19,12 @@ export function convertAbout(dataDir: string, verbose: number) {
 export class ConvertAbout extends Task {
     public triggerFiles: string[] = ['about.partial.html'];
 
-    constructor(dataDir: string) {
-        super(dataDir);
+    constructor(dataDir: string, outDirs: TaskOutDirs) {
+        super(dataDir, outDirs);
     }
+
     public async run(verbose: number, outputs: Map<string, TaskOutput>): Promise<TaskOutput> {
-        convertAbout(this.dataDir, verbose);
+        convertAbout(this.dataDir, this.outDirs.static, verbose);
         return {
             taskName: this.constructor.name,
             files: []

convert/convertAll.ts

@@ -0,0 +1,228 @@
+import { ConvertAbout } from './convertAbout';
+import { ConvertBadges } from './convertBadges';
+import { ConvertBooks } from './convertBooks';
+import { ConvertConfig } from './convertConfig';
+import { ConvertContents } from './convertContents';
+import { ConvertFirebase } from './convertFirebase';
+import { ConvertManifest } from './convertManifest';
+import { ConvertMedia } from './convertMedia';
+import { ConvertPlans } from './convertPlans';
+import { ConvertReverseIndex } from './convertReverseIndex';
+import { ConvertSQLite } from './convertSQLite';
+import { ConvertStyles } from './convertStyles';
+import { Task, TaskOutDirs, TaskOutput } from './Task';
+import { watch } from 'chokidar';
+import { writeFile } from 'fs';
+import path from 'path';
+
+export interface ConversionParams {
+    dataDir: string;
+    watch: boolean;
+    watchTimeout: number;
+    verbose: number;
+}
+
+export interface ConversionResult {
+    success: boolean;
+    killProcess: boolean;
+}
+
+//Classes common to both SAB and DAB
+const commonStepClasses = [
+    ConvertConfig,
+    ConvertContents,
+    ConvertStyles,
+    ConvertManifest,
+    ConvertMedia,
+    ConvertFirebase,
+    ConvertBadges,
+    ConvertAbout
+];
+
+//Classes only necessary for SAB
+const SABStepClasses = [ConvertPlans, ConvertBooks];
+
+const DABStepClasses = [ConvertReverseIndex, ConvertSQLite];
+
+function getProgramType(
+    dataDir: string,
+    outDirs: TaskOutDirs,
+    verbose: number
+): string | undefined {
+    const convertConfig = new ConvertConfig(dataDir, outDirs);
+    const config = convertConfig.run(verbose);
+    return config.data.programType;
+}
+
+function getStepClasses(dataDir: string, outDirs: TaskOutDirs, verbose: number): Task[] {
+    const programType = getProgramType(dataDir, outDirs, verbose);
+    return [
+        ...commonStepClasses,
+        ...(programType == 'SAB' ? SABStepClasses : []),
+        ...(programType == 'DAB' ? DABStepClasses : [])
+    ].map((x) => new x(dataDir, outDirs));
+}
+
+function getAllTriggerPaths(stepClasses: Task[]) {
+    return new Set(
+        stepClasses.reduce((acc, step) => acc.concat(step.triggerFiles), [] as string[])
+    );
+}
+
+export class ConvertAll {
+    constructor(
+        params: ConversionParams,
+        private outDirs: TaskOutDirs
+    ) {
+        this.dataDir = params.dataDir;
+        this.watch = params.watch;
+        this.watchTimeout = params.watchTimeout;
+        this.verbose = params.verbose;
+        this.stepClasses = getStepClasses(this.dataDir, this.outDirs, this.verbose);
+        this.allPaths = getAllTriggerPaths(this.stepClasses);
+        this.outputs = this.initialOutputs();
+    }
+
+    dataDir: string;
+    watch: boolean;
+    watchTimeout: number;
+    verbose: number;
+    stepClasses: Task[];
+    allPaths: Set<string>;
+    outputs: Map<string, TaskOutput | null>;
+
+    currentStep = 0;
+    lastStepOutput = '';
+
+    initialOutputs(): Map<string, TaskOutput | null> {
+        return new Map(this.stepClasses.map((x) => [x.constructor.name, null]));
+    }
+
+    stepLogger(...args: any[]) {
+        this.lastStepOutput += args.join(' ') + '\n';
+    }
+
+    async fullConvert(printDetails: boolean): Promise<boolean> {
+        this.currentStep = 0;
+        const oldConsoleError = console.error;
+        const oldConsoleLog = console.log;
+        if (!printDetails) {
+            console.error = this.stepLogger;
+            console.log = this.stepLogger;
+        }
+        for (const step of this.stepClasses) {
+            this.lastStepOutput = '';
+            oldConsoleLog(
+                step.constructor.name + ` (${this.currentStep + 1}/${this.stepClasses.length})`
+            );
+            try {
+                // step may be async, in which case it should be awaited
+                const out = await step.run(this.verbose, this.outputs, step.triggerFiles);
+                this.outputs.set(step.constructor.name, out);
+                await Promise.all(
+                    out.files.map(
+                        (f) =>
+                            new Promise((r) => {
+                                writeFile(f.path, f.content, r);
+                            })
+                    )
+                );
+            } catch (e) {
+                oldConsoleLog(this.lastStepOutput);
+                oldConsoleLog(e);
+                console.error = oldConsoleError;
+                console.log = oldConsoleLog;
+                return false;
+            }
+            // if (printDetails) oldConsoleLog(lastStepOutput);
+            this.currentStep++;
+        }
+        console.error = oldConsoleError;
+        console.log = oldConsoleLog;
+        return true;
+    }
+
+    async run(): Promise<ConversionResult> {
+        if (this.watch) {
+            (async () => {
+                console.log('Compiling...');
+                await this.fullConvert(false);
+                console.log('Watching for changes...');
+                let timer: NodeJS.Timeout;
+                let paths: string[] = [];
+                let firstRun = true;
+                watch(this.dataDir).on('all', async (event, watchPath) => {
+                    watchPath = watchPath.substring(this.dataDir.length + 1);
+                    if (
+                        !firstRun &&
+                        Array.from(this.allPaths.values()).some(
+                            (p) => watchPath.startsWith(p + path.sep) || watchPath === p
+                        )
+                    )
+                        console.log(`${watchPath} changed`);
+                    paths.push(watchPath);
+                    if (timer) clearTimeout(timer);
+                    timer = setTimeout(async () => {
+                        if (firstRun) {
+                            firstRun = false;
+                            paths = [];
+                            return;
+                        }
+                        const oldConsoleError = console.error;
+                        const oldConsoleLog = console.log;
+                        console.error = this.stepLogger;
+                        console.log = this.stepLogger;
+
+                        for (const step of this.stepClasses) {
+                            for (const triggerFile of step.triggerFiles) {
+                                // Trigger if the path is a trigger file OR
+                                // if the trigger file is a directory (ends with /) and the path is in that directory
+                                if (
+                                    paths.includes(triggerFile) ||
+                                    paths.find((p) => p.startsWith(triggerFile + path.sep))
+                                ) {
+                                    try {
+                                        oldConsoleLog('Running step ' + step.constructor.name);
+                                        const out = await step.run(
+                                            this.verbose,
+                                            this.outputs,
+                                            paths
+                                        );
+                                        this.outputs.set(step.constructor.name, out);
+                                        // Write all files to disk
+                                        /*await*/ // We don't need to await the file writes; next steps can continue running while writes occur
+                                        Promise.all(
+                                            out.files.map((f) => {
+                                                new Promise((r) => writeFile(f.path, f.content, r));
+                                            })
+                                        );
+                                    } catch (e) {
+                                        oldConsoleLog(this.lastStepOutput);
+                                        oldConsoleLog(e);
+                                        console.error = oldConsoleError;
+                                        console.log = oldConsoleLog;
+                                        return false;
+                                    }
+                                    break;
+                                }
+                            }
+                        }
+                        console.error = oldConsoleError;
+                        console.log = oldConsoleLog;
+
+                        console.log('Conversion successful');
+                        console.log('Watching for changes...');
+                        paths = [];
+                    }, this.watchTimeout);
+                });
+            })();
+            return {
+                success: true,
+                killProcess: false
+            };
+        } else {
+            const success = await this.fullConvert(true);
+            return { success, killProcess: true };
+        }
+    }
+}

convert/convertBadges.ts

@@ -1,14 +1,15 @@
-import { Promisable, Task, TaskOutput } from './Task';
+import { Promisable, Task, TaskOutDirs, TaskOutput } from './Task';
 import path from 'path';
 import { ConfigTaskOutput } from './convertConfig';
 import { copyFile, existsSync, mkdirSync, rmSync, writeFileSync } from 'fs';
 
 export async function convertBadges(
     badgesDir: string,
+    outDir: string,
     configData: ConfigTaskOutput,
     verbose: number
 ) {
-    const dstBadgeDir = path.join('static', 'badges');
+    const dstBadgeDir = path.join(outDir, 'badges');
     if (!configData.data.mainFeatures['share-apple-app-link']) {
         if (existsSync(dstBadgeDir)) {
             rmSync(dstBadgeDir, { recursive: true });
@@ -61,8 +62,8 @@ export interface BadgesTaskOutput extends TaskOutput {
 export class ConvertBadges extends Task {
     public triggerFiles: string[] = ['appdef.xml'];
     public badgesDir: string;
-    constructor(dataDir: string) {
-        super(dataDir);
+    constructor(dataDir: string, outDirs: TaskOutDirs) {
+        super(dataDir, outDirs);
         this.badgesDir = path.join(__dirname, 'badges');
     }
     public run(
@@ -72,7 +73,7 @@ export class ConvertBadges extends Task {
     ): Promisable<BadgesTaskOutput> {
         const config = outputs.get('ConvertConfig') as ConfigTaskOutput;
 
-        convertBadges(this.badgesDir, config, verbose);
+        convertBadges(this.badgesDir, this.outDirs.static, config, verbose);
         return {
             taskName: 'ConvertBadges',
             files: []