several fixes for sso-redirect workflow:

- if a seed page redirects to another page, and then back (such as for sso), ensure original seed is used for link extraction
- don't allow direct fetch if no mime type at all
- don't add --lang if using profile, display warning, as language override may invalidate profile settings
- add temp extra delay if seed page redirects, to ensure any sso-related redirects finish

Author: ikreymer

src/crawler.ts

@@ -49,6 +49,7 @@ import {
   SITEMAP_INITIAL_FETCH_TIMEOUT_SECS,
   ExitCodes,
   InterruptReason,
+  SEED_REDIRECT_ADD_DELAY,
 } from "./util/constants.js";
 
 import { AdBlockRules, BlockRuleDecl, BlockRules } from "./util/blockrules.js";
@@ -574,7 +575,14 @@ export class Crawler {
   extraChromeArgs() {
     const args = [];
     if (this.params.lang) {
-      args.push(`--accept-lang=${this.params.lang}`);
+      if (this.params.profile) {
+        logger.warn(
+          "Ignoring --lang option with profile, using language configured in the profile",
+          { lang: this.params.lang },
+        );
+      } else {
+        args.push(`--accept-lang=${this.params.lang}`);
+      }
     }
     return args;
   }
@@ -2071,6 +2079,8 @@ self.__bx_behaviors.selectMainBehavior();
 
     const respUrl = resp.url().split("#")[0];
     const isChromeError = page.url().startsWith("chrome-error://");
+    let thisPageDelay = 0;
+    let originalSeedId = null;
 
     if (
       depth === 0 &&
@@ -2079,6 +2089,7 @@ self.__bx_behaviors.selectMainBehavior();
       respUrl + "/" !== url &&
       !downloadResponse
     ) {
+      originalSeedId = data.seedId;
       data.seedId = await this.crawlState.addExtraSeed(
         this.seeds,
         this.numOriginalSeeds,
@@ -2090,6 +2101,7 @@ self.__bx_behaviors.selectMainBehavior();
         newUrl: respUrl,
         seedId: data.seedId,
       });
+      thisPageDelay = SEED_REDIRECT_ADD_DELAY;
     }
 
     const status = resp.status();
@@ -2176,7 +2188,7 @@ self.__bx_behaviors.selectMainBehavior();
 
     await this.netIdle(page, logDetails);
 
-    await this.awaitPageLoad(page.mainFrame(), logDetails);
+    await this.awaitPageLoad(page.mainFrame(), thisPageDelay, logDetails);
 
     // skip extraction if at max depth
     if (seed.isAtMaxDepth(depth, extraHops)) {
@@ -2190,6 +2202,27 @@ self.__bx_behaviors.selectMainBehavior();
       "links",
     );
 
+    const pageUrl = page.url().split("#")[0];
+
+    if (depth === 0 && respUrl !== urlNoHash) {
+      if (pageUrl === urlNoHash && originalSeedId !== null) {
+        logger.info("Seed page redirected back to original seed", { pageUrl });
+        data.seedId = originalSeedId;
+      } else {
+        data.seedId = await this.crawlState.addExtraSeed(
+          this.seeds,
+          this.numOriginalSeeds,
+          data.seedId,
+          pageUrl,
+        );
+        logger.info("Seed page redirected, adding redirected seed", {
+          origUrl: respUrl,
+          newUrl: pageUrl,
+          seedId: data.seedId,
+        });
+      }
+    }
+
     await this.extractLinks(page, data, this.params.selectLinks, logDetails);
   }
 
@@ -2211,7 +2244,7 @@ self.__bx_behaviors.selectMainBehavior();
     }
   }
 
-  async awaitPageLoad(frame: Frame, logDetails: LogDetails) {
+  async awaitPageLoad(frame: Frame, tempDelay: number, logDetails: LogDetails) {
     logger.debug(
       "Waiting for custom page load via behavior",
       logDetails,
@@ -2230,11 +2263,13 @@ self.__bx_behaviors.selectMainBehavior();
       logger.warn("Waiting for custom page load failed", e, "behavior");
     }
 
-    if (this.params.postLoadDelay) {
+    const delay = tempDelay + this.params.postLoadDelay;
+
+    if (delay) {
       logger.info("Awaiting post load delay", {
-        seconds: this.params.postLoadDelay,
+        seconds: delay,
       });
-      await sleep(this.params.postLoadDelay);
+      await sleep(delay);
     }
   }
 

src/replaycrawler.ts

@@ -446,7 +446,7 @@ export class ReplayCrawler extends Crawler {
     // optionally reload (todo: reevaluate if this is needed)
     // await page.reload();
 
-    await this.awaitPageLoad(replayFrame, logDetails);
+    await this.awaitPageLoad(replayFrame, 0, logDetails);
 
     data.isHTMLPage = true;
 

src/util/constants.ts

@@ -32,6 +32,7 @@ export const DEFAULT_MAX_RETRIES = 2;
 export const FETCH_HEADERS_TIMEOUT_SECS = 30;
 export const PAGE_OP_TIMEOUT_SECS = 5;
 export const SITEMAP_INITIAL_FETCH_TIMEOUT_SECS = 30;
+export const SEED_REDIRECT_ADD_DELAY = 20;
 
 export type ExtractSelector = {
   selector: string;