make run function accepts url as parameter and trackResponses, getCookies as optional parameter

Author: deathemperor

README.md

@@ -41,6 +41,28 @@ If you want to use latest chrome, run chrome/buildChrome.sh on EC2 having at lea
 See also [serverless-chrome](https://github.com/adieuadieu/serverless-chrome/blob/master/chrome/README.md).
 Once you build it, link to `headless_shell.tar.gz` in `chrome` dir.
 
+## Deploying to AWS
+The best way to deploy right now is to upload package.zip file to S3 and set the link to Lambda.
+
+Testing in Lambda:
+
+- Normal request
+```
+{
+  "url": "https://google.com",
+  "trackResponses": false,
+  "getCookies": false,
+}
+```
+- Request with XHR responses and cookies (shopee.vn is SPA written in React)
+```
+{
+  "url": "https://shopee.vn/Astaxanthin-i.25321200.413097508",
+  "trackResponses": true,
+  "getCookies": true,
+}
+```
+
 ## Article
 
 [Lambda上でPuppeteer/Headless Chromeを動かすStarter Kitを作った - sambaiz-net](https://www.sambaiz.net/article/132/)

src/index.js

@@ -1,19 +1,33 @@
 const setup = require('./starter-kit/setup');
+const urlParser = require('url');
 
 exports.handler = async (event, context, callback) => {
   // For keeping the browser launch
   context.callbackWaitsForEmptyEventLoop = false;
   const browser = await setup.getBrowser();
-  exports.run(browser).then(
+  exports.run(event.url, browser, event.trackResponses, event.getCookies).then(
     (result) => callback(null, result)
   ).catch(
     (err) => callback(err)
   );
 };
 
-exports.run = async (browser) => {
+exports.run = async (url, browser
+      , trackResponses = false, getCookies = false) => {
   const page = await browser.newPage();
-  await page.goto('https://www.google.co.jp');
+  let responses = {};
+  if (trackResponses) {
+    const parsedUrl = urlParser.parse(url);
+    page.on('response', async (response) => {
+      // make sure only tracks response from same host
+      if (response.url.indexOf(parsedUrl.hostname) !== -1) {
+          responses[response.url] = await response.text();
+      }
+    });
+  }
+  await page.goto(url);
+  const html = await page.content();
+  const cookies = getCookies ? await page.cookies() : '';
   await page.close();
-  return 'done';
+  return {html, responses, cookies};
 };

src/starter-kit/local.js

@@ -9,7 +9,7 @@ const puppeteer = require('puppeteer');
         dumpio: !!config.DEBUG,
         // use chrome installed by puppeteer
     });
-    await index.run(browser)
+    await index.run('https://google.com', browser, false, false)
     .then((result) => console.log(result))
     .catch((err) => console.error(err));
     await browser.close();

yarn.lock

@@ -2039,14 +2039,10 @@ safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
 
-sax@1.2.1:
+sax@1.2.1, sax@>=0.6.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
 
-sax@>=0.6.0:
-  version "1.2.4"
-  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
-
 semver@^5.3.0:
   version "5.4.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e"
@@ -2283,14 +2279,10 @@ util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
 
-uuid@3.0.1:
+uuid@3.0.1, uuid@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
 
-uuid@^3.0.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"
-
 v8flags@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/v8flags/-/v8flags-2.1.1.tgz#aab1a1fa30d45f88dd321148875ac02c0b55e5b4"