feat: rewrite docs generation

High level overview of the changes here:

- The source for the docs content has moved from `docs/content/` to
  `docs/lib/content/`. The generated markdown is still written to
  `docs/content/` but that directory is now ignored from git.
- All generated content sections of the docs have been removed and
  replaced with single placeholder html comments such as `<!--
  AUTOGENERATED CONFIG DESCRIPTIONS -->`
- Placeholders are replaced with generated content only as part of the
  `prepack` step, so generated markdown is no longer checked in to
  source and all docs related `make` commands have been removed
- All docs (and docs related) snapshots have been moved to a single test
  file that outputs command usage and formats it with functions imported
  from `docs/lib/index.js`. So tests will fail if docs content changes
  until `npm run snap` is run.

Author: lukekarrys

.github/workflows/ci-docs.yml .github/workflows/ci-npmcli-docs.yml

@@ -1,6 +1,6 @@
 # This file is automatically added by @npmcli/template-oss. Do not edit.
 
-name: CI - docs
+name: CI - @npmcli/docs
 
 on:
   workflow_dispatch:
@@ -71,9 +71,9 @@ jobs:
       - name: Reset Deps
         run: node . run resetdeps
       - name: Lint
-        run: node . run lint --ignore-scripts -w docs
+        run: node . run lint --ignore-scripts -w @npmcli/docs
       - name: Post Lint
-        run: node . run postlint --ignore-scripts -w docs
+        run: node . run postlint --ignore-scripts -w @npmcli/docs
 
   test:
     name: Test - ${{ matrix.platform.name }} - ${{ matrix.node-version }}
@@ -119,7 +119,7 @@ jobs:
       - name: Add Problem Matcher
         run: echo "::add-matcher::.github/matchers/tap.json"
       - name: Test
-        run: node . test --ignore-scripts -w docs
+        run: node . test --ignore-scripts -w @npmcli/docs
       - name: Check Git Status
         if: matrix && matrix.platform.os != 'windows-latest'
         run: node scripts/git-dirty.js

.github/workflows/ci.yml

@@ -135,32 +135,6 @@ jobs:
         if: matrix && matrix.platform.os != 'windows-latest'
         run: node scripts/git-dirty.js
 
-  check-docs:
-    name: Check Docs
-    if: github.repository_owner == 'npm'
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup Git User
-        run: |
-          git config --global user.email "npm-cli+bot@github.com"
-          git config --global user.name "npm CLI robot"
-      - name: Setup Node
-        uses: actions/setup-node@v3
-        with:
-          node-version: 18.x
-          cache: npm
-      - name: Reset Deps
-        run: node . run resetdeps
-      - name: Make Docs
-        run: make freshdocs
-      - name: Check Git Status
-        run: node scripts/git-dirty.js
-
   licenses:
     name: Check Licenses
     if: github.repository_owner == 'npm'

.github/workflows/create-cli-deps-pr.yml

@@ -64,7 +64,6 @@ jobs:
           echo "Prepping CLI repo for release"
           cd cli
           git checkout "$npm_vtag"
-          make
           make release
 
           echo "Removing old npm"

DEPENDENCIES.md

@@ -119,6 +119,7 @@ graph LR;
   npm-->npmcli-arborist["@npmcli/arborist"];
   npm-->npmcli-ci-detect["@npmcli/ci-detect"];
   npm-->npmcli-config["@npmcli/config"];
+  npm-->npmcli-docs["@npmcli/docs"];
   npm-->npmcli-eslint-config["@npmcli/eslint-config"];
   npm-->npmcli-fs["@npmcli/fs"];
   npm-->npmcli-git["@npmcli/git"];
@@ -192,6 +193,10 @@ graph LR;
   npmcli-config-->proc-log;
   npmcli-config-->read-package-json-fast;
   npmcli-config-->semver;
+  npmcli-docs-->ignore-walk;
+  npmcli-docs-->npmcli-eslint-config["@npmcli/eslint-config"];
+  npmcli-docs-->npmcli-fs["@npmcli/fs"];
+  npmcli-docs-->npmcli-template-oss["@npmcli/template-oss"];
   npmcli-fs-->semver;
   npmcli-git-->npm-pick-manifest;
   npmcli-git-->npmcli-promise-spawn["@npmcli/promise-spawn"];
@@ -255,6 +260,7 @@ graph LR;
   ansi-styles-->color-convert;
   are-we-there-yet-->delegates;
   are-we-there-yet-->readable-stream;
+  argparse-->sprintf-js;
   babel-code-frame-->babel-highlight["@babel/highlight"];
   babel-core-->babel-code-frame["@babel/code-frame"];
   babel-core-->babel-generator["@babel/generator"];
@@ -273,13 +279,10 @@ graph LR;
   babel-core-->semver;
   babel-core-->source-map;
   babel-generator-->babel-types["@babel/types"];
+  babel-generator-->jridgewell-gen-mapping["@jridgewell/gen-mapping"];
   babel-generator-->jsesc;
-  babel-generator-->source-map;
-  babel-helper-environment-visitor-->babel-types["@babel/types"];
-  babel-helper-function-name-->babel-helper-get-function-arity["@babel/helper-get-function-arity"];
   babel-helper-function-name-->babel-template["@babel/template"];
   babel-helper-function-name-->babel-types["@babel/types"];
-  babel-helper-get-function-arity-->babel-types["@babel/types"];
   babel-helper-hoist-variables-->babel-types["@babel/types"];
   babel-helper-module-imports-->babel-types["@babel/types"];
   babel-helper-module-transforms-->babel-helper-environment-visitor["@babel/helper-environment-visitor"];
@@ -325,6 +328,7 @@ graph LR;
   babel-traverse-->babel-types["@babel/types"];
   babel-traverse-->debug;
   babel-traverse-->globals;
+  babel-types-->babel-helper-string-parser["@babel/helper-string-parser"];
   babel-types-->babel-helper-validator-identifier["@babel/helper-validator-identifier"];
   babel-types-->to-fast-properties;
   bin-links-->cmd-shim;
@@ -338,7 +342,6 @@ graph LR;
   bl-->inherits;
   bl-->readable-stream;
   brace-expansion-->balanced-match;
-  brace-expansion-->concat-map;
   buffer-->base64-js;
   buffer-->ieee754;
   builtins-->semver;
@@ -376,7 +379,6 @@ graph LR;
   columnify-->strip-ansi;
   columnify-->wcwidth;
   combined-stream-->delayed-stream;
-  convert-source-map-->safe-buffer;
   cssstyle-->cssom;
   data-urls-->abab;
   data-urls-->whatwg-mimetype;
@@ -387,18 +389,6 @@ graph LR;
   detab-->repeat-string;
   dezalgo-->asap;
   dezalgo-->wrappy;
-  docs-->cmark-gfm;
-  docs-->jsdom;
-  docs-->marked-man;
-  docs-->mdx-js-mdx["@mdx-js/mdx"];
-  docs-->mkdirp;
-  docs-->npmcli-eslint-config["@npmcli/eslint-config"];
-  docs-->npmcli-fs["@npmcli/fs"];
-  docs-->npmcli-promise-spawn["@npmcli/promise-spawn"];
-  docs-->npmcli-template-oss["@npmcli/template-oss"];
-  docs-->tap;
-  docs-->which;
-  docs-->yaml;
   domexception-->webidl-conversions;
   encoding-->iconv-lite;
   end-of-stream-->once;
@@ -410,6 +400,7 @@ graph LR;
   form-data-->asynckit;
   form-data-->combined-stream;
   form-data-->mime-types;
+  front-matter-->js-yaml;
   fs-minipass-->minipass;
   gauge-->aproba;
   gauge-->console-control-strings;
@@ -483,6 +474,13 @@ graph LR;
   is-cidr-->cidr-regex;
   is-core-module-->has;
   is-fullwidth-code-point-->number-is-nan;
+  jridgewell-gen-mapping-->jridgewell-set-array["@jridgewell/set-array"];
+  jridgewell-gen-mapping-->jridgewell-sourcemap-codec["@jridgewell/sourcemap-codec"];
+  jridgewell-gen-mapping-->jridgewell-trace-mapping["@jridgewell/trace-mapping"];
+  jridgewell-trace-mapping-->jridgewell-resolve-uri["@jridgewell/resolve-uri"];
+  jridgewell-trace-mapping-->jridgewell-sourcemap-codec["@jridgewell/sourcemap-codec"];
+  js-yaml-->argparse;
+  js-yaml-->esprima;
   jsdom-->abab;
   jsdom-->acorn-globals;
   jsdom-->acorn;
@@ -697,7 +695,6 @@ graph LR;
   npm-->cli-columns;
   npm-->cli-table3;
   npm-->columnify;
-  npm-->docs;
   npm-->fastest-levenshtein;
   npm-->fs-minipass;
   npm-->glob;
@@ -741,6 +738,7 @@ graph LR;
   npm-->npmcli-arborist["@npmcli/arborist"];
   npm-->npmcli-ci-detect["@npmcli/ci-detect"];
   npm-->npmcli-config["@npmcli/config"];
+  npm-->npmcli-docs["@npmcli/docs"];
   npm-->npmcli-eslint-config["@npmcli/eslint-config"];
   npm-->npmcli-fs["@npmcli/fs"];
   npm-->npmcli-git["@npmcli/git"];
@@ -847,6 +845,19 @@ graph LR;
   npmcli-config-->semver;
   npmcli-config-->walk-up-path;
   npmcli-disparity-colors-->ansi-styles;
+  npmcli-docs-->cmark-gfm;
+  npmcli-docs-->front-matter;
+  npmcli-docs-->ignore-walk;
+  npmcli-docs-->isaacs-string-locale-compare["@isaacs/string-locale-compare"];
+  npmcli-docs-->jsdom;
+  npmcli-docs-->marked-man;
+  npmcli-docs-->mdx-js-mdx["@mdx-js/mdx"];
+  npmcli-docs-->mkdirp;
+  npmcli-docs-->npmcli-eslint-config["@npmcli/eslint-config"];
+  npmcli-docs-->npmcli-fs["@npmcli/fs"];
+  npmcli-docs-->npmcli-template-oss["@npmcli/template-oss"];
+  npmcli-docs-->tap;
+  npmcli-docs-->yaml;
   npmcli-fs-->gar-promisify["@gar/promisify"];
   npmcli-fs-->semver;
   npmcli-git-->lru-cache;
@@ -1042,6 +1053,7 @@ graph LR;
   tough-cookie-->psl;
   tough-cookie-->punycode;
   tough-cookie-->universalify;
+  tough-cookie-->url-parse;
   tr46-->punycode;
   tunnel-agent-->safe-buffer;
   type-check-->prelude-ls;
@@ -1065,6 +1077,8 @@ graph LR;
   unist-util-visit-->unist-util-visit-parents;
   unist-util-visit-parents-->types-unist["@types/unist"];
   unist-util-visit-parents-->unist-util-is;
+  url-parse-->querystringify;
+  url-parse-->requires-port;
   validate-npm-package-license-->spdx-correct;
   validate-npm-package-license-->spdx-expression-parse;
   validate-npm-package-name-->builtins;
@@ -1102,6 +1116,6 @@ packages higher up the chain.
  - pacote, libnpmaccess, libnpmhook, libnpmorg, libnpmsearch, libnpmteam, npm-profile
  - npm-registry-fetch, libnpmversion
  - @npmcli/git, make-fetch-happen, @npmcli/config, init-package-json
- - @npmcli/installed-package-contents, @npmcli/map-workspaces, cacache, npm-pick-manifest, @npmcli/run-script, read-package-json, @npmcli/query, readdir-scoped-modules, promzard
- - npm-bundled, read-package-json-fast, @npmcli/fs, unique-filename, @npmcli/promise-spawn, npm-install-checks, npm-package-arg, npm-packlist, normalize-package-data, @npmcli/package-json, bin-links, nopt, npmlog, parse-conflict-json, dezalgo, read
- - npm-normalize-package-bin, @npmcli/name-from-folder, json-parse-even-better-errors, semver, @npmcli/move-file, fs-minipass, infer-owner, ssri, unique-slug, hosted-git-info, proc-log, validate-npm-package-name, @npmcli/node-gyp, ignore-walk, minipass-fetch, cmd-shim, read-cmd-shim, write-file-atomic, abbrev, are-we-there-yet, gauge, wrappy, treeverse, @npmcli/eslint-config, @npmcli/template-oss, minify-registry-metadata, @npmcli/disparity-colors, @npmcli/ci-detect, mute-stream, ini, npm-audit-report, npm-user-validate
+ - @npmcli/docs, @npmcli/installed-package-contents, @npmcli/map-workspaces, cacache, npm-pick-manifest, @npmcli/run-script, read-package-json, @npmcli/query, readdir-scoped-modules, promzard
+ - @npmcli/fs, npm-bundled, read-package-json-fast, unique-filename, @npmcli/promise-spawn, npm-install-checks, npm-package-arg, npm-packlist, normalize-package-data, @npmcli/package-json, bin-links, nopt, npmlog, parse-conflict-json, dezalgo, read
+ - semver, ignore-walk, @npmcli/eslint-config, @npmcli/template-oss, npm-normalize-package-bin, @npmcli/name-from-folder, json-parse-even-better-errors, @npmcli/move-file, fs-minipass, infer-owner, ssri, unique-slug, hosted-git-info, proc-log, validate-npm-package-name, @npmcli/node-gyp, minipass-fetch, cmd-shim, read-cmd-shim, write-file-atomic, abbrev, are-we-there-yet, gauge, wrappy, treeverse, minify-registry-metadata, @npmcli/disparity-colors, @npmcli/ci-detect, mute-stream, ini, npm-audit-report, npm-user-validate

Makefile

@@ -3,94 +3,9 @@ SHELL = bash
 
 PUBLISHTAG = $(shell node scripts/publish-tag.js)
 
-# these docs have the @VERSION@ tag in them, so they have to be rebuilt
-# whenever the package.json is touched, in case the version changed.
-version_mandocs = $(shell grep -rl '@VERSION@' docs/content \
-									|sed 's|.md|.1|g' \
-									|sed 's|docs/content/commands/|man/man1/|g' )
-
-cli_mandocs = $(shell find docs/content/commands -name '*.md' \
-               |sed 's|.md|.1|g' \
-               |sed 's|docs/content/commands/|man/man1/|g' )
-
-files_mandocs = $(shell find docs/content/configuring-npm -name '*.md' \
-               |sed 's|.md|.5|g' \
-               |sed 's|docs/content/configuring-npm/|man/man5/|g' )
-
-misc_mandocs = $(shell find docs/content/using-npm -name '*.md' \
-               |sed 's|.md|.7|g' \
-               |sed 's|docs/content/using-npm/|man/man7/|g' )
-
-mandocs = $(cli_mandocs) $(files_mandocs) $(misc_mandocs)
-
-markdown_docs = $(shell for file in $(find lib/commands -name '*.js'); do echo docs/content/commands/npm-$(basename $file .js).md; done)
-
-all: docs
-
-docs: mandocs htmldocs $(markdown_docs)
-
-# don't regenerate the snapshot if we're generating
-# snapshots, since presumably we just did that.
-mandocs: deps $(mandocs)
-	@ ! [ "$${npm_lifecycle_event}" = "snap" ] && \
-	  ! [ "$${npm_lifecycle_event}" = "postsnap" ] && \
-	  TAP_SNAPSHOT=1 node test/lib/utils/config/definitions.js || true
-
-$(version_mandocs): package.json
-
-htmldocs: deps
-	node bin/npm-cli.js rebuild cmark-gfm
-	node docs/bin/dockhand.js
-
-clean: docsclean gitclean
-
-docsclean:
-	rm -rf man
-
 deps:
 	node bin/npm-cli.js run resetdeps
 
-## targets for man files, these are encouraged to be only built by running `make docs` or `make mandocs`
-man/man1/%.1: docs/content/commands/%.md docs/bin/docs-build.js
-	@[ -d man/man1 ] || mkdir -p man/man1
-	node docs/bin/docs-build.js $< $@
-
-man/man5/npm-json.5: man/man5/package.json.5
-	cp $< $@
-
-man/man5/npm-global.5: man/man5/folders.5
-	cp $< $@
-
-man/man5/%.5: docs/content/configuring-npm/%.md docs/bin/docs-build.js
-	@[ -d man/man5 ] || mkdir -p man/man5
-	node docs/bin/docs-build.js $< $@
-
-man/man7/%.7: docs/content/using-npm/%.md docs/bin/docs-build.js
-	@[ -d man/man7 ] || mkdir -p man/man7
-	node docs/bin/docs-build.js $< $@
-
-# Any time the config definitions description changes, automatically
-# update the documentation to account for it
-docs/content/using-npm/config.md: docs/bin/config-doc.js lib/utils/config/*.js
-	node docs/bin/config-doc.js
-
-mddocs: docs/bin/config-doc-command.js lib/utils/config/*.js lib/utils/cmd-list.js
-	@for file in $(shell find docs/content/commands -name 'npm-*.md'); do \
-		cmdname=$$(basename $$file .md) ;\
-		cmdname=$${cmdname##npm-} ;\
-		echo node docs/bin/config-doc-command.js $${file} lib/commands/$${cmdname}.js ;\
-		node docs/bin/config-doc-command.js $${file} lib/commands/$${cmdname}.js ;\
-	done
-
-docs/content/commands/npm-%.md: lib/commands/%.js
-	node docs/bin/config-doc-command.js $@ $<
-
-freshdocs:
-	touch lib/utils/config/definitions.js
-	touch docs/bin/*.js
-	make docs
-	make mddocs
-
 lint-all: deps
 	node bin/npm-cli.js run lint-all
 
@@ -113,10 +28,10 @@ prune: deps
 	node bin/npm-cli.js prune --omit=dev --no-save --no-audit --no-fund
 	node scripts/git-dirty.js
 
-publish: gitclean ls-ok link docs lint-all test-all prune
+publish: gitclean ls-ok link lint-all test-all prune
 	node bin/npm-cli.js publish --tag=$(PUBLISHTAG)
 
-release: gitclean ls-ok docs prune
+release: gitclean ls-ok prune
 	@bash scripts/release.sh
 
-.PHONY: all latest install dev link docs mddocs clean uninstall lint-all test-all man docsclean release ls-ok deps prune freshdocs
+.PHONY: link gitclean uninstall lint-all test-all release ls-ok deps prune

docs/.gitignore

@@ -10,12 +10,10 @@
 !/.gitignore
 !/bin/
 !/CHANGELOG*
-!/content/
 !/docs/
 !/lib/
 !/LICENSE*
 !/map.js
-!/nav.yml
 !/package.json
 !/README*
 !/scripts/

docs/bin/build.js

@@ -0,0 +1,12 @@
+const run = require('../lib/build.js')
+const { paths } = require('../lib/index')
+
+run({
+  verify: true,
+  ...paths,
+})
+  .then((res) => console.log(`Wrote ${res.length} files`))
+  .catch((err) => {
+    process.exitCode = 1
+    console.error(err)
+  })