From b903a4dfe70acfa7915519e5e7dd6325c2171784 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Sat, 30 Apr 2022 11:20:25 -0600 Subject: [PATCH 1/7] cli, dependency_source: support `--no-deps` --- pip_audit/_cli.py | 9 +++ pip_audit/_dependency_source/requirement.py | 63 +++++++++++++-------- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/pip_audit/_cli.py b/pip_audit/_cli.py index 5c1432b6..d8c1e53d 100644 --- a/pip_audit/_cli.py +++ b/pip_audit/_cli.py @@ -271,6 +271,12 @@ def _parser() -> argparse.ArgumentParser: action="store_true", help="don't audit packages that are marked as editable", ) + parser.add_argument( + "--no-deps", + action="store_true", + help="don't perform any dependency resolution; requires all requirements are pinned " + "to an exact version", + ) return parser @@ -313,6 +319,8 @@ def audit() -> None: parser.error("The --index-url flag can only be used with --requirement (-r)") elif args.extra_index_urls: parser.error("The --extra-index-url flag can only be used with --requirement (-r)") + elif args.no_deps: + parser.error("The --no-deps flag can only be used with --requirement (-r)") with ExitStack() as stack: actors = [] @@ -332,6 +340,7 @@ def audit() -> None: index_urls, args.timeout, args.cache_dir, args.skip_editable, state ), require_hashes=args.require_hashes, + no_deps=args.no_deps, state=state, ) elif args.project_path is not None: diff --git a/pip_audit/_dependency_source/requirement.py b/pip_audit/_dependency_source/requirement.py index 3df6e57f..6871958d 100644 --- a/pip_audit/_dependency_source/requirement.py +++ b/pip_audit/_dependency_source/requirement.py @@ -47,6 +47,7 @@ def __init__( resolver: DependencyResolver, *, require_hashes: bool = False, + no_deps: bool = False, state: AuditState = AuditState(), ) -> None: """ @@ -59,11 +60,16 @@ def __init__( `require_hashes` controls the hash policy: if `True`, dependency collection will fail unless all requirements include hashes. + `no_deps` controls the dependency resolution policy: if `True`, + dependency resolution is not performed and the inputs are checked + and treated as "frozen". + `state` is an `AuditState` to use for state callbacks. """ self._filenames = filenames self._resolver = resolver self._require_hashes = require_hashes + self._no_deps = no_deps self.state = state def collect(self) -> Iterator[Dependency]: @@ -79,17 +85,20 @@ def collect(self) -> Iterator[Dependency]: except PipError as pe: raise RequirementSourceError("requirement parsing raised an error") from pe - # If we're requiring hashes, we skip dependency resolution and check that each - # requirement is accompanied by a hash and is pinned. Files that include hashes must - # explicitly list all transitive dependencies so assuming that the requirements file is - # valid and able to be installed with `-r`, we can skip dependency resolution. + # There are three cases where we skip dependency resolution: # - # If at least one requirement has a hash, it implies that we require hashes for all - # requirements - if self._require_hashes or any( + # 1. The user has explicitly specified `--require-hashes`. + # 2. One or more parsed requirements has hashes specified, enabling + # hash checking for all requirements. + # 3. The user has explicitly specified `--no-deps`. + require_hashes = self._require_hashes or any( isinstance(req, ParsedRequirement) and req.hashes for req in reqs.values() - ): - yield from self._collect_hashed_deps(iter(reqs.values())) + ) + skip_deps = require_hashes or self._no_deps + if skip_deps: + yield from self._collect_preresolved_deps( + iter(reqs.values()), require_hashes=require_hashes + ) continue # Invoke the dependency resolver to turn requirements into dependencies @@ -178,26 +187,32 @@ def _recover_files(self, tmp_files: List[IO[str]]) -> None: logger.warning(f"encountered an exception during file recovery: {e}") continue - def _collect_hashed_deps( - self, reqs: Iterator[Union[ParsedRequirement, UnparsedRequirement]] + def _collect_preresolved_deps( + self, + reqs: Iterator[Union[ParsedRequirement, UnparsedRequirement]], + require_hashes: bool = False, ) -> Iterator[Dependency]: - # NOTE: Editable and hashed requirements are incompatible by definition, so - # we don't bother checking whether the user has asked us to skip editable requirements - # when we're doing hashed requirement collection. + """ + Collect pre-resolved (pinned) dependencies, optionally enforcing a + hash requirement policy. + """ for req in reqs: req = cast(ParsedRequirement, req) - if not req.hashes: + if require_hashes and not req.hashes: raise RequirementSourceError( - f"requirement {req.name} does not contain a hash: {str(req)}" + f"requirement {req.name} does not contain a hash {str(req)}" ) - if req.specifier is not None: - pinned_specifier_info = PINNED_SPECIFIER_RE.match(str(req.specifier)) - if pinned_specifier_info is not None: - # Yield a dependency with the hash - pinned_version = pinned_specifier_info.group("version") - yield ResolvedDependency(req.name, Version(pinned_version), req.hashes) - continue - raise RequirementSourceError(f"requirement {req.name} is not pinned: {str(req)}") + + if req.specifier is None: + raise RequirementSourceError(f"requirement {req.name} is not pinned: {str(req)}") + + pinned_specifier = PINNED_SPECIFIER_RE.match(str(req.specifier)) + if pinned_specifier is None: + raise RequirementSourceError(f"requirement {req.name} is not pinned: {str(req)}") + + yield ResolvedDependency( + req.name, Version(pinned_specifier.group("version")), req.hashes + ) class RequirementSourceError(DependencySourceError): From 919f24102d6c5d0ba3aa5b2492bfaf1e0d538e07 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Sat, 30 Apr 2022 12:16:01 -0600 Subject: [PATCH 2/7] pip_audit, test: update tests, coverage --- pip_audit/_dependency_source/requirement.py | 2 +- test/dependency_source/test_requirement.py | 31 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/pip_audit/_dependency_source/requirement.py b/pip_audit/_dependency_source/requirement.py index 6871958d..7210645e 100644 --- a/pip_audit/_dependency_source/requirement.py +++ b/pip_audit/_dependency_source/requirement.py @@ -203,7 +203,7 @@ def _collect_preresolved_deps( f"requirement {req.name} does not contain a hash {str(req)}" ) - if req.specifier is None: + if not req.specifier: raise RequirementSourceError(f"requirement {req.name} is not pinned: {str(req)}") pinned_specifier = PINNED_SPECIFIER_RE.match(str(req.specifier)) diff --git a/test/dependency_source/test_requirement.py b/test/dependency_source/test_requirement.py index 0fb0903e..e367d22c 100644 --- a/test/dependency_source/test_requirement.py +++ b/test/dependency_source/test_requirement.py @@ -353,3 +353,34 @@ def test_requirement_source_require_hashes_unpinned(monkeypatch): # version number with pytest.raises(DependencySourceError): list(source.collect()) + + +def test_requirement_source_no_deps(monkeypatch): + source = requirement.RequirementSource( + [Path("requirements.txt")], ResolveLibResolver(), no_deps=True + ) + + monkeypatch.setattr( + _parse_requirements, + "_read_file", + lambda _: ["flask==2.0.1"], + ) + + specs = list(source.collect()) + assert specs == [ResolvedDependency("flask", Version("2.0.1"), hashes={})] + + +def test_requirement_source_no_deps_unpinned(monkeypatch): + source = requirement.RequirementSource( + [Path("requirements.txt")], ResolveLibResolver(), no_deps=True + ) + + monkeypatch.setattr( + _parse_requirements, + "_read_file", + lambda _: ["flask\nrequests>=1.0"], + ) + + # When dependency resolution is disabled, all requirements must be pinned. + with pytest.raises(DependencySourceError): + list(source.collect()) From 87cc438f3e9d026ff1109ff55bb1b56d8c687b47 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Sat, 30 Apr 2022 14:00:22 -0600 Subject: [PATCH 3/7] README: update `--help` --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d25ed25a..8867ac5d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ usage: pip-audit [-h] [-V] [-l] [-r REQUIREMENTS] [-f FORMAT] [-s SERVICE] [--progress-spinner {on,off}] [--timeout TIMEOUT] [--path PATHS] [-v] [--fix] [--require-hashes] [--index-url INDEX_URL] [--extra-index-url EXTRA_INDEX_URLS] - [--skip-editable] + [--skip-editable] [--no-deps] [project_path] audit the Python environment for dependencies with known vulnerabilities @@ -137,6 +137,9 @@ optional arguments: `--index-url` (default: []) --skip-editable don't audit packages that are marked as editable (default: False) + --no-deps don't perform any dependency resolution; requires all + requirements are pinned to an exact version (default: + False) ``` From e21cf5ddf5fd146c8b51552c34da7888db3b629f Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Sat, 30 Apr 2022 14:02:01 -0600 Subject: [PATCH 4/7] CHANGELOG: record changes --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16824878..6cae18b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ All versions prior to 0.0.9 are untracked. ## [Unreleased] - ReleaseDate +### Added + +* CLI: The `--no-deps` flag has been added, allowing users to skip dependency + resolution entirely when `pip-audit` is used in requirements mode. + ## [2.1.1] - 2022-03-29 ### Fixed From 009d21f72a021ccaf39db75d0380425117b9e8c9 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Sat, 30 Apr 2022 23:54:07 -0600 Subject: [PATCH 5/7] CHANGELOG: add link --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cae18b0..7249557c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ All versions prior to 0.0.9 are untracked. ### Added * CLI: The `--no-deps` flag has been added, allowing users to skip dependency - resolution entirely when `pip-audit` is used in requirements mode. + resolution entirely when `pip-audit` is used in requirements mode + ([#255](https://github.com/trailofbits/pip-audit/pull/255)) ## [2.1.1] - 2022-03-29 From d9e155063048ccbd52059e63bcdc6ee715104126 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 2 May 2022 11:08:12 -0600 Subject: [PATCH 6/7] cli: `make lint` --- pip_audit/_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip_audit/_cli.py b/pip_audit/_cli.py index 931c70d5..aadc8d3b 100644 --- a/pip_audit/_cli.py +++ b/pip_audit/_cli.py @@ -285,7 +285,7 @@ def _parser() -> argparse.ArgumentParser: help="output results to the given file", # NOTE: Ideally we would set default=sys.stdout here, but # argparse's default renderer uses __repr__ and produces - # a pretty unpleasant help message. + # a pretty unpleasant help message. ) return parser From ce21361874d8040527bfa85523e1b5fc914dc1bc Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 2 May 2022 17:40:47 -0600 Subject: [PATCH 7/7] cli: add some nudges --- pip_audit/_cli.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pip_audit/_cli.py b/pip_audit/_cli.py index aadc8d3b..10f15f9c 100644 --- a/pip_audit/_cli.py +++ b/pip_audit/_cli.py @@ -335,6 +335,20 @@ def audit() -> None: elif args.no_deps: parser.error("The --no-deps flag can only be used with --requirement (-r)") + # Nudge users to consider alternate workflows. + if args.require_hashes and args.no_deps: + logger.warning("The --no-deps flag is redundant when used with --require-hashes") + + if args.no_deps: + logger.warning( + "--no-deps is supported, but users are encouraged to fully hash their " + "pinned dependencies" + ) + logger.warning( + "Consider using a tool like `pip-compile`: " + "https://pip-tools.readthedocs.io/en/latest/#using-hashes" + ) + with ExitStack() as stack: actors = [] if args.progress_spinner: