Changes to integrate new parser into packaging + test adjustments

Author: hrnciar

packaging/markers.py

@@ -8,6 +8,8 @@
 import sys
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
+from ._parser import parse_quoted_marker
+from ._tokenizer import ParseException, Tokenizer
 from .specifiers import InvalidSpecifier, Specifier
 
 __all__ = [
@@ -199,11 +201,11 @@ def default_environment() -> Dict[str, str]:
 class Marker:
     def __init__(self, marker: str) -> None:
         try:
-            self._markers = _coerce_parse_result(MARKER.parseString(marker))
+            self._markers = parse_quoted_marker(Tokenizer(marker))
         except ParseException as e:
             raise InvalidMarker(
                 f"Invalid marker: {marker!r}, parse error at "
-                f"{marker[e.loc : e.loc + 8]!r}"
+                f"{marker[e.position : e.position + 8]!r}"
             )
 
     def __str__(self) -> str:
@@ -224,5 +226,4 @@ def evaluate(self, environment: Optional[Dict[str, str]] = None) -> bool:
         current_environment = default_environment()
         if environment is not None:
             current_environment.update(environment)
-
         return _evaluate_markers(self._markers, current_environment)

packaging/requirements.py

@@ -2,13 +2,14 @@
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
 
-import re
-import string
 import urllib.parse
+from collections import namedtuple
 from typing import List, Optional as TOptional, Set
 
-from .markers import Marker
-from .specifiers import LegacySpecifier, Specifier, SpecifierSet
+from ._parser import parse_named_requirement
+from ._tokenizer import ParseException
+from .markers import InvalidMarker, Marker
+from .specifiers import SpecifierSet
 
 
 class InvalidRequirement(ValueError):
@@ -31,12 +32,13 @@ class Requirement:
     # TODO: Can we normalize the name and extra name?
 
     def __init__(self, requirement_string: str) -> None:
+        _RequirementTuple = namedtuple(
+            "_RequirementTuple", ["name", "url", "extras", "specifier", "marker"]
+        )
         try:
-            req = REQUIREMENT.parseString(requirement_string)
+            req = _RequirementTuple(*parse_named_requirement(requirement_string))
         except ParseException as e:
-            raise InvalidRequirement(
-                f'Parse error at "{ requirement_string[e.loc : e.loc + 8]!r}": {e.msg}'
-            )
+            raise InvalidRequirement(str(e))
 
         self.name: str = req.name
         if req.url:
@@ -51,9 +53,12 @@ def __init__(self, requirement_string: str) -> None:
             self.url: TOptional[str] = req.url
         else:
             self.url = None
-        self.extras: Set[str] = set(req.extras.asList() if req.extras else [])
+        self.extras: Set[str] = set(req.extras if req.extras else [])
         self.specifier: SpecifierSet = SpecifierSet(req.specifier)
-        self.marker: TOptional[Marker] = req.marker if req.marker else None
+        try:
+            self.marker: TOptional[Marker] = Marker(req.marker) if req.marker else None
+        except InvalidMarker as e:
+            raise InvalidRequirement(str(e))
 
     def __str__(self) -> str:
         parts: List[str] = [self.name]

tests/test_markers.py

@@ -161,6 +161,7 @@ def test_parses_valid(self, marker_string):
             "python_version",
             "(python_version)",
             "python_version >= 1.0 and (python_version)",
+            '(python_version == "2.7" and os_name == "linux"',
         ],
     )
     def test_parses_invalid(self, marker_string):

tests/test_requirements.py

@@ -5,7 +5,7 @@
 import pytest
 
 from packaging.markers import Marker
-from packaging.requirements import URL, URL_AND_MARKER, InvalidRequirement, Requirement
+from packaging.requirements import InvalidRequirement, Requirement
 from packaging.specifiers import SpecifierSet
 
 
@@ -59,10 +59,20 @@ def test_name_with_version(self):
         req = Requirement("name>=3")
         self._assert_requirement(req, "name", specifier=">=3")
 
+    def test_name_with_missing_version(self):
+        with pytest.raises(InvalidRequirement) as e:
+            Requirement("name>=")
+        assert "Missing version" in str(e)
+
     def test_version_with_parens_and_whitespace(self):
         req = Requirement("name (==4)")
         self._assert_requirement(req, "name", specifier="==4")
 
+    def test_version_with_missing_closing_paren(self):
+        with pytest.raises(InvalidRequirement) as e:
+            Requirement("name(==4")
+        assert "Closing right parenthesis is missing" in str(e)
+
     def test_name_with_multiple_versions(self):
         req = Requirement("name>=3,<2")
         self._assert_requirement(req, "name", specifier="<2,>=3")
@@ -79,21 +89,27 @@ def test_empty_extras(self):
         req = Requirement("foo[]")
         self._assert_requirement(req, "foo")
 
+    def test_unclosed_extras(self):
+        with pytest.raises(InvalidRequirement) as e:
+            Requirement("foo[")
+        assert "Closing square bracket is missing" in str(e)
+
     def test_url(self):
-        url_section = "@ http://example.com"
-        parsed = URL.parseString(url_section)
-        assert parsed.url == "http://example.com"
+        url_section = "test @ http://example.com"
+        req = Requirement(url_section)
+        self._assert_requirement(req, "test", "http://example.com", extras=[])
 
     def test_url_and_marker(self):
-        instring = "@ http://example.com ; os_name=='a'"
-        parsed = URL_AND_MARKER.parseString(instring)
-        assert parsed.url == "http://example.com"
-        assert str(parsed.marker) == 'os_name == "a"'
+        instring = "test @ http://example.com ; os_name=='a'"
+        req = Requirement(instring)
+        self._assert_requirement(
+            req, "test", "http://example.com", extras=[], marker='os_name == "a"'
+        )
 
     def test_invalid_url(self):
         with pytest.raises(InvalidRequirement) as e:
             Requirement("name @ gopher:/foo/com")
-        assert "Invalid URL: " in str(e.value)
+        assert "Invalid URL: " in str(e)
         assert "gopher:/foo/com" in str(e.value)
 
     def test_file_url(self):