Changes to integrate new parser into packaging + test adjustments

Author: hrnciar

packaging/markers.py

@@ -8,6 +8,8 @@
 import sys
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
+from ._parser import parse_quoted_marker
+from ._tokenizer import ParseException, Tokenizer
 from .specifiers import InvalidSpecifier, Specifier
 
 __all__ = [
@@ -199,11 +201,11 @@ def default_environment() -> Dict[str, str]:
 class Marker:
     def __init__(self, marker: str) -> None:
         try:
-            self._markers = _coerce_parse_result(MARKER.parseString(marker))
+            self._markers = parse_quoted_marker(Tokenizer(marker))
         except ParseException as e:
             raise InvalidMarker(
                 f"Invalid marker: {marker!r}, parse error at "
-                f"{marker[e.loc : e.loc + 8]!r}"
+                f"{marker[e.position : e.position + 8]!r}"
             )
 
     def __str__(self) -> str:
@@ -224,5 +226,4 @@ def evaluate(self, environment: Optional[Dict[str, str]] = None) -> bool:
         current_environment = default_environment()
         if environment is not None:
             current_environment.update(environment)
-
         return _evaluate_markers(self._markers, current_environment)

packaging/requirements.py

@@ -2,13 +2,14 @@
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
 
-import re
-import string
 import urllib.parse
+from collections import namedtuple
 from typing import List, Optional as TOptional, Set
 
-from .markers import Marker
-from .specifiers import LegacySpecifier, Specifier, SpecifierSet
+from ._parser import parse_named_requirement
+from ._tokenizer import ParseException
+from .markers import InvalidMarker, Marker
+from .specifiers import SpecifierSet
 
 
 class InvalidRequirement(ValueError):
@@ -31,12 +32,13 @@ class Requirement:
     # TODO: Can we normalize the name and extra name?
 
     def __init__(self, requirement_string: str) -> None:
+        _RequirementTuple = namedtuple(
+            "_RequirementTuple", ["name", "url", "extras", "specifier", "marker"]
+        )
         try:
-            req = REQUIREMENT.parseString(requirement_string)
+            req = _RequirementTuple(*parse_named_requirement(requirement_string))
         except ParseException as e:
-            raise InvalidRequirement(
-                f'Parse error at "{ requirement_string[e.loc : e.loc + 8]!r}": {e.msg}'
-            )
+            raise InvalidRequirement(str(e))
 
         self.name: str = req.name
         if req.url:
@@ -51,9 +53,12 @@ def __init__(self, requirement_string: str) -> None:
             self.url: TOptional[str] = req.url
         else:
             self.url = None
-        self.extras: Set[str] = set(req.extras.asList() if req.extras else [])
+        self.extras: Set[str] = set(req.extras if req.extras else [])
         self.specifier: SpecifierSet = SpecifierSet(req.specifier)
-        self.marker: TOptional[Marker] = req.marker if req.marker else None
+        try:
+            self.marker: TOptional[Marker] = Marker(req.marker) if req.marker else None
+        except InvalidMarker as e:
+            raise InvalidRequirement(str(e))
 
     def __str__(self) -> str:
         parts: List[str] = [self.name]

tests/test_requirements.py

@@ -5,7 +5,7 @@
 import pytest
 
 from packaging.markers import Marker
-from packaging.requirements import URL, URL_AND_MARKER, InvalidRequirement, Requirement
+from packaging.requirements import InvalidRequirement, Requirement
 from packaging.specifiers import SpecifierSet
 
 
@@ -80,20 +80,21 @@ def test_empty_extras(self):
         self._assert_requirement(req, "foo")
 
     def test_url(self):
-        url_section = "@ http://example.com"
-        parsed = URL.parseString(url_section)
-        assert parsed.url == "http://example.com"
+        url_section = "test @ http://example.com"
+        req = Requirement(url_section)
+        self._assert_requirement(req, "test", "http://example.com", extras=[])
 
     def test_url_and_marker(self):
-        instring = "@ http://example.com ; os_name=='a'"
-        parsed = URL_AND_MARKER.parseString(instring)
-        assert parsed.url == "http://example.com"
-        assert str(parsed.marker) == 'os_name == "a"'
+        instring = "test @ http://example.com ; os_name=='a'"
+        req = Requirement(instring)
+        self._assert_requirement(
+            req, "test", "http://example.com", extras=[], marker='os_name == "a"'
+        )
 
     def test_invalid_url(self):
         with pytest.raises(InvalidRequirement) as e:
             Requirement("name @ gopher:/foo/com")
-        assert "Invalid URL: " in str(e.value)
+        assert "Invalid URL: " in str(e)
         assert "gopher:/foo/com" in str(e.value)
 
     def test_file_url(self):