From e2126341751c68486dd96d0d1c4feb4d6f3d243a Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Tue, 3 Oct 2023 16:12:37 -0700 Subject: [PATCH 1/4] Expand ranges when reading thread_siblings_list Handle both ranges and individual values --- legate/util/system.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/legate/util/system.py b/legate/util/system.py index ba48e6ac5..d164669fb 100644 --- a/legate/util/system.py +++ b/legate/util/system.py @@ -86,6 +86,13 @@ def LIB_PATH(self) -> str: def cpus(self) -> tuple[CPUInfo, ...]: """A list of CPUs on the system.""" + def expand_range(value: str) -> tuple[int, ...]: + if "-" not in value: + return tuple((int(value),)) + start, stop = value.split("-") + + return tuple(x for x in range(int(start), int(stop) + 1)) + N = multiprocessing.cpu_count() if sys.platform == "darwin": @@ -98,9 +105,11 @@ def cpus(self) -> tuple[CPUInfo, ...]: line = open( f"/sys/devices/system/cpu/cpu{i}/topology/thread_siblings_list" # noqa E501 ).read() - sibling_sets.add( - tuple(sorted(int(x) for x in line.strip().split(","))) - ) + + flattened = [] + for x in (expand_range(value) for value in line.strip().split(",")): + flattened += [y for y in x] + sibling_sets.add(tuple(sorted(flattened))) return tuple( CPUInfo(siblings) for siblings in sorted(sibling_sets) ) From f715090301d06de8eaf39525ea7517a34f950fc3 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Tue, 3 Oct 2023 16:53:49 -0700 Subject: [PATCH 2/4] Simplify with itertools.chain --- legate/util/system.py | 31 ++++++++++++++++----------- tests/unit/legate/util/test_system.py | 27 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/legate/util/system.py b/legate/util/system.py index d164669fb..3cd4481d2 100644 --- a/legate/util/system.py +++ b/legate/util/system.py @@ -19,6 +19,7 @@ import platform import sys from functools import cached_property +from itertools import chain from .fs import get_legate_paths, get_legion_paths from .types import CPUInfo, GPUInfo, LegatePaths, LegionPaths @@ -86,13 +87,6 @@ def LIB_PATH(self) -> str: def cpus(self) -> tuple[CPUInfo, ...]: """A list of CPUs on the system.""" - def expand_range(value: str) -> tuple[int, ...]: - if "-" not in value: - return tuple((int(value),)) - start, stop = value.split("-") - - return tuple(x for x in range(int(start), int(stop) + 1)) - N = multiprocessing.cpu_count() if sys.platform == "darwin": @@ -105,11 +99,16 @@ def expand_range(value: str) -> tuple[int, ...]: line = open( f"/sys/devices/system/cpu/cpu{i}/topology/thread_siblings_list" # noqa E501 ).read() - - flattened = [] - for x in (expand_range(value) for value in line.strip().split(",")): - flattened += [y for y in x] - sibling_sets.add(tuple(sorted(flattened))) + sibling_sets.add( + tuple( + sorted( + chain.from_iterable( + expand_range(r) + for r in line.strip().split(",") + ) + ) + ) + ) return tuple( CPUInfo(siblings) for siblings in sorted(sibling_sets) ) @@ -148,3 +147,11 @@ def gpus(self) -> tuple[GPUInfo, ...]: results.append(GPUInfo(i, info.total)) return tuple(results) + + +def expand_range(value: str) -> tuple[int, ...]: + if "-" not in value: + return tuple((int(value),)) + start, stop = value.split("-") + + return tuple(range(int(start), int(stop) + 1)) diff --git a/tests/unit/legate/util/test_system.py b/tests/unit/legate/util/test_system.py index 44e99a9c0..c6744dda1 100644 --- a/tests/unit/legate/util/test_system.py +++ b/tests/unit/legate/util/test_system.py @@ -16,6 +16,7 @@ import os import platform +from itertools import chain import pytest from pytest_mock import MockerFixture @@ -105,3 +106,29 @@ def test_gpus_osx(self) -> None: msg = "GPU execution is not available on OSX." with pytest.raises(RuntimeError, match=msg): s.gpus + + def test_expand_range(self) -> None: + def convert(line: str) -> tuple[int, ...]: + return tuple( + sorted( + chain.from_iterable( + m.expand_range(r) for r in line.strip().split(",") + ) + ) + ) + + old_line: str = "1" + new_line: tuple[int, ...] = (1,) + assert convert(old_line) == new_line + + old_line = "1,3" + new_line = (1, 3) + assert convert(old_line) == new_line + + old_line = "1-3" + new_line = (1, 2, 3) + assert convert(old_line) == new_line + + old_line = "1-3,4,5,8-12,15,20" + new_line = (1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 15, 20) + assert convert(old_line) == new_line From 15af2f9bc637533277a285373af53107a4c2d714 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Wed, 4 Oct 2023 11:00:05 -0700 Subject: [PATCH 3/4] Add tests for expand_range and extract_values --- legate/util/system.py | 24 ++++--- tests/unit/legate/util/test_system.py | 94 ++++++++++++++++++++------- 2 files changed, 83 insertions(+), 35 deletions(-) diff --git a/legate/util/system.py b/legate/util/system.py index 3cd4481d2..bcb7e58f2 100644 --- a/legate/util/system.py +++ b/legate/util/system.py @@ -99,16 +99,7 @@ def cpus(self) -> tuple[CPUInfo, ...]: line = open( f"/sys/devices/system/cpu/cpu{i}/topology/thread_siblings_list" # noqa E501 ).read() - sibling_sets.add( - tuple( - sorted( - chain.from_iterable( - expand_range(r) - for r in line.strip().split(",") - ) - ) - ) - ) + sibling_sets.add(extract_values(line.strip())) return tuple( CPUInfo(siblings) for siblings in sorted(sibling_sets) ) @@ -150,8 +141,21 @@ def gpus(self) -> tuple[GPUInfo, ...]: def expand_range(value: str) -> tuple[int, ...]: + if value == "": + return tuple() if "-" not in value: return tuple((int(value),)) start, stop = value.split("-") return tuple(range(int(start), int(stop) + 1)) + + +def extract_values(line: str) -> tuple[int, ...]: + return tuple( + sorted( + chain.from_iterable( + expand_range(r) + for r in line.strip().split(",") + ) + ) + ) diff --git a/tests/unit/legate/util/test_system.py b/tests/unit/legate/util/test_system.py index c6744dda1..5277e420d 100644 --- a/tests/unit/legate/util/test_system.py +++ b/tests/unit/legate/util/test_system.py @@ -107,28 +107,72 @@ def test_gpus_osx(self) -> None: with pytest.raises(RuntimeError, match=msg): s.gpus - def test_expand_range(self) -> None: - def convert(line: str) -> tuple[int, ...]: - return tuple( - sorted( - chain.from_iterable( - m.expand_range(r) for r in line.strip().split(",") - ) - ) - ) - - old_line: str = "1" - new_line: tuple[int, ...] = (1,) - assert convert(old_line) == new_line - - old_line = "1,3" - new_line = (1, 3) - assert convert(old_line) == new_line - - old_line = "1-3" - new_line = (1, 2, 3) - assert convert(old_line) == new_line - - old_line = "1-3,4,5,8-12,15,20" - new_line = (1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 15, 20) - assert convert(old_line) == new_line +class Test_expand_range: + + def test_errors(self) -> None: + msg = "" + with pytest.raises(ValueError): + m.expand_range("foo") + + def test_empty(self) -> None: + assert m.expand_range("") == () + + @pytest.mark.parametrize("val", ("0", "1", "12", "100")) + def test_single_number(self, val) -> None: + assert m.expand_range(val) == (int(val),) + + @pytest.mark.parametrize("val", ("0-10", "1-2", "12-25")) + def test_range(self, val) -> None: + start, stop = val.split('-') + assert m.expand_range(val) == tuple(range(int(start), int(stop) + 1)) + + +class Test_extract_values: + + def test_errors(self) -> None: + msg = "" + with pytest.raises(ValueError): + m.extract_values("foo") + + def test_empty(self) -> None: + assert m.extract_values("") == () + + @pytest.mark.parametrize("val", ("0", "1,2", "3,5,7")) + def test_individual(self, val) -> None: + expected = {"0": (0,), + "1,2": (1, 2), + "3,5,7": (3, 5, 7), + } + assert m.extract_values(val) == expected[val] + + @pytest.mark.parametrize("val", ("2,1", "8,5,3,2", "1,3,2,5,4,7,6")) + def test_individual_ordered(self, val) -> None: + expected = {"2,1": (1, 2), + "8,5,3,2": (2, 3, 5, 8), + "1,3,2,5,4,7,6": (1, 2, 3, 4, 5, 6, 7), + } + assert m.extract_values(val) == expected[val] + + @pytest.mark.parametrize("val", ("0-2", "0-2,4-5", "0-1,3-5,8-11")) + def test_range(self, val) -> None: + expected = {"0-2": (0, 1, 2), + "0-2,4-5": (0, 1, 2, 4, 5), + "0-1,3-5,8-11": (0, 1, 3, 4, 5, 8, 9, 10 ,11), + } + assert m.extract_values(val) == expected[val] + + @pytest.mark.parametrize("val", ("2-3,0-1", "0-1,4-5,2-3")) + def test_range_ordered(self, val) -> None: + expected = {"2-3,0-1": (0, 1, 2, 3), + "0-1,4-5,2-3": (0, 1, 2, 3, 4, 5), + } + assert m.extract_values(val) == expected[val] + + @pytest.mark.parametrize("val", ("0,1-2", "1-2,0", "0,1-2,3,4-5,6", "5-6,4,1-3,0")) + def test_mixed(self, val) -> None: + expected = {"0,1-2": (0, 1, 2), + "1-2,0": (0, 1, 2), + "0,1-2,3,4-5,6": (0, 1, 2, 3, 4, 5, 6), + "5-6,4,1-3,0": (0, 1, 2, 3, 4, 5, 6), + } + assert m.extract_values(val) == expected[val] From 0c2780df1451329dc60776acbcdde8f877a02e85 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Wed, 4 Oct 2023 11:02:22 -0700 Subject: [PATCH 4/4] Run pre-commit --- legate/util/system.py | 3 +- tests/unit/legate/util/test_system.py | 71 ++++++++++++++------------- 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/legate/util/system.py b/legate/util/system.py index bcb7e58f2..61a463ac6 100644 --- a/legate/util/system.py +++ b/legate/util/system.py @@ -154,8 +154,7 @@ def extract_values(line: str) -> tuple[int, ...]: return tuple( sorted( chain.from_iterable( - expand_range(r) - for r in line.strip().split(",") + expand_range(r) for r in line.strip().split(",") ) ) ) diff --git a/tests/unit/legate/util/test_system.py b/tests/unit/legate/util/test_system.py index 5277e420d..662527a31 100644 --- a/tests/unit/legate/util/test_system.py +++ b/tests/unit/legate/util/test_system.py @@ -16,7 +16,6 @@ import os import platform -from itertools import chain import pytest from pytest_mock import MockerFixture @@ -107,10 +106,9 @@ def test_gpus_osx(self) -> None: with pytest.raises(RuntimeError, match=msg): s.gpus -class Test_expand_range: +class Test_expand_range: def test_errors(self) -> None: - msg = "" with pytest.raises(ValueError): m.expand_range("foo") @@ -118,19 +116,17 @@ def test_empty(self) -> None: assert m.expand_range("") == () @pytest.mark.parametrize("val", ("0", "1", "12", "100")) - def test_single_number(self, val) -> None: + def test_single_number(self, val: str) -> None: assert m.expand_range(val) == (int(val),) @pytest.mark.parametrize("val", ("0-10", "1-2", "12-25")) - def test_range(self, val) -> None: - start, stop = val.split('-') + def test_range(self, val: str) -> None: + start, stop = val.split("-") assert m.expand_range(val) == tuple(range(int(start), int(stop) + 1)) class Test_extract_values: - def test_errors(self) -> None: - msg = "" with pytest.raises(ValueError): m.extract_values("foo") @@ -138,41 +134,48 @@ def test_empty(self) -> None: assert m.extract_values("") == () @pytest.mark.parametrize("val", ("0", "1,2", "3,5,7")) - def test_individual(self, val) -> None: - expected = {"0": (0,), - "1,2": (1, 2), - "3,5,7": (3, 5, 7), - } + def test_individual(self, val: str) -> None: + expected = { + "0": (0,), + "1,2": (1, 2), + "3,5,7": (3, 5, 7), + } assert m.extract_values(val) == expected[val] @pytest.mark.parametrize("val", ("2,1", "8,5,3,2", "1,3,2,5,4,7,6")) - def test_individual_ordered(self, val) -> None: - expected = {"2,1": (1, 2), - "8,5,3,2": (2, 3, 5, 8), - "1,3,2,5,4,7,6": (1, 2, 3, 4, 5, 6, 7), - } + def test_individual_ordered(self, val: str) -> None: + expected = { + "2,1": (1, 2), + "8,5,3,2": (2, 3, 5, 8), + "1,3,2,5,4,7,6": (1, 2, 3, 4, 5, 6, 7), + } assert m.extract_values(val) == expected[val] @pytest.mark.parametrize("val", ("0-2", "0-2,4-5", "0-1,3-5,8-11")) - def test_range(self, val) -> None: - expected = {"0-2": (0, 1, 2), - "0-2,4-5": (0, 1, 2, 4, 5), - "0-1,3-5,8-11": (0, 1, 3, 4, 5, 8, 9, 10 ,11), - } + def test_range(self, val: str) -> None: + expected = { + "0-2": (0, 1, 2), + "0-2,4-5": (0, 1, 2, 4, 5), + "0-1,3-5,8-11": (0, 1, 3, 4, 5, 8, 9, 10, 11), + } assert m.extract_values(val) == expected[val] @pytest.mark.parametrize("val", ("2-3,0-1", "0-1,4-5,2-3")) - def test_range_ordered(self, val) -> None: - expected = {"2-3,0-1": (0, 1, 2, 3), - "0-1,4-5,2-3": (0, 1, 2, 3, 4, 5), - } + def test_range_ordered(self, val: str) -> None: + expected = { + "2-3,0-1": (0, 1, 2, 3), + "0-1,4-5,2-3": (0, 1, 2, 3, 4, 5), + } assert m.extract_values(val) == expected[val] - @pytest.mark.parametrize("val", ("0,1-2", "1-2,0", "0,1-2,3,4-5,6", "5-6,4,1-3,0")) - def test_mixed(self, val) -> None: - expected = {"0,1-2": (0, 1, 2), - "1-2,0": (0, 1, 2), - "0,1-2,3,4-5,6": (0, 1, 2, 3, 4, 5, 6), - "5-6,4,1-3,0": (0, 1, 2, 3, 4, 5, 6), - } + @pytest.mark.parametrize( + "val", ("0,1-2", "1-2,0", "0,1-2,3,4-5,6", "5-6,4,1-3,0") + ) + def test_mixed(self, val: str) -> None: + expected = { + "0,1-2": (0, 1, 2), + "1-2,0": (0, 1, 2), + "0,1-2,3,4-5,6": (0, 1, 2, 3, 4, 5, 6), + "5-6,4,1-3,0": (0, 1, 2, 3, 4, 5, 6), + } assert m.extract_values(val) == expected[val]