lint and headers

Author: sfc-gh-jkew

modin/core/storage_formats/base/query_compiler.py

@@ -109,42 +109,46 @@ def axis_setter(self, labels):
 
 
 class QCCoercionCost(Enum):
-    '''
+    """
     Coercion costs between query compilers can be expressed
     as integers in the range -1 to 1000, where 1000 is
     considered impossible. Since coercsion costs can be a
-    function of many variables ( dataset size, partitioning, 
+    function of many variables ( dataset size, partitioning,
     network throughput, and query time ) we define a set range
     of cost values to simplify comparisons between two query
     compilers / engines in a unified way.
-    
+
     COST_UNKNOWN means we do not know the cost associated with changing
     query compilers.
-    
+
     COST_ZERO means there is no cost associated, or that the query compilers
     are the same.
-    
+
     COST_IMPOSSIBLE means the coercion is effectively impossible, which can
     occur if the target system is unable to store the data as a result
     of the coercion.
-    '''
+    """
+
     COST_UNKNOWN = -1
     COST_ZERO = 0
     COST_LOW = 250
     COST_MEDIUM = 500
     COST_HIGH = 750
     COST_IMPOSSIBLE = 1000
 
-    def validate_coercsion_cost(cost:QCCoercionCost):
-        if int(cost) < int(QCCoercionCost.COST_UNKNOWN) or int(cost) > int(QCCoercionCost.COST_IMPOSSIBLE):
-            raise ValueError("Query compiler coercsion cost out of range") 
-        
+    def validate_coercsion_cost(cost: QCCoercionCost):
+        if int(cost) < int(QCCoercionCost.COST_UNKNOWN) or int(cost) > int(
+            QCCoercionCost.COST_IMPOSSIBLE
+        ):
+            raise ValueError("Query compiler coercsion cost out of range")
+
     def __int__(self):
         return self.value
-    
+
     def __add__(self, other) -> int:
         return int(self) + int(other)
 
+
 # FIXME: many of the BaseQueryCompiler methods are hiding actual arguments
 # by using *args and **kwargs. They should be spread into actual parameters.
 # Currently actual arguments are placed in the methods docstrings, but since they're
@@ -286,17 +290,17 @@ def default_to_pandas(self, pandas_op, *args, **kwargs) -> Self:
         return self.__wrap_in_qc(result)
 
     def qc_engine_switch_cost(self, other_qc) -> dict[type, int]:
-        '''
+        """
         Coercion costs to and from other_qc
-        
+
         Returns a map of type to QCCoercionCost, where type is the type we are casting to.
         This provides a mechanism for the query compilers to provide information to
         modin on the cost of moving data to another query compiler ( or the other way ).
-        '''
+        """
         if isinstance(type(self), type(other_qc)):
-            return  {type(self): QCCoercionCost.COST_ZERO}
+            return {type(self): QCCoercionCost.COST_ZERO}
         return {}
-        
+
     # Abstract Methods and Fields: Must implement in children classes
     # In some cases, there you may be able to use the same implementation for
     # some of these abstract methods, but for the sake of generality they are

modin/core/storage_formats/pandas/query_compiler_caster.py

@@ -27,19 +27,23 @@
 
 from pandas.core.indexes.frozen import FrozenList
 
-from modin.core.storage_formats.base.query_compiler import BaseQueryCompiler, QCCoercionCost
+from modin.core.storage_formats.base.query_compiler import (
+    BaseQueryCompiler,
+    QCCoercionCost,
+)
 
 Fn = TypeVar("Fn", bound=Any)
 
+
 class QueryCompilerCasterCalculator:
-    
+
     def __init__(self):
         self._caster_costing_map = {}
         self._data_cls_map = {}
         self._qc_list = []
         self._qc_cls_list = []
         self._result_type = None
-    
+
     def add_query_compiler(self, query_compiler):
         if isinstance(query_compiler, type):
             # class
@@ -50,16 +54,16 @@ def add_query_compiler(self, query_compiler):
             self._qc_list.append(query_compiler)
             self._data_cls_map[qc_type] = query_compiler._modin_frame
         self._qc_cls_list.append(qc_type)
-    
+
     def calculate(self):
         if self._result_type is not None:
             return self._result_type
         if len(self._qc_cls_list) == 1:
             return self._qc_cls_list[0]
         if len(self._qc_cls_list) == 0:
             raise ValueError("No query compilers registered")
-        
-        for (qc_1, qc_2) in combinations(self._qc_list, 2):
+
+        for qc_1, qc_2 in combinations(self._qc_list, 2):
             costs_1 = qc_1.qc_engine_switch_cost(qc_2)
             costs_2 = qc_2.qc_engine_switch_cost(qc_1)
             self._add_cost_data(costs_1)
@@ -73,20 +77,24 @@ def calculate(self):
                 self._result_type = key
                 break
         return self._result_type
-            
-    def _add_cost_data(self, costs:dict):
+
+    def _add_cost_data(self, costs: dict):
         for k, v in costs.items():
             # filter out any extranious query compilers not in this operation
             if k in self._qc_cls_list:
                 QCCoercionCost.validate_coercsion_cost(v)
                 # Adds the costs associated with all coercions to a type, k
-                self._caster_costing_map[k] = v + self._caster_costing_map[k] if k in self._caster_costing_map else v
-    
+                self._caster_costing_map[k] = (
+                    v + self._caster_costing_map[k]
+                    if k in self._caster_costing_map
+                    else v
+                )
+
     def result_data_frame(self):
         qc_type = self.calculate()
         return self._data_cls_map[qc_type]
-        
-    
+
+
 class QueryCompilerCaster:
     """Cast all query compiler arguments of the member function to current query compiler."""
 
@@ -111,9 +119,7 @@ def __init_subclass__(
         apply_argument_cast(cls)
 
 
-def visit_nested_args(arguments, 
-                      current_qc:BaseQueryCompiler, 
-                      fn:callable):
+def visit_nested_args(arguments, current_qc: BaseQueryCompiler, fn: callable):
     """
     Cast all arguments in nested fashion to current query compiler.
 
@@ -166,7 +172,6 @@ def apply_argument_cast(obj: Fn) -> Fn:
     if isinstance(obj, type):
         all_attrs = dict(inspect.getmembers(obj))
 
-
         # This is required because inspect converts class methods to member functions
         current_class_attrs = vars(obj)
         for key in current_class_attrs:
@@ -216,38 +221,36 @@ def arg_needs_casting(arg):
             if isinstance(arg, current_qc_type):
                 return False
             return True
-        
+
         def register_query_compilers(arg):
             if not arg_needs_casting(arg):
                 return arg
             calculator.add_query_compiler(arg)
             return arg
-        
+
         def cast_to_qc(arg):
             if not arg_needs_casting(arg):
                 return arg
             qc_type = calculator.calculate()
-            if qc_type == None or qc_type == type(arg):
+            if qc_type is None or qc_type is type(arg):
                 return arg
             frame_data = calculator.result_data_frame()
             result = qc_type.from_pandas(arg.to_pandas(), frame_data)
             return result
-        
-            
+
         if isinstance(current_qc, BaseQueryCompiler):
             visit_nested_args(kwargs, current_qc, register_query_compilers)
             visit_nested_args(args, current_qc, register_query_compilers)
-            
+
             args = visit_nested_args(args, current_qc, cast_to_qc)
             kwargs = visit_nested_args(kwargs, current_qc, cast_to_qc)
 
-        
         qc = calculator.calculate()
 
-        if qc == None or qc == type(current_qc):
+        if qc is None or qc is type(current_qc):
             return obj(*args, **kwargs)
 
-        #breakpoint()
+        # breakpoint()
         # we need to cast current_qc to a new query compiler
         if qc != current_qc:
             data_cls = current_qc._modin_frame

modin/tests/pandas/native_df_interoperability/test_compiler_caster.py

@@ -1,4 +1,15 @@
-
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
 
 import pandas
 import pytest
@@ -7,80 +18,103 @@
 
 
 class CloudQC(NativeQueryCompiler):
-    'Represents a cloud-hosted query compiler'
+    "Represents a cloud-hosted query compiler"
+
     def __init__(self, pandas_frame):
         self._modin_frame = pandas_frame
         super().__init__(pandas_frame)
-      
+
     def qc_engine_switch_cost(self, other_qc):
-        return {CloudQC: QCCoercionCost.COST_ZERO,
-                ClusterQC: QCCoercionCost.COST_MEDIUM,
-                LocalMachineQC: QCCoercionCost.COST_HIGH,
-                PicoQC: QCCoercionCost.COST_IMPOSSIBLE}
+        return {
+            CloudQC: QCCoercionCost.COST_ZERO,
+            ClusterQC: QCCoercionCost.COST_MEDIUM,
+            LocalMachineQC: QCCoercionCost.COST_HIGH,
+            PicoQC: QCCoercionCost.COST_IMPOSSIBLE,
+        }
+
 
 class ClusterQC(NativeQueryCompiler):
-    'Represents a local network cluster query compiler'
+    "Represents a local network cluster query compiler"
+
     def __init__(self, pandas_frame):
         self._modin_frame = pandas_frame
         super().__init__(pandas_frame)
-        
+
     def qc_engine_switch_cost(self, other_qc):
-        return {CloudQC: QCCoercionCost.COST_MEDIUM,
-                ClusterQC: QCCoercionCost.COST_ZERO,
-                LocalMachineQC: QCCoercionCost.COST_MEDIUM,
-                PicoQC: QCCoercionCost.COST_HIGH}
-    
+        return {
+            CloudQC: QCCoercionCost.COST_MEDIUM,
+            ClusterQC: QCCoercionCost.COST_ZERO,
+            LocalMachineQC: QCCoercionCost.COST_MEDIUM,
+            PicoQC: QCCoercionCost.COST_HIGH,
+        }
+
+
 class LocalMachineQC(NativeQueryCompiler):
-    'Represents a local machine query compiler'
+    "Represents a local machine query compiler"
+
     def __init__(self, pandas_frame):
         self._modin_frame = pandas_frame
         super().__init__(pandas_frame)
-        
+
     def qc_engine_switch_cost(self, other_qc):
-        return {CloudQC: QCCoercionCost.COST_MEDIUM,
-                ClusterQC: QCCoercionCost.COST_LOW,
-                LocalMachineQC: QCCoercionCost.COST_ZERO,
-                PicoQC: QCCoercionCost.COST_MEDIUM}
+        return {
+            CloudQC: QCCoercionCost.COST_MEDIUM,
+            ClusterQC: QCCoercionCost.COST_LOW,
+            LocalMachineQC: QCCoercionCost.COST_ZERO,
+            PicoQC: QCCoercionCost.COST_MEDIUM,
+        }
+
 
 class PicoQC(NativeQueryCompiler):
-    'Represents a query compiler with very few resources'
+    "Represents a query compiler with very few resources"
+
     def __init__(self, pandas_frame):
         self._modin_frame = pandas_frame
         super().__init__(pandas_frame)
-        
+
     def qc_engine_switch_cost(self, other_qc):
-        return {CloudQC: QCCoercionCost.COST_LOW,
-                ClusterQC: QCCoercionCost.COST_LOW,
-                LocalMachineQC: QCCoercionCost.COST_LOW,
-                PicoQC: QCCoercionCost.COST_ZERO}
+        return {
+            CloudQC: QCCoercionCost.COST_LOW,
+            ClusterQC: QCCoercionCost.COST_LOW,
+            LocalMachineQC: QCCoercionCost.COST_LOW,
+            PicoQC: QCCoercionCost.COST_ZERO,
+        }
+
 
 @pytest.fixture()
 def cloud_df():
     return CloudQC(pandas.DataFrame([0, 1, 2]))
 
+
 @pytest.fixture()
 def cluster_df():
     return ClusterQC(pandas.DataFrame([0, 1, 2]))
 
+
 @pytest.fixture()
 def local_df():
     return LocalMachineQC(pandas.DataFrame([0, 1, 2]))
 
+
 @pytest.fixture()
 def pico_df():
     return PicoQC(pandas.DataFrame([0, 1, 2]))
 
+
 def test_two_same_qc_types_noop(pico_df):
     df3 = pico_df.concat(axis=1, other=pico_df)
-    assert(type(df3) == type(pico_df))
+    assert type(df3) is type(pico_df)
+
 
 def test_two_two_qc_types_rhs(pico_df, cluster_df):
     df3 = pico_df.concat(axis=1, other=cluster_df)
-    assert(type(df3) == type(cluster_df)) # should move to cluster
+    assert type(df3) is type(cluster_df)  # should move to cluster
+
 
 def test_two_two_qc_types_lhs(pico_df, cluster_df):
     df3 = cluster_df.concat(axis=1, other=pico_df)
-    assert(type(df3) == type(cluster_df)) # should move to cluster
+    assert type(df3) is type(cluster_df)  # should move to cluster
+
 
 @pytest.mark.parametrize(
     "df1, df2, df3, df4, result_type",
@@ -103,12 +137,15 @@ def test_mixed_dfs(df1, df2, df3, df4, result_type, request):
     df3 = request.getfixturevalue(df3)
     df4 = request.getfixturevalue(df4)
     result = df1.concat(axis=1, other=[df2, df3, df4])
-    assert(type(result) == result_type)
+    assert type(result) is result_type
+
 
 # This currently passes because we have no "max cost" associated
 # with a particular QC, so we would move all data to the PicoQC
 # As soon as we can represent "max-cost" the result of this operation
 # should be to move all dfs to the CloudQC
 def test_extreme_pico(pico_df, cloud_df):
-    result = cloud_df.concat(axis=1, other=[pico_df, pico_df, pico_df, pico_df, pico_df, pico_df, pico_df])
-    assert(type(result) == PicoQC)
+    result = cloud_df.concat(
+        axis=1, other=[pico_df, pico_df, pico_df, pico_df, pico_df, pico_df, pico_df]
+    )
+    assert type(result) is PicoQC