From ac7a4d1cab5843bce79c24e1775b829c2f9f0255 Mon Sep 17 00:00:00 2001 From: Nate Jensen Date: Mon, 26 Aug 2024 23:01:35 -0500 Subject: [PATCH 1/5] add new ClassEnquirer for declaring Python packages The AllowPythonClassEnquirer enables users to declare packages that should always be imported from Python. This can be used to resolve issues when Java package names on the classpath match Python package names and the desired import is of the Python package. Example use cases are py4j and tensorflow. --- .../java/jep/AllowPythonClassEnquirer.java | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/main/java/jep/AllowPythonClassEnquirer.java diff --git a/src/main/java/jep/AllowPythonClassEnquirer.java b/src/main/java/jep/AllowPythonClassEnquirer.java new file mode 100644 index 00000000..8cc8f0e2 --- /dev/null +++ b/src/main/java/jep/AllowPythonClassEnquirer.java @@ -0,0 +1,96 @@ +/** + * Copyright (c) 2024 JEP AUTHORS. + * + * This file is licensed under the the zlib/libpng License. + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any + * damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +package jep; + +import java.util.HashSet; +import java.util.Set; + +/** + *

+ * A {@link ClassEnquirer} that defines specific packages as Python packages. + * This implementation takes a delegate ClassEnquirer to delegate all import + * determinations as Java or Python except in the cases of the specific Python + * package names provided. Imports of the specified packages in the embedded + * Python interpreter will use the Python importer and not the Java importer. + * This is useful when you have package name conflicts between Java and Python. + * Some examples are projects like py4j and tensorflow which have Python package + * names matching Java package names. In those cases, using the default + * ClassEnquirer of {@link ClassList} will lead to the Java packages on the + * classpath taking precedence on imports instead of the Python package. That + * can be solved by using this ClassEnquirer with a delegate of ClassList's + * instance. + *

+ * + * @author Nate Jensen + * + * @since 4.2.1 + */ +public class AllowPythonClassEnquirer implements ClassEnquirer { + + protected ClassEnquirer delegate; + + protected Set pyPkgNames = new HashSet<>(); + + /** + * Constructor + * + * @param delegate + * the ClassEnquirer instance to delegate method calls to except + * in the case of the specified Python package names + * @param pythonPackageNames + * the names of Python packages that should not be treated as a + * potential import from Java and instead should immediately + * default to being imported by the normal Python importer + */ + public AllowPythonClassEnquirer(ClassEnquirer delegate, + String... pythonPackageNames) { + this.delegate = delegate; + for (String pyPkg : pythonPackageNames) { + pyPkgNames.add(pyPkg); + } + } + + @Override + public boolean isJavaPackage(String name) { + for (String pyPkg : pyPkgNames) { + if (name.startsWith(pyPkg)) { + return false; + } + } + + return delegate.isJavaPackage(name); + } + + @Override + public String[] getClassNames(String pkgName) { + return delegate.getClassNames(pkgName); + } + + @Override + public String[] getSubPackages(String pkgName) { + return delegate.getSubPackages(pkgName); + } + +} From 3d752e3aa48322c8bd3740296e8a74ae52c90629 Mon Sep 17 00:00:00 2001 From: Nate Jensen Date: Thu, 29 Aug 2024 22:21:52 -0500 Subject: [PATCH 2/5] add unit test for new ClassEnquirer --- .../java/jep/AllowPythonClassEnquirer.java | 2 +- .../jep/test/TestAllowPythonEnquirer.java | 128 ++++++++++++++++++ src/test/python/test_allow_python_enquirer.py | 9 ++ 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 src/test/java/jep/test/TestAllowPythonEnquirer.java create mode 100644 src/test/python/test_allow_python_enquirer.py diff --git a/src/main/java/jep/AllowPythonClassEnquirer.java b/src/main/java/jep/AllowPythonClassEnquirer.java index 8cc8f0e2..51c2dfba 100644 --- a/src/main/java/jep/AllowPythonClassEnquirer.java +++ b/src/main/java/jep/AllowPythonClassEnquirer.java @@ -75,7 +75,7 @@ public AllowPythonClassEnquirer(ClassEnquirer delegate, @Override public boolean isJavaPackage(String name) { for (String pyPkg : pyPkgNames) { - if (name.startsWith(pyPkg)) { + if (name.equals(pyPkg) || name.startsWith(pyPkg + ".")) { return false; } } diff --git a/src/test/java/jep/test/TestAllowPythonEnquirer.java b/src/test/java/jep/test/TestAllowPythonEnquirer.java new file mode 100644 index 00000000..7814e67f --- /dev/null +++ b/src/test/java/jep/test/TestAllowPythonEnquirer.java @@ -0,0 +1,128 @@ +package jep.test; + +import jep.AllowPythonClassEnquirer; +import jep.ClassEnquirer; +import jep.Interpreter; +import jep.JepConfig; +import jep.JepException; + +/** + * Tests for the AllowPythonClassEnquirer. + * + * Created: August 2024 + * + * @author Nate Jensen + */ +public class TestAllowPythonEnquirer { + + /** + * Mock naive ClassEnquirer that thinks every package is a Java package. + */ + public static class EverythingIsJavaClassEnquirer implements ClassEnquirer { + + @Override + public boolean isJavaPackage(String name) { + return true; + } + + @Override + public String[] getClassNames(String pkgName) { + return null; + } + + @Override + public String[] getSubPackages(String pkgName) { + return null; + } + + } + + public static void main(String[] args) throws JepException { + /* + * Test that the always Java enquirer fails to import a Python type that + * exists + */ + JepConfig config = new JepConfig(); + ClassEnquirer allJavaEnquirer = new EverythingIsJavaClassEnquirer(); + config.setClassEnquirer(allJavaEnquirer); + try (Interpreter interp = config.createSubInterpreter()) { + + boolean gotClassNotFoundException = false; + try { + interp.exec("import sys"); + interp.exec("if 'io' in sys.modules:\n" + + " sys.modules.pop('io')"); + interp.exec("from io import BytesIO"); + } catch (JepException e) { + if (e.getCause() instanceof ClassNotFoundException) { + /* + * Tested ok, we expected a failure to import io as it tried + * to import BytesIO from Java. ClassNotFoundException + * indicates it was a failure to import from Java. + */ + gotClassNotFoundException = true; + } + } + + if (!gotClassNotFoundException) { + System.err.println( + "Expected a failed Java import of 'from io import BytesIO'"); + System.exit(1); + } + } + + /* + * Test that the allow python enquirer does not delegate to the all Java + * enquirer when encountering io + */ + config = new JepConfig(); + config.setClassEnquirer( + new AllowPythonClassEnquirer(allJavaEnquirer, "io")); + try (Interpreter interp = config.createSubInterpreter()) { + interp.exec("import sys"); + interp.exec("if 'io' in sys.modules:\n sys.modules.pop('io')"); + interp.exec("from io import BytesIO"); + /* + * This should still work as java.util.HashMap is on the classpath + * and the enquirer should indicate it's a Java import + */ + interp.exec("from java.util import HashMap"); + } catch (Exception e) { + System.err.println( + "Expected a successful Python import of 'from io import BytesIO'" + + " and a successful Java import of 'from java.util import HashMap"); + System.exit(1); + } + + /* + * Test that even if a Java package is available, if it's declared as a + * Python package then it will go to the Python importer. + */ + config = new JepConfig(); + config.setClassEnquirer( + new AllowPythonClassEnquirer(allJavaEnquirer, "java")); + try (Interpreter interp = config.createSubInterpreter()) { + interp.exec("moduleNotFound = False"); + interp.exec("try:\n" + " from java.util import ArrayList\n" + + "except ModuleNotFoundError as e:\n" + + " moduleNotFound = True"); + boolean moduleNotFoundErrorWasRaised = interp + .getValue("moduleNotFound", Boolean.class); + if (!moduleNotFoundErrorWasRaised) { + System.err.println("Expected ModuleNotFoundError when running " + + "'from java import util'" + + " with ClassEnquirer that considers java package as a Python package"); + System.exit(1); + } + + /* + * Verify javax still works even though java was declared as a + * Python package (i.e. so it's a little smarter than doing just a + * String.startsWith()). It would throw an uncaught exception if the + * import failed. + */ + interp.exec("from javax.xml.bind import JAXB"); + } + } + +} diff --git a/src/test/python/test_allow_python_enquirer.py b/src/test/python/test_allow_python_enquirer.py new file mode 100644 index 00000000..c4d158ef --- /dev/null +++ b/src/test/python/test_allow_python_enquirer.py @@ -0,0 +1,9 @@ +import unittest + +from jep_pipe import jep_pipe +from jep_pipe import build_java_process_cmd + +class TestRunScript(unittest.TestCase): + + def test_compiledScript(self): + jep_pipe(build_java_process_cmd('jep.test.TestAllowPythonEnquirer')) From 178452e5a52aa2cc23b3082d939b7744c458fbb7 Mon Sep 17 00:00:00 2001 From: Nate Jensen Date: Fri, 30 Aug 2024 16:44:45 -0500 Subject: [PATCH 3/5] fix unit test working with Java 11 and 17 --- .../java/jep/test/TestAllowPythonEnquirer.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/test/java/jep/test/TestAllowPythonEnquirer.java b/src/test/java/jep/test/TestAllowPythonEnquirer.java index 7814e67f..d4c90755 100644 --- a/src/test/java/jep/test/TestAllowPythonEnquirer.java +++ b/src/test/java/jep/test/TestAllowPythonEnquirer.java @@ -88,6 +88,7 @@ public static void main(String[] args) throws JepException { */ interp.exec("from java.util import HashMap"); } catch (Exception e) { + e.printStackTrace(); System.err.println( "Expected a successful Python import of 'from io import BytesIO'" + " and a successful Java import of 'from java.util import HashMap"); @@ -100,10 +101,10 @@ public static void main(String[] args) throws JepException { */ config = new JepConfig(); config.setClassEnquirer( - new AllowPythonClassEnquirer(allJavaEnquirer, "java")); + new AllowPythonClassEnquirer(allJavaEnquirer, "java.lang.ref")); try (Interpreter interp = config.createSubInterpreter()) { interp.exec("moduleNotFound = False"); - interp.exec("try:\n" + " from java.util import ArrayList\n" + interp.exec("try:\n" + " from java.lang.ref import Reference\n" + "except ModuleNotFoundError as e:\n" + " moduleNotFound = True"); boolean moduleNotFoundErrorWasRaised = interp @@ -116,12 +117,12 @@ public static void main(String[] args) throws JepException { } /* - * Verify javax still works even though java was declared as a - * Python package (i.e. so it's a little smarter than doing just a - * String.startsWith()). It would throw an uncaught exception if the - * import failed. + * Verify java.lang.reflect still works even though java.lang.ref + * was declared as a Python package (i.e. so it's a little smarter + * than doing just a String.startsWith()). It would throw an + * uncaught exception if the import failed. */ - interp.exec("from javax.xml.bind import JAXB"); + interp.exec("from java.lang.reflect import Method"); } } From 541f304ee52d3acb5cbbfd98977323815a077b4c Mon Sep 17 00:00:00 2001 From: Nate Jensen Date: Fri, 30 Aug 2024 22:16:35 -0500 Subject: [PATCH 4/5] improve unit test to work against newer python --- .../jep/test/TestAllowPythonEnquirer.java | 48 +++++++++---------- src/test/python/test_allow_python_enquirer.py | 4 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/test/java/jep/test/TestAllowPythonEnquirer.java b/src/test/java/jep/test/TestAllowPythonEnquirer.java index d4c90755..6179dd87 100644 --- a/src/test/java/jep/test/TestAllowPythonEnquirer.java +++ b/src/test/java/jep/test/TestAllowPythonEnquirer.java @@ -16,13 +16,14 @@ public class TestAllowPythonEnquirer { /** - * Mock naive ClassEnquirer that thinks every package is a Java package. + * Mock naive ClassEnquirer that thinks Java packages are re and java. */ - public static class EverythingIsJavaClassEnquirer implements ClassEnquirer { + public static class ReAndJavaClassEnquirer implements ClassEnquirer { @Override public boolean isJavaPackage(String name) { - return true; + return "re".equals(name) || "java".equals(name) + || name.startsWith("java."); } @Override @@ -39,49 +40,47 @@ public String[] getSubPackages(String pkgName) { public static void main(String[] args) throws JepException { /* - * Test that the always Java enquirer fails to import a Python type that + * Test that the re and java enquirer fails to import a Python type that * exists */ JepConfig config = new JepConfig(); - ClassEnquirer allJavaEnquirer = new EverythingIsJavaClassEnquirer(); - config.setClassEnquirer(allJavaEnquirer); + ClassEnquirer reJavaEnquirer = new ReAndJavaClassEnquirer(); + config.setClassEnquirer(reJavaEnquirer); try (Interpreter interp = config.createSubInterpreter()) { boolean gotClassNotFoundException = false; try { - interp.exec("import sys"); - interp.exec("if 'io' in sys.modules:\n" - + " sys.modules.pop('io')"); - interp.exec("from io import BytesIO"); + interp.exec("from re import Pattern"); } catch (JepException e) { if (e.getCause() instanceof ClassNotFoundException) { /* - * Tested ok, we expected a failure to import io as it tried - * to import BytesIO from Java. ClassNotFoundException - * indicates it was a failure to import from Java. + * Tested ok, we expected a failure to import as it tried to + * import Pattern from re package as if it was a Java class. + * ClassNotFoundException indicates it was a failure to + * import from Java. */ gotClassNotFoundException = true; + } else { + throw e; } } if (!gotClassNotFoundException) { System.err.println( - "Expected a failed Java import of 'from io import BytesIO'"); + "Expected a failed Java import of 'from re import Pattern'"); System.exit(1); } } /* - * Test that the allow python enquirer does not delegate to the all Java - * enquirer when encountering io + * Test that the allow python enquirer does not delegate to the re and java + * enquirer when encountering re */ config = new JepConfig(); config.setClassEnquirer( - new AllowPythonClassEnquirer(allJavaEnquirer, "io")); + new AllowPythonClassEnquirer(reJavaEnquirer, "re")); try (Interpreter interp = config.createSubInterpreter()) { - interp.exec("import sys"); - interp.exec("if 'io' in sys.modules:\n sys.modules.pop('io')"); - interp.exec("from io import BytesIO"); + interp.exec("from re import Pattern"); /* * This should still work as java.util.HashMap is on the classpath * and the enquirer should indicate it's a Java import @@ -90,7 +89,7 @@ public static void main(String[] args) throws JepException { } catch (Exception e) { e.printStackTrace(); System.err.println( - "Expected a successful Python import of 'from io import BytesIO'" + "Expected a successful Python import of 'from re import Pattern'" + " and a successful Java import of 'from java.util import HashMap"); System.exit(1); } @@ -101,7 +100,7 @@ public static void main(String[] args) throws JepException { */ config = new JepConfig(); config.setClassEnquirer( - new AllowPythonClassEnquirer(allJavaEnquirer, "java.lang.ref")); + new AllowPythonClassEnquirer(reJavaEnquirer, "java.lang.ref")); try (Interpreter interp = config.createSubInterpreter()) { interp.exec("moduleNotFound = False"); interp.exec("try:\n" + " from java.lang.ref import Reference\n" @@ -111,8 +110,9 @@ public static void main(String[] args) throws JepException { .getValue("moduleNotFound", Boolean.class); if (!moduleNotFoundErrorWasRaised) { System.err.println("Expected ModuleNotFoundError when running " - + "'from java import util'" - + " with ClassEnquirer that considers java package as a Python package"); + + "'from java.lang.ref import Reference'" + + " with ClassEnquirer that considers the " + + "java.lang.ref package as a Python package"); System.exit(1); } diff --git a/src/test/python/test_allow_python_enquirer.py b/src/test/python/test_allow_python_enquirer.py index c4d158ef..e4847c2e 100644 --- a/src/test/python/test_allow_python_enquirer.py +++ b/src/test/python/test_allow_python_enquirer.py @@ -3,7 +3,7 @@ from jep_pipe import jep_pipe from jep_pipe import build_java_process_cmd -class TestRunScript(unittest.TestCase): +class TestAllowPythonEnquirer(unittest.TestCase): - def test_compiledScript(self): + def test_allow_python_enquirer(self): jep_pipe(build_java_process_cmd('jep.test.TestAllowPythonEnquirer')) From 68295111274daa829a561372a8b3b28ba3bea48c Mon Sep 17 00:00:00 2001 From: Nate Jensen Date: Mon, 2 Sep 2024 21:15:21 -0500 Subject: [PATCH 5/5] sys.modules.pop('re') if in test's sub-interpreter --- src/test/java/jep/test/TestAllowPythonEnquirer.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/java/jep/test/TestAllowPythonEnquirer.java b/src/test/java/jep/test/TestAllowPythonEnquirer.java index 6179dd87..33c8fc3d 100644 --- a/src/test/java/jep/test/TestAllowPythonEnquirer.java +++ b/src/test/java/jep/test/TestAllowPythonEnquirer.java @@ -50,6 +50,9 @@ public static void main(String[] args) throws JepException { boolean gotClassNotFoundException = false; try { + interp.exec("import sys"); + interp.exec("if 're' in sys.modules:\n" + + " sys.modules.pop('re')"); interp.exec("from re import Pattern"); } catch (JepException e) { if (e.getCause() instanceof ClassNotFoundException) { @@ -73,8 +76,8 @@ public static void main(String[] args) throws JepException { } /* - * Test that the allow python enquirer does not delegate to the re and java - * enquirer when encountering re + * Test that the allow python enquirer does not delegate to the re and + * java enquirer when encountering re */ config = new JepConfig(); config.setClassEnquirer(