Add Java dependency analysis types and launcher using javaparser library.

src/python/pants/backend/java/dependency_inference/BUILD

@@ -1,5 +1,12 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-python_library()
+python_library(
+	dependencies=[':java_src']
+)
 python_tests(name="tests", timeout=240)
+
+resources(
+  name='java_src',
+  sources=['*.java'],
+)

src/python/pants/backend/java/dependency_inference/PantsJavaParserLauncher.java

@@ -0,0 +1,83 @@
+package org.pantsbuild.javaparser;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.javaparser.ast.body.TypeDeclaration;
+import com.github.javaparser.ast.CompilationUnit;
+import com.github.javaparser.ast.expr.Name;
+import com.github.javaparser.ast.ImportDeclaration;
+import com.github.javaparser.ast.PackageDeclaration;
+import com.github.javaparser.StaticJavaParser;
+import java.io.File;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+class Import {
+    Import(String name, boolean isStatic, boolean isAsterisk) {
+        this.name = name;
+        this.isStatic = isStatic;
+        this.isAsterisk = isAsterisk;
+    }
+
+    public static Import fromImportDeclaration(ImportDeclaration imp) {
+        return new Import(imp.getName().toString(), imp.isStatic(), imp.isAsterisk());
+    }
+
+    public final String name;
+    public final boolean isStatic;
+    public final boolean isAsterisk;
+}
+
+class CompilationUnitAnalysis {
+    CompilationUnitAnalysis(String declaredPackage, ArrayList<Import> imports, ArrayList<String> topLevelTypes) {
+        this.declaredPackage = declaredPackage;
+        this.imports = imports;
+        this.topLevelTypes = topLevelTypes;
+    }
+
+    public final String declaredPackage;
+    public final ArrayList<Import> imports;
+    public final ArrayList<String> topLevelTypes;
+}
+
+
+public class PantsJavaParserLauncher {
+    public static void main(String[] args) throws Exception {
+        String analysisOutputPath = args[0];
+        String sourceToAnalyze = args[1];
+
+        CompilationUnit cu = StaticJavaParser.parse(new File(sourceToAnalyze));
+
+        // Get the source's declare package.
+        String declaredPackage = cu.getPackageDeclaration()
+            .map(PackageDeclaration::getName)
+            .map(Name::toString)
+            .orElse("");
+
+        // Get the source's imports.
+        ArrayList<Import> imports = new ArrayList<Import>(
+            cu.getImports().stream()
+                .map(Import::fromImportDeclaration)
+                .collect(Collectors.toList()));
+
+        // Get the source's top level types
+        ArrayList<String> topLevelTypes = new ArrayList<String>(
+            cu.getTypes().stream()
+                .filter(TypeDeclaration::isTopLevelType)
+                .map(TypeDeclaration::getFullyQualifiedName)
+                // TODO(#12293): In Java 9+ we could just flatMap(Optional::stream),
+                // but we're not guaranteed Java 9+ yet.
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .collect(Collectors.toList()));
+
+        CompilationUnitAnalysis analysis = new CompilationUnitAnalysis(declaredPackage, imports, topLevelTypes);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.writeValue(new File(analysisOutputPath), analysis);
+    }
+}

src/python/pants/backend/java/dependency_inference/java_parser.py

@@ -0,0 +1,128 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import json
+import logging
+import os.path
+from dataclasses import dataclass
+
+from pants.backend.java.dependency_inference.java_parser_launcher import (
+    JavaParserCompiledClassfiles,
+    java_parser_artifact_requirements,
+)
+from pants.backend.java.dependency_inference.types import JavaSourceDependencyAnalysis
+from pants.core.util_rules.source_files import SourceFiles
+from pants.engine.fs import AddPrefix, Digest, DigestContents, MergeDigests
+from pants.engine.process import BashBinary, FallibleProcessResult, Process, ProcessExecutionFailure
+from pants.engine.rules import Get, collect_rules, rule
+from pants.jvm.resolve.coursier_fetch import MaterializedClasspath, MaterializedClasspathRequest
+from pants.jvm.resolve.coursier_setup import Coursier
+from pants.util.logging import LogLevel
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class FallibleJavaSourceDependencyAnalysisResult:
+    process_result: FallibleProcessResult
+
+
+@rule(level=LogLevel.DEBUG)
+async def resolve_fallible_result_to_analysis(
+    fallible_result: FallibleJavaSourceDependencyAnalysisResult,
+) -> JavaSourceDependencyAnalysis:
+    # TODO(#12725): Just convert directly to a ProcessResult like this:
+    # result = await Get(ProcessResult, FallibleProcessResult, fallible_result.process_result)
+    if fallible_result.process_result.exit_code == 0:
+        analysis_contents = await Get(
+            DigestContents, Digest, fallible_result.process_result.output_digest
+        )
+        analysis = json.loads(analysis_contents[0].content)
+        return JavaSourceDependencyAnalysis.from_json_dict(analysis)
+    raise ProcessExecutionFailure(
+        fallible_result.process_result.exit_code,
+        fallible_result.process_result.stdout,
+        fallible_result.process_result.stderr,
+        "Java source dependency analysis failed.",
+    )
+
+
+@rule(level=LogLevel.DEBUG)
+async def analyze_java_source_dependencies(
+    bash: BashBinary,
+    coursier: Coursier,
+    processor_classfiles: JavaParserCompiledClassfiles,
+    source_files: SourceFiles,
+) -> FallibleJavaSourceDependencyAnalysisResult:
+    if len(source_files.snapshot.files) > 1:
+        raise ValueError(
+            f"parse_java_package expects sources with exactly 1 source file, but found {len(source_files.snapshot.files)}."
+        )
+    elif len(source_files.snapshot.files) == 0:
+        raise ValueError(
+            "parse_java_package expects sources with exactly 1 source file, but found none."
+        )
+    source_prefix = "__source_to_analyze"
+    source_path = os.path.join(source_prefix, source_files.snapshot.files[0])
+    processorcp_relpath = "__processorcp"
+
+    tool_classpath = await Get(
+        MaterializedClasspath,
+        MaterializedClasspathRequest(
+            prefix="__toolcp",
+            artifact_requirements=(java_parser_artifact_requirements(),),
+        ),
+    )
+    prefixed_processor_classfiles_digest = await Get(
+        Digest, AddPrefix(processor_classfiles.digest, processorcp_relpath)
+    )
+    prefixed_source_files_digest = await Get(
+        Digest, AddPrefix(source_files.snapshot.digest, source_prefix)
+    )
+
+    merged_digest = await Get(
+        Digest,
+        MergeDigests(
+            (
+                prefixed_processor_classfiles_digest,
+                tool_classpath.digest,
+                coursier.digest,
+                prefixed_source_files_digest,
+            )
+        ),
+    )
+
+    analysis_output_path = "__source_analysis.json"
+
+    proc = Process(
+        argv=[
+            coursier.coursier.exe,
+            "java",
+            "--system-jvm",  # TODO(#12293): use a fixed JDK version from a subsystem.
+            "-cp",
+            ":".join([tool_classpath.classpath_arg(), processorcp_relpath]),
+            "org.pantsbuild.javaparser.PantsJavaParserLauncher",
+            analysis_output_path,
+            source_path,
+        ],
+        input_digest=merged_digest,
+        output_files=(analysis_output_path,),
+        description="Run Spoon analysis against Java source",
+        level=LogLevel.DEBUG,
+    )
+
+    process_result = await Get(
+        FallibleProcessResult,
+        Process,
+        proc,
+    )
+
+    return FallibleJavaSourceDependencyAnalysisResult(process_result=process_result)
+
+
+def rules():
+    return [
+        *collect_rules(),
+    ]

src/python/pants/backend/java/dependency_inference/java_parser_launcher.py

@@ -0,0 +1,112 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import logging
+
+import pkg_resources
+
+from pants.backend.java.compile.javac import CompiledClassfiles
+from pants.backend.java.compile.javac_binary import JavacBinary
+from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests, RemovePrefix
+from pants.engine.process import BashBinary, Process, ProcessResult
+from pants.engine.rules import Get, collect_rules, rule
+from pants.jvm.resolve.coursier_fetch import (
+    ArtifactRequirements,
+    Coordinate,
+    MaterializedClasspath,
+    MaterializedClasspathRequest,
+)
+from pants.util.logging import LogLevel
+
+logger = logging.getLogger(__name__)
+
+
+_LAUNCHER_BASENAME = "PantsJavaParserLauncher.java"
+
+
+def _load_javaparser_launcher_source() -> bytes:
+    return pkg_resources.resource_string(__name__, _LAUNCHER_BASENAME)
+
+
+def java_parser_artifact_requirements() -> ArtifactRequirements:
+    return ArtifactRequirements(
+        [
+            Coordinate(
+                group="com.fasterxml.jackson.core", artifact="jackson-databind", version="2.12.4"
+            ),
+            Coordinate(
+                group="com.github.javaparser",
+                artifact="javaparser-symbol-solver-core",
+                version="3.23.0",
+            ),
+        ],
+    )
+
+
+class JavaParserCompiledClassfiles(CompiledClassfiles):
+    pass
+
+
+@rule
+async def build_processors(bash: BashBinary, javac: JavacBinary) -> JavaParserCompiledClassfiles:
+    materialized_classpath = await Get(
+        MaterializedClasspath,
+        MaterializedClasspathRequest(
+            prefix="__toolcp",
+            artifact_requirements=(java_parser_artifact_requirements(),),
+        ),
+    )
+
+    source_digest = await Get(
+        Digest,
+        CreateDigest(
+            [
+                FileContent(
+                    path=_LAUNCHER_BASENAME,
+                    content=_load_javaparser_launcher_source(),
+                )
+            ]
+        ),
+    )
+
+    merged_digest = await Get(
+        Digest,
+        MergeDigests(
+            (
+                materialized_classpath.digest,
+                javac.digest,
+                source_digest,
+            )
+        ),
+    )
+
+    process_result = await Get(
+        ProcessResult,
+        Process(
+            argv=[
+                bash.path,
+                javac.javac_wrapper_script,
+                "-cp",
+                materialized_classpath.classpath_arg(),
+                "-d",
+                "classfiles",
+                _LAUNCHER_BASENAME,
+            ],
+            input_digest=merged_digest,
+            output_directories=("classfiles",),
+            description=f"Compile {_LAUNCHER_BASENAME} import processors with javac",
+            level=LogLevel.DEBUG,
+        ),
+    )
+    stripped_classfiles_digest = await Get(
+        Digest, RemovePrefix(process_result.output_digest, "classfiles")
+    )
+    return JavaParserCompiledClassfiles(digest=stripped_classfiles_digest)
+
+
+def rules():
+    return [
+        *collect_rules(),
+    ]