CMake: Support for XML documentation

Add new function generate_doc_source to python_callouts.cmake
Update Test extension to use generate_doc_source function and include generated file in the build.

Cleanup:
Fix godotcpp.py imports after rebase
Pre-Commit hook sorted python imports in doc_source_generator.py
- replace ${CMAKE_CURRENT_SOURCE_DIR} with ${godot-cpp_SOURCE_DIR} when referencing current working directory and script locations when invoking python scripts.

Co-authored-by: David Snopek <dsnopek@gmail.com>

Author: enetheru

cmake/python_callouts.cmake

@@ -18,8 +18,12 @@ Its usage is listed as:
 ]]
 function( build_profile_generate_trimmed_api BUILD_PROFILE INPUT_JSON OUTPUT_JSON )
     execute_process(
-            COMMAND "${Python3_EXECUTABLE}" "build_profile.py" "${BUILD_PROFILE}" "${INPUT_JSON}" "${OUTPUT_JSON}"
-            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+            COMMAND "${Python3_EXECUTABLE}"
+                "${godot-cpp_SOURCE_DIR}/build_profile.py"
+                "${BUILD_PROFILE}"
+                "${INPUT_JSON}"
+                "${OUTPUT_JSON}"
+            WORKING_DIRECTORY ${godot-cpp_SOURCE_DIR}
     )
 endfunction(  )
 
@@ -45,7 +49,7 @@ function( binding_generator_get_file_list OUT_VAR_NAME API_FILEPATH OUTPUT_DIR )
     string( REGEX REPLACE "\n *" " " PYTHON_SCRIPT "${PYTHON_SCRIPT}" )
 
     execute_process( COMMAND "${Python3_EXECUTABLE}" "-c" "${PYTHON_SCRIPT}"
-            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+            WORKING_DIRECTORY "${godot-cpp_SOURCE_DIR}"
             OUTPUT_VARIABLE GENERATED_FILES_LIST
             OUTPUT_STRIP_TRAILING_WHITESPACE
     )
@@ -91,9 +95,33 @@ function( binding_generator_generate_bindings API_FILE USE_TEMPLATE_GET_NODE, BI
     add_custom_command(OUTPUT ${GENERATED_FILES_LIST}
             COMMAND "${Python3_EXECUTABLE}" "-c" "${PYTHON_SCRIPT}"
             VERBATIM
-            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+            WORKING_DIRECTORY ${godot-cpp_SOURCE_DIR}
             MAIN_DEPENDENCY ${GODOT_GDEXTENSION_API_FILE}
-            DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/binding_generator.py
+            DEPENDS ${godot-cpp_SOURCE_DIR}/binding_generator.py
             COMMENT "Generating bindings"
     )
 endfunction(  )
+
+#[[ Generate doc_data.cpp
+The documentation displayed in the Godot editor is compiled into the extension.
+It takes a list of XML source files, and transforms them into a cpp file that
+is added to the sources list.]]
+function( generate_doc_source OUTPUT_PATH XML_SOURCES )
+    # Transform the CMake list into the content of a python list
+    # quote and join to form the interior of a python array
+    list( TRANSFORM XML_SOURCES REPLACE "(.*\.xml)" "'\\1'" )
+    list( JOIN XML_SOURCES "," XML_SOURCES )
+
+    # Python one-liner to run our command
+    # lists in CMake are just strings delimited by ';', so this works.
+    set( PYTHON_SCRIPT "from doc_source_generator import generate_doc_source"
+            "generate_doc_source( '${OUTPUT_PATH}', [${XML_SOURCES}] )" )
+
+    add_custom_command( OUTPUT "${OUTPUT_PATH}"
+            COMMAND "${Python3_EXECUTABLE}" "-c" "${PYTHON_SCRIPT}"
+            VERBATIM
+            WORKING_DIRECTORY "${godot-cpp_SOURCE_DIR}"
+            DEPENDS "${godot-cpp_SOURCE_DIR}/doc_source_generator.py"
+            COMMENT "Generating Doc Data"
+    )
+endfunction()

doc_source_generator.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+import glob
+import os
+import zlib
+
+
+def generate_doc_source(dst, source):
+    g = open(dst, "w", encoding="utf-8")
+    buf = ""
+    docbegin = ""
+    docend = ""
+    for src in source:
+        src_path = str(src)
+        if not src_path.endswith(".xml"):
+            continue
+        with open(src_path, "r", encoding="utf-8") as f:
+            content = f.read()
+        buf += content
+
+    buf = (docbegin + buf + docend).encode("utf-8")
+    decomp_size = len(buf)
+
+    # Use maximum zlib compression level to further reduce file size
+    # (at the cost of initial build times).
+    buf = zlib.compress(buf, zlib.Z_BEST_COMPRESSION)
+
+    g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
+    g.write("\n")
+    g.write("#include <godot_cpp/godot.hpp>\n")
+    g.write("\n")
+
+    g.write('static const char *_doc_data_hash = "' + str(hash(buf)) + '";\n')
+    g.write("static const int _doc_data_uncompressed_size = " + str(decomp_size) + ";\n")
+    g.write("static const int _doc_data_compressed_size = " + str(len(buf)) + ";\n")
+    g.write("static const unsigned char _doc_data_compressed[] = {\n")
+    for i in range(len(buf)):
+        g.write("\t" + str(buf[i]) + ",\n")
+    g.write("};\n")
+    g.write("\n")
+
+    g.write(
+        "static godot::internal::DocDataRegistration _doc_data_registration(_doc_data_hash, _doc_data_uncompressed_size, _doc_data_compressed_size, _doc_data_compressed);\n"
+    )
+    g.write("\n")
+
+    g.close()
+
+
+def scons_generate_doc_source(target, source, env):
+    generate_doc_source(str(target[0]), source)
+
+
+def generate_doc_source_from_directory(target, directory):
+    generate_doc_source(target, glob.glob(os.path.join(directory, "*.xml")))

test/CMakeLists.txt

@@ -3,11 +3,19 @@ Integration Testing
 -------------------
 
 The Test target used to validate changes in the GitHub CI.
-
 ]=======================================================================]
 
 message( STATUS "Testing Integration targets are enabled.")
 
+# Generate Doc Data
+file( GLOB_RECURSE DOC_XML
+    LIST_DIRECTORIES NO
+    CONFIGURE_DEPENDS
+    "${CMAKE_CURRENT_SOURCE_DIR}/doc_classes/*.xml" )
+
+set( DOC_DATA_SOURCE "${CMAKE_CURRENT_BINARY_DIR}/src/gen/doc_data.gen.cpp" )
+generate_doc_source( "${DOC_DATA_SOURCE}" "${DOC_XML}" )
+
 foreach( TARGET_ALIAS template_debug template_release editor )
     set( TARGET_NAME "godot-cpp.test.${TARGET_ALIAS}" )
     set( LINK_TARGET "godot-cpp::${TARGET_ALIAS}" )
@@ -23,6 +31,11 @@ foreach( TARGET_ALIAS template_debug template_release editor )
             src/tests.h
     )
 
+    # conditionally add doc data to compile output
+    if( TARGET_ALIAS MATCHES "editor|template_debug" )
+        target_sources( ${TARGET_NAME} PRIVATE "${DOC_DATA_SOURCE}" )
+    endif(   )
+
     target_link_libraries( ${TARGET_NAME} PRIVATE ${LINK_TARGET} )
 
     ### Get useful properties of the library

tools/godotcpp.py

@@ -12,6 +12,7 @@
 
 from binding_generator import _generate_bindings, _get_file_list, get_file_list
 from build_profile import generate_trimmed_api
+from doc_source_generator import scons_generate_doc_source
 
 
 def add_sources(sources, dir, extension):
@@ -377,51 +378,6 @@ def options(opts, env):
             tool.options(opts)
 
 
-def make_doc_source(target, source, env):
-    import zlib
-
-    dst = str(target[0])
-    g = open(dst, "w", encoding="utf-8")
-    buf = ""
-    docbegin = ""
-    docend = ""
-    for src in source:
-        src_path = str(src)
-        if not src_path.endswith(".xml"):
-            continue
-        with open(src_path, "r", encoding="utf-8") as f:
-            content = f.read()
-        buf += content
-
-    buf = (docbegin + buf + docend).encode("utf-8")
-    decomp_size = len(buf)
-
-    # Use maximum zlib compression level to further reduce file size
-    # (at the cost of initial build times).
-    buf = zlib.compress(buf, zlib.Z_BEST_COMPRESSION)
-
-    g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
-    g.write("\n")
-    g.write("#include <godot_cpp/godot.hpp>\n")
-    g.write("\n")
-
-    g.write('static const char *_doc_data_hash = "' + str(hash(buf)) + '";\n')
-    g.write("static const int _doc_data_uncompressed_size = " + str(decomp_size) + ";\n")
-    g.write("static const int _doc_data_compressed_size = " + str(len(buf)) + ";\n")
-    g.write("static const unsigned char _doc_data_compressed[] = {\n")
-    for i in range(len(buf)):
-        g.write("\t" + str(buf[i]) + ",\n")
-    g.write("};\n")
-    g.write("\n")
-
-    g.write(
-        "static godot::internal::DocDataRegistration _doc_data_registration(_doc_data_hash, _doc_data_uncompressed_size, _doc_data_compressed_size, _doc_data_compressed);\n"
-    )
-    g.write("\n")
-
-    g.close()
-
-
 def generate(env):
     # Default num_jobs to local cpu count if not user specified.
     # SCons has a peculiarity where user-specified options won't be overridden
@@ -554,7 +510,7 @@ def generate(env):
     env.Append(
         BUILDERS={
             "GodotCPPBindings": Builder(action=Action(scons_generate_bindings, "$GENCOMSTR"), emitter=scons_emit_files),
-            "GodotCPPDocData": Builder(action=make_doc_source),
+            "GodotCPPDocData": Builder(action=scons_generate_doc_source),
         }
     )
     env.AddMethod(_godot_cpp, "GodotCPP")