impl services cython

Summary: Cython changes to enable build. The basic idea is the invese of py3 client. To produce serialized message, py3 server uses regular serializer with cpp2 struct. Previously, the cpp2 struct was the internal data member of thrift-py3 object. Now, the cpp2 struct is converted using thrift-python C-API from thrift-python struct composed inside thrift-py3 wrapper.

Reviewed By: prakashgayasen

Differential Revision: D71353613

fbshipit-source-id: ed901ae1f9ed9ea702c1f69203af3bf7625a2caa

Author: ahilger

thrift/compiler/generate/templates/py3/services.pyx.mustache

@@ -69,7 +69,8 @@ from thrift.python.common cimport (
     MetadataBox as __MetadataBox,
 )
 
-from thrift.py3.types cimport make_unique
+from thrift.py3.types cimport make_unique{{!
+}}{{#program:inplace_migrate?}}, deref_const{{/program:inplace_migrate?}}
 
 cimport folly.futures
 from folly.executor cimport get_executor
@@ -84,6 +85,9 @@ from thrift.py3.stream cimport cServerStream, cServerStreamPublisher, cResponseA
 {{/program:has_stream?}}
 cimport {{#program:py3Namespaces}}{{value}}.{{/program:py3Namespaces}}{{program:name}}.types as _{{#program:py3Namespaces}}{{value}}_{{/program:py3Namespaces}}{{program:name}}_types
 cimport {{#program:py3Namespaces}}{{value}}.{{/program:py3Namespaces}}{{program:name}}.cbindings as _{{#program:py3Namespaces}}{{value}}_{{/program:py3Namespaces}}{{program:name}}_cbindings
+{{#program:inplace_migrate?}}
+cimport {{#program:py3Namespaces}}{{value}}.{{/program:py3Namespaces}}{{program:name}}.thrift_converter as _{{#program:py3Namespaces}}{{value}}_{{/program:py3Namespaces}}{{program:name}}_thrift_converter
+{{/program:inplace_migrate?}}
 import {{#program:py3Namespaces}}{{value}}.{{/program:py3Namespaces}}{{program:name}}.types as _{{#program:py3Namespaces}}{{value}}_{{/program:py3Namespaces}}{{program:name}}_types
 {{#program:includeNamespaces}}
 {{#hasServices?}}
@@ -94,6 +98,9 @@ import {{#includeNamespace}}{{value}}.{{/includeNamespace}}services as _{{#inclu
 import {{#includeNamespace}}{{value}}.{{/includeNamespace}}types as _{{#includeNamespace}}{{value}}_{{/includeNamespace}}types
 cimport {{#includeNamespace}}{{value}}.{{/includeNamespace}}types as _{{#includeNamespace}}{{value}}_{{/includeNamespace}}types
 cimport {{#includeNamespace}}{{value}}.{{/includeNamespace}}cbindings as _{{#includeNamespace}}{{value}}_{{/includeNamespace}}cbindings
+{{#program:inplace_migrate?}}
+cimport {{#includeNamespace}}{{value}}.{{/includeNamespace}}thrift_converter as _{{#includeNamespace}}{{value}}_{{/includeNamespace}}thrift_converter
+{{/program:inplace_migrate?}}
 {{/hasTypes?}}
 {{/program:includeNamespaces}}
 

thrift/compiler/generate/templates/py3/services/callback.mustache

@@ -112,8 +112,13 @@ async def {{service:name}}_{{function:name}}_coro(
     {{#function:exceptions}}
     {{#field:type}}
     except {{> types/cython_python_type}} as ex:
-        promise.cPromise.setException(deref((<{{> types/cython_python_type}}> ex).{{> types/cpp_obj}}))
-    {{/field:type}}
+        promise.cPromise.setException{{!
+}}{{^program:inplace_migrate?}}{{!
+          }}(deref((<{{> types/cython_python_type}}> ex).{{> types/cpp_obj}})){{!
+}}{{/program:inplace_migrate?}}{{#program:inplace_migrate?}}{{#type:struct}}{{!
+          }}({{type:capi_converter_path}}.{{struct:name}}_convert_to_cpp(ex._to_python())){{!
+}}{{/type:struct}}{{/program:inplace_migrate?}}{{!
+    }}{{/field:type}}
     {{/function:exceptions}}
     except __ApplicationError as ex:
         # If the handler raised an ApplicationError convert it to a C++ one

thrift/compiler/generate/templates/py3/services/cython_cpp_to_python.mustache

@@ -36,9 +36,12 @@ to be converted from unicode to binary representation.
 }}{{#type:float?}}{{field:py_name}}{{/type:float?}}{{!
 }}{{#type:string?}}(deref({{field:py_name}})).data().decode('UTF-8'){{/type:string?}}{{!
 }}{{#type:binary?}}(deref({{field:py_name}})){{/type:binary?}}{{!
-}}{{#type:struct?}}{{> types/cython_python_type}}._create_FBTHRIFT_ONLY_DO_NOT_USE({{!
+}}{{#type:struct}}{{^program:inplace_migrate?}}{{!
+}}{{> types/cython_python_type}}._create_FBTHRIFT_ONLY_DO_NOT_USE({{!
   }}shared_ptr[{{> types/cython_cpp_type}}]({{field:py_name}}.release())){{!
-}}{{/type:struct?}}{{!
+}}{{/program:inplace_migrate?}}{{#program:inplace_migrate?}}{{!
+  }}{{type:capi_converter_path}}.{{struct:name}}_from_cpp(deref_const({{field:py_name}}))._to_py3(){{!
+}}{{/program:inplace_migrate?}}{{/type:struct}}{{!
 }}{{#type:container?}}{{!
   }}{{> types/container_from_cpp}}(deref({{field:py_name}})){{!
 }}{{/type:container?}}{{!

thrift/compiler/generate/templates/py3/services/cython_return_value.mustache

@@ -55,7 +55,11 @@ to the client.
 }}{{/type:binary?}}{{!
 }}{{#type:struct}}{{!
   }}{{^function:stack_arguments?}}make_unique[{{> types/cython_cpp_type}}]({{/function:stack_arguments?}}{{!
+  }}{{^program:inplace_migrate?}}{{!
     }}deref((<{{> types/cython_python_type}}?> result).{{> types/cpp_obj}}){{!
+  }}{{/program:inplace_migrate?}}{{#program:inplace_migrate?}}{{!
+    }}{{type:capi_converter_path}}.{{struct:name}}_convert_to_cpp(result._to_python()){{!
+  }}{{/program:inplace_migrate?}}{{!
   }}{{^function:stack_arguments?}}){{/function:stack_arguments?}}{{!
 }}{{/type:struct}}{{!
 }}{{#type:container?}}{{!

thrift/compiler/test/fixtures/basic/out/py3_inplace/gen-py3/test/fixtures/basic/module/services.pyx

@@ -48,7 +48,7 @@ from thrift.python.common cimport (
     MetadataBox as __MetadataBox,
 )
 
-from thrift.py3.types cimport make_unique
+from thrift.py3.types cimport make_unique, deref_const
 
 cimport folly.futures
 from folly.executor cimport get_executor
@@ -59,6 +59,7 @@ from folly.memory cimport to_shared_ptr as __to_shared_ptr
 
 cimport test.fixtures.basic.module.types as _test_fixtures_basic_module_types
 cimport test.fixtures.basic.module.cbindings as _test_fixtures_basic_module_cbindings
+cimport test.fixtures.basic.module.thrift_converter as _test_fixtures_basic_module_thrift_converter
 import test.fixtures.basic.module.types as _test_fixtures_basic_module_types
 
 cimport test.fixtures.basic.module.services_interface as _fbthrift_services_interface
@@ -604,7 +605,7 @@ async def FB303Service_simple_rpc_coro(
             cTApplicationExceptionType__UNKNOWN, (f'Application was cancelled on the server with message: {str(ex)}').encode('UTF-8')
         ))
     else:
-        promise.cPromise.setValue(make_unique[_test_fixtures_basic_module_cbindings.cReservedKeyword](deref((<_test_fixtures_basic_module_types.ReservedKeyword?> result)._cpp_obj_FBTHRIFT_ONLY_DO_NOT_USE)))
+        promise.cPromise.setValue(make_unique[_test_fixtures_basic_module_cbindings.cReservedKeyword](_test_fixtures_basic_module_thrift_converter.ReservedKeyword_convert_to_cpp(result._to_python())))
 
 async def FB303Service_onStartServing_coro(
     object self,

thrift/compiler/test/fixtures/py3/out/py3_inplace/gen-py3/module/services.pyx

@@ -48,7 +48,7 @@ from thrift.python.common cimport (
     MetadataBox as __MetadataBox,
 )
 
-from thrift.py3.types cimport make_unique
+from thrift.py3.types cimport make_unique, deref_const
 
 cimport folly.futures
 from folly.executor cimport get_executor
@@ -59,6 +59,7 @@ from folly.memory cimport to_shared_ptr as __to_shared_ptr
 
 cimport module.types as _module_types
 cimport module.cbindings as _module_cbindings
+cimport module.thrift_converter as _module_thrift_converter
 import module.types as _module_types
 
 cimport module.services_interface as _fbthrift_services_interface
@@ -995,7 +996,7 @@ cdef api void call_cy_SimpleService_get_value(
     unique_ptr[_module_cbindings.cSimpleStruct] simple_struct
 ) noexcept:
     cdef Promise_cint32_t __promise = Promise_cint32_t._fbthrift_create(cmove(cPromise))
-    arg_simple_struct = _module_types.SimpleStruct._create_FBTHRIFT_ONLY_DO_NOT_USE(shared_ptr[_module_cbindings.cSimpleStruct](simple_struct.release()))
+    arg_simple_struct = _module_thrift_converter.SimpleStruct_from_cpp(deref_const(simple_struct))._to_py3()
     __context = RequestContext._fbthrift_create(ctx)
     __context_token = __THRIFT_REQUEST_CONTEXT.set(__context)
     asyncio.get_event_loop().create_task(
@@ -1319,7 +1320,7 @@ cdef api void call_cy_SimpleService_complex_sum_i32(
     unique_ptr[_module_cbindings.cComplexStruct] counter
 ) noexcept:
     cdef Promise_cint32_t __promise = Promise_cint32_t._fbthrift_create(cmove(cPromise))
-    arg_counter = _module_types.ComplexStruct._create_FBTHRIFT_ONLY_DO_NOT_USE(shared_ptr[_module_cbindings.cComplexStruct](counter.release()))
+    arg_counter = _module_thrift_converter.ComplexStruct_from_cpp(deref_const(counter))._to_py3()
     __context = RequestContext._fbthrift_create(ctx)
     __context_token = __THRIFT_REQUEST_CONTEXT.set(__context)
     asyncio.get_event_loop().create_task(
@@ -1337,7 +1338,7 @@ cdef api void call_cy_SimpleService_repeat_name(
     unique_ptr[_module_cbindings.cComplexStruct] counter
 ) noexcept:
     cdef Promise_string __promise = Promise_string._fbthrift_create(cmove(cPromise))
-    arg_counter = _module_types.ComplexStruct._create_FBTHRIFT_ONLY_DO_NOT_USE(shared_ptr[_module_cbindings.cComplexStruct](counter.release()))
+    arg_counter = _module_thrift_converter.ComplexStruct_from_cpp(deref_const(counter))._to_py3()
     __context = RequestContext._fbthrift_create(ctx)
     __context_token = __THRIFT_REQUEST_CONTEXT.set(__context)
     asyncio.get_event_loop().create_task(
@@ -1643,7 +1644,7 @@ cdef api void call_cy_SimpleService_get_binary_union_struct(
     unique_ptr[_module_cbindings.cBinaryUnion] u
 ) noexcept:
     cdef Promise__module_cbindings_cBinaryUnionStruct __promise = Promise__module_cbindings_cBinaryUnionStruct._fbthrift_create(cmove(cPromise))
-    arg_u = _module_types.BinaryUnion._create_FBTHRIFT_ONLY_DO_NOT_USE(shared_ptr[_module_cbindings.cBinaryUnion](u.release()))
+    arg_u = _module_thrift_converter.BinaryUnion_from_cpp(deref_const(u))._to_py3()
     __context = RequestContext._fbthrift_create(ctx)
     __context_token = __THRIFT_REQUEST_CONTEXT.set(__context)
     asyncio.get_event_loop().create_task(
@@ -1981,7 +1982,7 @@ async def SimpleService_expected_exception_coro(
     try:
         result = await self.expected_exception()
     except _module_types.SimpleException as ex:
-        promise.cPromise.setException(deref((<_module_types.SimpleException> ex)._cpp_obj_FBTHRIFT_ONLY_DO_NOT_USE))
+        promise.cPromise.setException(_module_thrift_converter.SimpleException_convert_to_cpp(ex._to_python()))
     except __ApplicationError as ex:
         # If the handler raised an ApplicationError convert it to a C++ one
         promise.cPromise.setException(cTApplicationException(
@@ -2422,7 +2423,7 @@ async def SimpleService_get_struct_coro(
             cTApplicationExceptionType__UNKNOWN, (f'Application was cancelled on the server with message: {str(ex)}').encode('UTF-8')
         ))
     else:
-        promise.cPromise.setValue(make_unique[_module_cbindings.cSimpleStruct](deref((<_module_types.SimpleStruct?> result)._cpp_obj_FBTHRIFT_ONLY_DO_NOT_USE)))
+        promise.cPromise.setValue(make_unique[_module_cbindings.cSimpleStruct](_module_thrift_converter.SimpleStruct_convert_to_cpp(result._to_python())))
 
 async def SimpleService_fib_coro(
     object self,
@@ -2914,7 +2915,7 @@ async def SimpleService_get_binary_union_struct_coro(
             cTApplicationExceptionType__UNKNOWN, (f'Application was cancelled on the server with message: {str(ex)}').encode('UTF-8')
         ))
     else:
-        promise.cPromise.setValue(make_unique[_module_cbindings.cBinaryUnionStruct](deref((<_module_types.BinaryUnionStruct?> result)._cpp_obj_FBTHRIFT_ONLY_DO_NOT_USE)))
+        promise.cPromise.setValue(make_unique[_module_cbindings.cBinaryUnionStruct](_module_thrift_converter.BinaryUnionStruct_convert_to_cpp(result._to_python())))
 
 async def SimpleService_onStartServing_coro(
     object self,

thrift/compiler/test/fixtures/types/out/py3_inplace/gen-py3/apache/thrift/fixtures/types/module/services.pyx

@@ -48,7 +48,7 @@ from thrift.python.common cimport (
     MetadataBox as __MetadataBox,
 )
 
-from thrift.py3.types cimport make_unique
+from thrift.py3.types cimport make_unique, deref_const
 
 cimport folly.futures
 from folly.executor cimport get_executor
@@ -59,10 +59,12 @@ from folly.memory cimport to_shared_ptr as __to_shared_ptr
 
 cimport apache.thrift.fixtures.types.module.types as _apache_thrift_fixtures_types_module_types
 cimport apache.thrift.fixtures.types.module.cbindings as _apache_thrift_fixtures_types_module_cbindings
+cimport apache.thrift.fixtures.types.module.thrift_converter as _apache_thrift_fixtures_types_module_thrift_converter
 import apache.thrift.fixtures.types.module.types as _apache_thrift_fixtures_types_module_types
 import apache.thrift.fixtures.types.included.types as _apache_thrift_fixtures_types_included_types
 cimport apache.thrift.fixtures.types.included.types as _apache_thrift_fixtures_types_included_types
 cimport apache.thrift.fixtures.types.included.cbindings as _apache_thrift_fixtures_types_included_cbindings
+cimport apache.thrift.fixtures.types.included.thrift_converter as _apache_thrift_fixtures_types_included_thrift_converter
 
 cimport apache.thrift.fixtures.types.module.services_interface as _fbthrift_services_interface
 

thrift/lib/py3/types.h

@@ -37,6 +37,11 @@ std::shared_ptr<T> constant_shared_ptr(const T& x) {
   return std::shared_ptr<T>(std::shared_ptr<T>{}, const_cast<T*>(&x));
 }
 
+template <typename T>
+const T& deref_const(const std::unique_ptr<T>& ptr) {
+  return *ptr;
+}
+
 // the folly::remove_cvref_t conversions work around Cython's limitation on
 // const/reference qualifier when custom cpp.type with those are defined
 template <typename T, typename S>

thrift/lib/py3/types.pxd

@@ -50,6 +50,7 @@ cdef extern from "thrift/lib/py3/types.h" namespace "::thrift::py3" nogil:
     string_view get_field_name_by_index[T](size_t idx) except +
     object init_unicode_from_cpp(...)
     T* get_union_field_value[T](...) except +
+    const T& deref_const[T](const unique_ptr[T]& ptr)
 
     cdef cppclass set_iter[T]:
         set_iter()