[flow] Make the defense against cyclic tvars more robust

Summary:
It turns out that it's still possible to crash with cyclic tvars under the builtin defense.

In `ForcingState.map`, we are directly mapping a potentially bad cyclic lazy payload.

This diff adds in extra defense. I added a test (`invalid_self_recursive_mapped`) which will fail without the defense.

Note that these are edge cases. In practice, the lazy cyclic tvar we created are not that crazy and usually has extra indirections like `AnnotT` and `EvalT`, so that's why it hasn't blown up LTI 2.0 yet.

Changelog: [internal]

Reviewed By: panagosg7

Differential Revision: D55728897

fbshipit-source-id: ea3809edfbf47385ef125eb43d46f2561808a642

Author: SamChou19815

src/typing/__tests__/type_test.ml

@@ -0,0 +1,62 @@
+(*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open OUnit2
+open Type
+
+let assert_forced_to_any s =
+  match Constraint.ForcingState.force ~on_error:AnyT.error s with
+  | AnyT _ -> ()
+  | _ -> failwith "Invalid type"
+
+let forcing_state_tests =
+  [
+    ( "invalid_self_recursive" >:: fun _ ->
+      let rec s_lazy =
+        lazy
+          (Constraint.ForcingState.of_lazy_t
+             ~error_reason:Reason.(locationless_reason RNull)
+             (lazy (Constraint.ForcingState.force ~on_error:AnyT.error (Lazy.force s_lazy)))
+          )
+      in
+      let s = Lazy.force s_lazy in
+      assert_forced_to_any s
+    );
+    ( "invalid_mutually_recursive" >:: fun _ ->
+      let rec s1_lazy =
+        lazy
+          (Constraint.ForcingState.of_lazy_t
+             ~error_reason:Reason.(locationless_reason RNull)
+             (lazy (Constraint.ForcingState.force ~on_error:AnyT.error (Lazy.force s2_lazy)))
+          )
+      and s2_lazy =
+        lazy
+          (Constraint.ForcingState.of_lazy_t
+             ~error_reason:Reason.(locationless_reason RNull)
+             (lazy (Constraint.ForcingState.force ~on_error:AnyT.error (Lazy.force s1_lazy)))
+          )
+      in
+      let s1 = Lazy.force s1_lazy in
+      let s2 = Lazy.force s2_lazy in
+      assert_forced_to_any s1;
+      assert_forced_to_any s2
+    );
+    ( "invalid_self_recursive_mapped" >:: fun _ ->
+      let rec s_lazy =
+        lazy
+          (Constraint.ForcingState.of_lazy_t
+             ~error_reason:Reason.(locationless_reason RNull)
+             (lazy (Constraint.ForcingState.force ~on_error:AnyT.error (Lazy.force s_lazy)))
+          )
+      in
+      let s = Lazy.force s_lazy in
+      let s' = Constraint.ForcingState.map ~on_error:AnyT.error ~f:Base.Fn.id s in
+      assert_forced_to_any s'
+    );
+  ]
+
+let tests = "type" >::: ["forcing_state" >::: forcing_state_tests]

src/typing/__tests__/typing_tests.ml

@@ -7,7 +7,7 @@
 
 open OUnit2
 
-let tests = "typing" >::: [Type_hint_test.tests]
+let tests = "typing" >::: [Type_test.tests; Type_hint_test.tests]
 
 let _handle =
   let one_gig = 1024 * 1024 * 1024 in

src/typing/context.ml

@@ -984,14 +984,14 @@ let find_root cx id = Type.Constraint.find_root cx.ccx.sig_cx.graph id
 
 let find_root_id cx id = Type.Constraint.find_root_id cx.ccx.sig_cx.graph id
 
+let on_cyclic_tvar_error cx reason =
+  let msg = Error_message.(ETrivialRecursiveDefinition (Reason.loc_of_reason reason, reason)) in
+  let error = Flow_error.error_of_msg ~trace_reasons:[] ~source_file:cx.file msg in
+  add_error cx error;
+  Type.AnyT.error reason
+
 let force_fully_resolved_tvar cx =
-  let on_error reason =
-    let msg = Error_message.(ETrivialRecursiveDefinition (Reason.loc_of_reason reason, reason)) in
-    let error = Flow_error.error_of_msg ~trace_reasons:[] ~source_file:cx.file msg in
-    add_error cx error;
-    Type.AnyT.error reason
-  in
-  Type.Constraint.ForcingState.force ~on_error
+  Type.Constraint.ForcingState.force ~on_error:(on_cyclic_tvar_error cx)
 
 let find_resolved =
   let rec loop cx seen t_in =

src/typing/context.mli

@@ -471,6 +471,8 @@ val find_root : t -> Type.ident -> Type.ident * Type.Constraint.node * Type.Cons
 
 val find_root_id : t -> Type.ident -> Type.ident
 
+val on_cyclic_tvar_error : t -> Reason.reason -> Type.t
+
 val force_fully_resolved_tvar : t -> Type.Constraint.ForcingState.t -> Type.t
 
 val find_resolved : t -> Type.t -> Type.t option

src/typing/merge_js.ml

@@ -648,6 +648,7 @@ let copier =
               failwith "unexpected unresolved constraint"
             | FullyResolved s ->
               ForcingState.map
+                ~on_error:(Context.on_cyclic_tvar_error src_cx)
                 ~f:(fun t ->
                   let (_ : Context.t) = self#type_ src_cx pole dst_cx t in
                   t)

src/typing/type.ml

@@ -3256,7 +3256,7 @@ module Constraint = struct
 
     val get_forced : t -> TypeTerm.t option
 
-    val map : f:(TypeTerm.t -> TypeTerm.t) -> t -> t
+    val map : on_error:(Reason.t -> TypeTerm.t) -> f:(TypeTerm.t -> TypeTerm.t) -> t -> t
   end = struct
     type state =
       | Unforced
@@ -3294,7 +3294,8 @@ module Constraint = struct
         None
       | Forced -> Some (Lazy.force_val s.valid)
 
-    let map ~f s = { valid = Lazy.map f s.valid; error_reason = s.error_reason; state = Unforced }
+    let map ~on_error ~f s =
+      { valid = lazy (f (force ~on_error s)); error_reason = s.error_reason; state = Unforced }
   end
 
   (** Constraints carry type information that narrows down the possible solutions