diff --git a/middle_end/flambda2/flambda2.ml b/middle_end/flambda2/flambda2.ml index 206fb35ef09..16b8fa43c54 100644 --- a/middle_end/flambda2/flambda2.ml +++ b/middle_end/flambda2/flambda2.ml @@ -155,6 +155,10 @@ let lambda_to_cmm ~ppf_dump:ppf ~prefixname ~filename ~module_ident in Compiler_hooks.execute Raw_flambda2 raw_flambda; print_rawflambda ppf raw_flambda; + (if Flambda_features.inlining_report () + then + let output_prefix = prefixname ^ ".cps_conv" in + Inlining_report.output_then_forget_decisions ~output_prefix); let flambda, cmx, all_code = if Flambda_features.classic_mode () then raw_flambda, None, code diff --git a/middle_end/flambda2/from_lambda/closure_conversion.ml b/middle_end/flambda2/from_lambda/closure_conversion.ml index 90fde430b51..b99bc42a9fd 100644 --- a/middle_end/flambda2/from_lambda/closure_conversion.ml +++ b/middle_end/flambda2/from_lambda/closure_conversion.ml @@ -168,6 +168,202 @@ let find_simple acc env (simple : IR.simple) = let find_simples acc env ids = List.fold_left_map (fun acc id -> find_simple acc env id) acc ids +module Inlining = struct + type inlinable_result = + | Not_inlinable + | Inlinable of Code.t + + let threshold () = + let inline_threshold = + Clflags.Float_arg_helper.get ~key:0 !Clflags.inline_threshold + in + let magic_scale_constant = 8. in + int_of_float (inline_threshold *. magic_scale_constant) + + (* CR keryan: we need to emit warnings *) + let inlinable env apply = + let callee = Apply_expr.callee apply in + let dbg = Apply_expr.dbg apply in + match Env.find_value_approximation env callee with + | Value_unknown -> + Inlining_report.(record_decision ~dbg (At_call_site Unknown_function)); + Not_inlinable + | Block_approximation _ -> assert false + | Closure_approximation (code_id, None) -> + Inlining_report.record_decision ~dbg + (At_call_site + (Inlining_report.Known_function + { code_id = Code_id.export code_id; + decision = Definition_says_not_to_inline + })); + Not_inlinable + | Closure_approximation (code_id, Some code) -> + let fun_params_length = + Code.params_arity code |> Flambda_arity.With_subkinds.to_arity + |> Flambda_arity.length + in + if fun_params_length > List.length (Apply_expr.args apply) + then begin + Inlining_report.record_decision ~dbg + (At_call_site + (Inlining_report.Known_function + { code_id = Code_id.export code_id; + decision = Definition_says_not_to_inline + })); + Not_inlinable + end + else + let inlined_call = Apply_expr.inlined apply in + let decision, res = + match inlined_call with + | Never_inlined -> + Call_site_inlining_decision.Never_inlined_attribute, Not_inlinable + | Always_inlined | Hint_inlined -> + Call_site_inlining_decision.Attribute_always, Inlinable code + | Default_inlined -> + Call_site_inlining_decision.Definition_says_inline, Inlinable code + | Unroll _ -> assert false + in + Inlining_report.record_decision ~dbg + (At_call_site + (Inlining_report.Known_function + { code_id = Code_id.export code_id; decision })); + res + + let make_inlined_body acc ~callee ~params ~args ~my_closure ~my_depth ~body + ~free_names_of_body ~exn_continuation ~return_continuation + ~apply_exn_continuation ~apply_return_continuation ~apply_depth = + let params = List.map Bound_parameter.var params in + let rec_info = + match apply_depth with + | None -> Rec_info_expr.initial + | Some depth -> Rec_info_expr.var depth + in + let bind_params ~params ~args ~body:(acc, body) = + let acc = Acc.with_free_names free_names_of_body acc in + List.fold_left2 + (fun (acc, body) param arg -> + Let_with_acc.create acc + (Bound_pattern.singleton (VB.create param Name_mode.normal)) + (Named.create_simple arg) ~body + |> Expr_with_acc.create_let) + (acc, body) params args + in + let bind_depth ~my_depth ~rec_info ~body:(acc, body) = + Let_with_acc.create acc + (Bound_pattern.singleton (VB.create my_depth Name_mode.normal)) + (Named.create_rec_info rec_info) + ~body + |> Expr_with_acc.create_let + in + let apply_renaming (acc, body) perm = + let acc = + Acc.with_free_names + (Name_occurrences.apply_renaming (Acc.free_names acc) perm) + acc + in + acc, Expr.apply_renaming body perm + in + Inlining_helpers.make_inlined_body ~callee ~params ~args ~my_closure + ~my_depth ~rec_info ~body:(acc, body) ~exn_continuation + ~return_continuation ~apply_exn_continuation ~apply_return_continuation + ~bind_params ~bind_depth ~apply_renaming + + let wrap_inlined_body_for_exn_support acc ~extra_args ~apply_exn_continuation + ~apply_return_continuation ~result_arity ~make_inlined_body = + let apply_cont_create acc ~trap_action cont ~args ~dbg = + let acc, apply_cont = + Apply_cont_with_acc.create acc ~trap_action cont ~args ~dbg + in + Expr_with_acc.create_apply_cont acc apply_cont + in + let let_cont_create acc cont ~handler_params ~handler ~body ~is_exn_handler + = + Let_cont_with_acc.build_non_recursive acc cont ~handler_params ~handler + ~body ~is_exn_handler + in + Inlining_helpers.wrap_inlined_body_for_exn_support acc ~extra_args + ~apply_exn_continuation ~apply_return_continuation ~result_arity + ~make_inlined_body ~apply_cont_create ~let_cont_create + + let inline acc ~apply ~apply_depth ~func_desc:code = + let callee = Apply.callee apply in + let args = Apply.args apply in + let apply_return_continuation = Apply.continuation apply in + let apply_exn_continuation = Apply.exn_continuation apply in + let params_and_body = Code.params_and_body code in + Function_params_and_body.pattern_match params_and_body + ~f:(fun + ~return_continuation + ~exn_continuation + params + ~body + ~my_closure + ~is_my_closure_used:_ + ~my_depth + ~free_names_of_body + -> + let args, remain_args = + let rec split l1 l2 = + match l1, l2 with + | _, [] -> [], l1 + | [], _ -> assert false + | e1 :: l1, _ :: l2 -> + let args, remains = split l1 l2 in + e1 :: args, remains + in + split args params + in + let free_names_of_body = + match free_names_of_body with + | Unknown -> + Misc.fatal_error + "Params_and_body needs free_names_of_body in [Closure_conversion]" + | Known free_names -> free_names + in + let make_inlined_body = + make_inlined_body ~callee ~params ~args ~my_closure ~my_depth ~body + ~free_names_of_body ~exn_continuation ~return_continuation + ~apply_depth + in + let acc = Acc.with_free_names Name_occurrences.empty acc in + let body apply_return_continuation acc = + match Exn_continuation.extra_args apply_exn_continuation with + | [] -> + make_inlined_body acc + ~apply_exn_continuation: + (Exn_continuation.exn_handler apply_exn_continuation) + ~apply_return_continuation + | extra_args -> + wrap_inlined_body_for_exn_support acc ~extra_args + ~apply_exn_continuation ~apply_return_continuation + ~result_arity:(Code.result_arity code) ~make_inlined_body + in + match remain_args with + | [] -> body apply_return_continuation acc + | args -> + let wrapper_cont = Continuation.create () in + let continuation = Apply.Result_continuation.Return wrapper_cont in + let returned_func = Variable.create "func" in + let call_kind = Call_kind.indirect_function_call_unknown_arity () in + let handler acc = + let over_apply = + Apply.create ~callee:(Simple.var returned_func) + ~continuation:apply_return_continuation apply_exn_continuation + ~args ~call_kind (Apply.dbg apply) + ~inlined:(Apply.inlined apply) + ~inlining_state:(Inlining_state.default ~round:0) + ~probe_name:(Apply.probe_name apply) + in + Expr_with_acc.create_apply acc over_apply + in + let body = body continuation in + Let_cont_with_acc.build_non_recursive acc wrapper_cont + ~handler_params: + [Bound_parameter.create returned_func K.With_subkind.any_value] + ~handler ~body ~is_exn_handler:false) +end + let close_c_call acc ~let_bound_var ({ prim_name; prim_arity; @@ -472,17 +668,50 @@ let close_let acc env id user_visible defining_expr | Some (Simple simple) -> let body_env = Env.add_simple_to_substitute env id simple in body acc body_env - | Some _ | None -> ( + | None -> body acc body_env + | Some defining_expr -> (* CR pchambart: Not tail ! *) + let body_env = + match defining_expr with + | Prim (Variadic (Make_block (_, Immutable), fields), _) -> + let approxs = + List.map (Env.find_value_approximation body_env) fields + |> Array.of_list + in + Env.add_block_approximation body_env (Name.var var) approxs + | Prim (Binary (Block_load _, block, field), _) -> begin + match Env.find_value_approximation body_env block with + | Value_unknown -> body_env + | Closure_approximation _ -> + Misc.fatal_errorf + "Closure approximation found when block approximation was \ + expected in [Closure_conversion]: %a" + Named.print defining_expr + | Block_approximation approx -> + let approx : Env.value_approximation = + Simple.pattern_match field + ~const:(fun const -> + match Reg_width_things.Const.descr const with + | Tagged_immediate i -> + let i = Targetint_31_63.(Imm.to_int (to_targetint i)) in + if i >= Array.length approx + then + Misc.fatal_errorf + "Trying to access the %dth field of a block \ + approximation of length %d." + i (Array.length approx); + approx.(i) + | _ -> Env.Value_unknown) + ~name:(fun _ ~coercion:_ -> Env.Value_unknown) + in + Env.add_value_approximation body_env (Name.var var) approx + end + | _ -> body_env + in let acc, body = body acc body_env in - match defining_expr with - | None -> acc, body - | Some defining_expr -> - let var = VB.create var Name_mode.normal in - Let_with_acc.create acc - (Bound_pattern.singleton var) - defining_expr ~body - |> Expr_with_acc.create_let) + let var = VB.create var Name_mode.normal in + Let_with_acc.create acc (Bound_pattern.singleton var) defining_expr ~body + |> Expr_with_acc.create_let in close_named acc env ~let_bound_var:var defining_expr cont @@ -512,7 +741,18 @@ let close_let_cont acc env ~name ~is_exn_handler ~params Bound_parameter.create param (LC.value_kind kind)) params params_with_kinds in - let handler acc = handler acc handler_env in + let handler acc = + let handler_env = + match Acc.continuation_known_arguments ~cont:name acc with + | None -> handler_env + | Some args -> + List.fold_left2 + (fun env arg_approx param -> + Env.add_value_approximation env (Name.var param) arg_approx) + handler_env args params + in + handler acc handler_env + in let body acc = body acc env in match recursive with | Nonrecursive -> @@ -544,27 +784,38 @@ let close_apply acc env let acc, obj = find_simple acc env obj in acc, Call_kind.method_call (LC.method_kind kind) ~obj in - let acc, exn_continuation = close_exn_continuation acc env exn_continuation in + let acc, apply_exn_continuation = + close_exn_continuation acc env exn_continuation + in let callee = find_simple_from_id env func in let acc, args = find_simples acc env args in + let inlined_call = LC.inlined_attribute inlined in let probe_name = match probe with None -> None | Some { name } -> Some name in let apply = - Apply.create ~callee ~continuation:(Return continuation) exn_continuation - ~args ~call_kind + Apply.create ~callee ~continuation:(Return continuation) + apply_exn_continuation ~args ~call_kind (Debuginfo.from_location loc) - ~inlined:(LC.inlined_attribute inlined) + ~inlined:inlined_call ~inlining_state:(Inlining_state.default ~round:0) ~probe_name in - Expr_with_acc.create_apply acc apply + if Flambda_features.classic_mode () + then + match Inlining.inlinable env apply with + | Not_inlinable -> Expr_with_acc.create_apply acc apply + | Inlinable func_desc -> + Inlining.inline acc ~apply ~apply_depth:(Env.current_depth env) ~func_desc + else Expr_with_acc.create_apply acc apply let close_apply_cont acc env cont trap_action args : Acc.t * Expr_with_acc.t = let acc, args = find_simples acc env args in let trap_action = close_trap_action_opt trap_action in + let args_approx = List.map (Env.find_value_approximation env) args in let acc, apply_cont = - Apply_cont_with_acc.create acc ?trap_action cont ~args ~dbg:Debuginfo.none + Apply_cont_with_acc.create acc ?trap_action ~args_approx cont ~args + ~dbg:Debuginfo.none in Expr_with_acc.create_apply_cont acc apply_cont @@ -739,8 +990,18 @@ let close_one_function acc ~external_env ~by_closure_id decl in let closure_env_without_parameters = let empty_env = Env.clear_local_bindings external_env in - let env = Env.add_var_map empty_env var_within_closures_for_idents in - Env.add_simple_to_substitute_map env simples_for_project_closure + let env_with_vars = + Ident.Map.fold + (fun id var env -> + Simple.pattern_match + (find_simple_from_id external_env id) + ~const:(fun _ -> assert false) + ~name:(fun name ~coercion:_ -> + Env.add_approximation_alias (Env.add_var env id var) name + (Name.var var))) + var_within_closures_for_idents empty_env + in + Env.add_simple_to_substitute_map env_with_vars simples_for_project_closure in let closure_env = List.fold_right @@ -749,6 +1010,7 @@ let close_one_function acc ~external_env ~by_closure_id decl env) params closure_env_without_parameters in + let closure_env = Env.with_depth closure_env my_depth in (* CR-someday pchambart: eta-expansion wrappers for primitives are not marked as stubs but certainly should be. *) let stub = Function_decl.stub decl in @@ -863,6 +1125,28 @@ let close_one_function acc ~external_env ~by_closure_id decl let is_tupled = match Function_decl.kind decl with Curried -> false | Tupled -> true in + let code_size = Cost_metrics.size cost_metrics in + let inline_threshold = Inlining.threshold () in + let inlining_decision = + if Flambda_features.classic_mode () + then + match inline with + | Never_inline -> + Function_decl_inlining_decision_type.Never_inline_attribute + | Always_inline | Available_inline -> + Function_decl_inlining_decision_type.Attribute_inline + | _ -> + if Code_size.to_int code_size <= inline_threshold + then + Function_decl_inlining_decision_type.Small_function + { size = code_size; + small_function_size = Code_size.of_int inline_threshold + } + else + Function_decl_inlining_decision_type.Function_body_too_large + (Code_size.of_int inline_threshold) + else Function_decl_inlining_decision_type.Not_yet_decided + in let code = Code.create code_id ~params_and_body ~free_names_of_params_and_body:(Acc.free_names acc) ~params_arity @@ -873,11 +1157,26 @@ let close_one_function acc ~external_env ~by_closure_id decl ~dbg ~is_tupled ~is_my_closure_used: (Function_params_and_body.is_my_closure_used params_and_body) - ~inlining_decision:Not_yet_decided + ~inlining_decision + in + let approx = + if Flambda_features.classic_mode () + then begin + Inlining_report.record_decision ~dbg + (At_function_declaration + { pass = After_closure_conversion; + code_id = Code_id.export code_id; + decision = inlining_decision + }); + if Function_decl_inlining_decision_type.must_be_inlined inlining_decision + then Some code + else None + end + else None in let acc = Acc.add_code ~code_id ~code acc in let acc = Acc.with_seen_a_function acc true in - acc, Closure_id.Map.add my_closure_id code_id by_closure_id + acc, Closure_id.Map.add my_closure_id (code_id, approx) by_closure_id let close_functions acc external_env function_declarations = let compilation_unit = Compilation_unit.get_current_exn () in @@ -919,7 +1218,7 @@ let close_functions acc external_env function_declarations = Ident.Map.add id closure_id map) Ident.Map.empty func_decl_list in - let acc, funs = + let acc, approximations = List.fold_left (fun (acc, by_closure_id) function_decl -> let _, _, acc, expr = @@ -935,7 +1234,15 @@ let close_functions acc external_env function_declarations = let acc = Acc.with_free_names Name_occurrences.empty acc in (* CR lmaurer: funs has arbitrary order (ultimately coming from function_declarations) *) - let funs = Closure_id.Lmap.of_list (Closure_id.Map.bindings funs) in + let funs, approximations = + let funs, approxs = + Closure_id.Map.fold + (fun cid (code_id, desc) (funs, approxs) -> + (cid, code_id) :: funs, (code_id, desc) :: approxs) + approximations ([], []) + in + Closure_id.Lmap.of_list (List.rev funs), List.rev approxs + in let function_decls = Function_declarations.create funs in let closure_elements = Ident.Map.fold @@ -947,7 +1254,7 @@ let close_functions acc external_env function_declarations = Var_within_closure.Map.add var_within_closure external_simple map) var_within_closures_from_idents Var_within_closure.Map.empty in - acc, Set_of_closures.create function_decls ~closure_elements + acc, Set_of_closures.create function_decls ~closure_elements, approximations let close_let_rec acc env ~function_declarations ~(body : Acc.t -> Env.t -> Acc.t * Expr_with_acc.t) = @@ -971,7 +1278,7 @@ let close_let_rec acc env ~function_declarations Closure_id.Map.add closure_id closure_var closure_vars) Closure_id.Map.empty function_declarations in - let acc, set_of_closures = + let acc, set_of_closures, approximations = close_functions acc env (Function_decls.create function_declarations) in (* CR mshinwell: We should maybe have something more elegant here *) @@ -998,6 +1305,12 @@ let close_let_rec acc env ~function_declarations (Set_of_closures.function_decls set_of_closures) |> Closure_id.Lmap.bindings) in + let env = + List.fold_left2 + (fun env var approx -> + Env.add_closure_approximation env (Name.var (VB.var var)) approx) + env closure_vars approximations + in let acc, body = body acc env in let named = Named.create_set_of_closures set_of_closures in Let_with_acc.create acc diff --git a/middle_end/flambda2/from_lambda/closure_conversion_aux.ml b/middle_end/flambda2/from_lambda/closure_conversion_aux.ml index 9c6743d4eee..d71425dc2db 100644 --- a/middle_end/flambda2/from_lambda/closure_conversion_aux.ml +++ b/middle_end/flambda2/from_lambda/closure_conversion_aux.ml @@ -89,12 +89,19 @@ module IR = struct end module Env = struct + type value_approximation = + | Value_unknown + | Closure_approximation of Code_id.t * Code.t option + | Block_approximation of value_approximation array + type t = { variables : Variable.t Ident.Map.t; globals : Symbol.t Numeric_types.Int.Map.t; simples_to_substitute : Simple.t Ident.Map.t; current_unit_id : Ident.t; + current_depth : Variable.t option; symbol_for_global : Ident.t -> Symbol.t; + value_approximations : value_approximation Name.Map.t; big_endian : bool } @@ -104,12 +111,16 @@ module Env = struct let big_endian t = t.big_endian + let current_depth t = t.current_depth + let create ~symbol_for_global ~big_endian = let compilation_unit = Compilation_unit.get_current_exn () in { variables = Ident.Map.empty; globals = Numeric_types.Int.Map.empty; simples_to_substitute = Ident.Map.empty; current_unit_id = Compilation_unit.get_persistent_ident compilation_unit; + current_depth = None; + value_approximations = Name.Map.empty; symbol_for_global; big_endian } @@ -120,6 +131,8 @@ module Env = struct simples_to_substitute; current_unit_id; symbol_for_global; + current_depth; + value_approximations; big_endian } = let simples_to_substitute = @@ -131,10 +144,14 @@ module Env = struct globals; simples_to_substitute; current_unit_id; + current_depth; + value_approximations; symbol_for_global; big_endian } + let with_depth t depth_var = { t with current_depth = Some depth_var } + let add_var t id var = { t with variables = Ident.Map.add id var t.variables } let add_vars t ids vars = List.fold_left2 add_var t ids vars @@ -208,14 +225,47 @@ module Env = struct let find_simple_to_substitute_exn t id = Ident.Map.find id t.simples_to_substitute + + let add_value_approximation t name approx = + if approx = Value_unknown + then t + else + { t with + value_approximations = Name.Map.add name approx t.value_approximations + } + + let add_closure_approximation t name (code_id, approx) = + add_value_approximation t name (Closure_approximation (code_id, approx)) + + let add_block_approximation t name approxs = + if Array.for_all (( = ) Value_unknown) approxs + then t + else add_value_approximation t name (Block_approximation approxs) + + let find_value_approximation t simple = + Simple.pattern_match simple + ~const:(fun _ -> Value_unknown) + ~name:(fun name ~coercion:_ -> + try Name.Map.find name t.value_approximations + with Not_found -> Value_unknown) + + let add_approximation_alias t name alias = + match find_value_approximation t (Simple.name name) with + | Value_unknown -> t + | approx -> add_value_approximation t alias approx end module Acc = struct + type continuation_application = + | Trackable_arguments of Env.value_approximation list + | Untrackable + type t = { declared_symbols : (Symbol.t * Static_const.t) list; shareable_constants : Symbol.t Static_const.Map.t; code : Code.t Code_id.Map.t; free_names : Name_occurrences.t; + continuation_applications : continuation_application Continuation.Map.t; cost_metrics : Cost_metrics.t; seen_a_function : bool; symbol_for_global : Ident.t -> Symbol.t @@ -237,6 +287,7 @@ module Acc = struct shareable_constants = Static_const.Map.empty; code = Code_id.Map.empty; free_names = Name_occurrences.empty; + continuation_applications = Continuation.Map.empty; cost_metrics = Cost_metrics.zero; seen_a_function = false; symbol_for_global @@ -287,14 +338,36 @@ module Acc = struct let remove_var_from_free_names var t = { t with free_names = Name_occurrences.remove_var t.free_names var } + let add_continuation_application ~cont args_approx t = + let continuation_application = + match args_approx with + | None -> Untrackable + | Some args -> + if Continuation.Map.mem cont t.continuation_applications + then Untrackable + else Trackable_arguments args + in + { t with + continuation_applications = + Continuation.Map.add cont continuation_application + t.continuation_applications + } + let remove_continuation_from_free_names cont t = { t with - free_names = Name_occurrences.remove_continuation t.free_names cont + free_names = Name_occurrences.remove_continuation t.free_names cont; + continuation_applications = + Continuation.Map.remove cont t.continuation_applications } let remove_code_id_from_free_names code_id t = remove_code_id_or_symbol_from_free_names (Code_id code_id) t + let continuation_known_arguments ~cont t = + match Continuation.Map.find cont t.continuation_applications with + | (exception Not_found) | Untrackable -> None + | Trackable_arguments args -> Some args + let with_free_names free_names t = { t with free_names } let eval_branch_free_names t ~f = @@ -480,12 +553,14 @@ module Expr_with_acc = struct end module Apply_cont_with_acc = struct - let create acc ?trap_action cont ~args ~dbg = + let create acc ?trap_action ?args_approx cont ~args ~dbg = let apply_cont = Apply_cont.create ?trap_action cont ~args ~dbg in + let acc = Acc.add_continuation_application ~cont args_approx acc in let acc = Acc.add_free_names (Apply_cont.free_names apply_cont) acc in acc, apply_cont - let goto acc cont = create acc cont ~args:[] ~dbg:Debuginfo.none + let goto acc cont = + create acc cont ~args:[] ?args_approx:None ~dbg:Debuginfo.none end module Let_with_acc = struct @@ -602,15 +677,18 @@ module Let_cont_with_acc = struct let build_non_recursive acc cont ~handler_params ~handler ~body ~is_exn_handler = + (* We need to evaluate the body before the handler to pass along information + on the argument for inlining *) + let free_names_of_body, acc, body = + Acc.eval_branch_free_names acc ~f:body + in let cost_metrics_of_handler, handler_free_names, acc, handler = Acc.measure_cost_metrics acc ~f:(fun acc -> let acc, handler = handler acc in Continuation_handler_with_acc.create acc handler_params ~handler ~is_exn_handler) in - let free_names_of_body, acc, body = - Acc.eval_branch_free_names acc ~f:body - in + (* [create_non_recursive] assumes [acc] contains free names of the body *) let acc, expr = create_non_recursive (Acc.with_free_names free_names_of_body acc) diff --git a/middle_end/flambda2/from_lambda/closure_conversion_aux.mli b/middle_end/flambda2/from_lambda/closure_conversion_aux.mli index 196c2452f61..772bf733eb5 100644 --- a/middle_end/flambda2/from_lambda/closure_conversion_aux.mli +++ b/middle_end/flambda2/from_lambda/closure_conversion_aux.mli @@ -79,6 +79,11 @@ end values during closure conversion, and similarly for static exception identifiers. *) module Env : sig + type value_approximation = + | Value_unknown + | Closure_approximation of Code_id.t * Code.t option + | Block_approximation of value_approximation array + type t val create : symbol_for_global:(Ident.t -> Symbol.t) -> big_endian:bool -> t @@ -116,6 +121,20 @@ module Env : sig val find_simple_to_substitute_exn : t -> Ident.t -> Simple.t + val add_value_approximation : t -> Name.t -> value_approximation -> t + + val add_closure_approximation : t -> Name.t -> Code_id.t * Code.t option -> t + + val add_block_approximation : t -> Name.t -> value_approximation array -> t + + val add_approximation_alias : t -> Name.t -> Name.t -> t + + val find_value_approximation : t -> Simple.t -> value_approximation + + val current_depth : t -> Variable.t option + + val with_depth : t -> Variable.t -> t + val current_unit_id : t -> Ident.t val symbol_for_global : t -> Ident.t -> Symbol.t @@ -154,6 +173,9 @@ module Acc : sig val remove_continuation_from_free_names : Continuation.t -> t -> t + val continuation_known_arguments : + cont:Continuation.t -> t -> Env.value_approximation list option + val with_free_names : Name_occurrences.t -> t -> t (* This is intended to evaluate a distinct free_names from the one in acc, one @@ -264,6 +286,7 @@ module Apply_cont_with_acc : sig val create : Acc.t -> ?trap_action:Trap_action.t -> + ?args_approx:Env.value_approximation list -> Continuation.t -> args:Simple.t list -> dbg:Debuginfo.t -> diff --git a/middle_end/flambda2/from_lambda/dune b/middle_end/flambda2/from_lambda/dune index 5b7ce37fce3..c73017c3688 100644 --- a/middle_end/flambda2/from_lambda/dune +++ b/middle_end/flambda2/from_lambda/dune @@ -28,7 +28,9 @@ -open Flambda2_terms -open - Flambda2_ui)) + Flambda2_ui + -open + Flambda2_simplify)) (ocamlopt_flags (:standard -O3)) (libraries @@ -44,4 +46,5 @@ flambda2_numbers flambda2_term_basics flambda2_terms - flambda2_ui)) + flambda2_ui + flambda2_simplify)) diff --git a/middle_end/flambda2/simplify/inlining/call_site_inlining_decision.mli b/middle_end/flambda2/simplify/inlining/call_site_inlining_decision.mli index 5ed6b4a2076..a1ca1bd1984 100644 --- a/middle_end/flambda2/simplify/inlining/call_site_inlining_decision.mli +++ b/middle_end/flambda2/simplify/inlining/call_site_inlining_decision.mli @@ -21,7 +21,7 @@ open! Flambda.Import (* CR-someday mshinwell: Maybe have two types, one giving the reasons why something can be inlined, and one giving the reasons why something cannot be inlined. *) -type t = private +type t = | Missing_code | Definition_says_not_to_inline | Environment_says_never_inline diff --git a/middle_end/flambda2/simplify/inlining/inlining_report.ml b/middle_end/flambda2/simplify/inlining/inlining_report.ml index 7f958cccc1e..575e713df53 100644 --- a/middle_end/flambda2/simplify/inlining/inlining_report.ml +++ b/middle_end/flambda2/simplify/inlining/inlining_report.ml @@ -25,6 +25,7 @@ type at_call_site = | Unknown_function type fundecl_pass = + | After_closure_conversion | Before_simplify of { dbg_including_inlining_stack : Debuginfo.t } | After_simplify @@ -84,6 +85,16 @@ let [@ocamlformat "disable"] rec print ~depth fmt = function if depth <> 0 then Misc.fatal_errorf "incoherent depth at end of inlining report" + (* After closure conversion of a function *) + | { dbg; decision = At_function_declaration { + pass = After_closure_conversion; code_id; decision; } } :: r -> + Format.fprintf fmt "%a Definition of %s{%a}@\n" + stars depth Code_id.(name (import code_id)) print_debuginfo dbg; + Format.fprintf fmt "%a @[After closure conversion:@ @ %a@]@\n@\n" + stars (depth + 1) + Function_decl_inlining_decision_type.report decision; + print ~depth fmt r + (* Entering a function declaration (possibly nested) *) | { dbg; decision = At_function_declaration { pass = Before_simplify _; code_id; decision; } } :: r -> diff --git a/middle_end/flambda2/simplify/inlining/inlining_report.mli b/middle_end/flambda2/simplify/inlining/inlining_report.mli index 7cecffa4435..85ffa00c2fb 100644 --- a/middle_end/flambda2/simplify/inlining/inlining_report.mli +++ b/middle_end/flambda2/simplify/inlining/inlining_report.mli @@ -21,10 +21,11 @@ type at_call_site = } (** Function call where the function's type is known *) | Unknown_function (** Function call where the function's type is unknown. *) -(** There are two decisions made for each function declaration: one before - simplifying the body, and one after (this is useful for e.g. recursive - functions). *) +(** There are three decisions made for each function declaration: on after + conversion in CPS and closure, one before simplifying the body, and one + after (this is useful for e.g. recursive functions). *) type fundecl_pass = + | After_closure_conversion | Before_simplify of { dbg_including_inlining_stack : Debuginfo.t } | After_simplify (**) @@ -38,6 +39,7 @@ type at_function_declaration = (** This defines the various kinds of decisions related to inlining that will be reported, together with some additional information to better identify to what the decision refers to. *) + type decision = | At_call_site of at_call_site | At_function_declaration of at_function_declaration diff --git a/middle_end/flambda2/simplify/inlining/inlining_transforms.ml b/middle_end/flambda2/simplify/inlining/inlining_transforms.ml index 534ae1867d4..ff0f1c90730 100644 --- a/middle_end/flambda2/simplify/inlining/inlining_transforms.ml +++ b/middle_end/flambda2/simplify/inlining/inlining_transforms.ml @@ -25,15 +25,6 @@ module VB = Bound_var let make_inlined_body ~callee ~unroll_to ~params ~args ~my_closure ~my_depth ~rec_info ~body ~exn_continuation ~return_continuation ~apply_exn_continuation ~apply_return_continuation = - let perm = Renaming.empty in - let perm = - match (apply_return_continuation : Apply.Result_continuation.t) with - | Return k -> Renaming.add_continuation perm return_continuation k - | Never_returns -> perm - in - let perm = - Renaming.add_continuation perm exn_continuation apply_exn_continuation - in let callee, rec_info = match unroll_to with | None -> callee, rec_info @@ -47,135 +38,39 @@ let make_inlined_body ~callee ~unroll_to ~params ~args ~my_closure ~my_depth in callee, unrolled_rec_info in - let body = - Let.create - (Bound_pattern.singleton (VB.create my_closure Name_mode.normal)) - (Named.create_simple callee) - ~body - (* Here and below, we don't need to give any name occurrence information - (thank goodness!) since the entirety of the expression we're building - will be re-simplified. *) - ~free_names_of_body:Unknown - |> Expr.create_let + let my_closure = + Bound_parameter.create my_closure Flambda_kind.With_subkind.any_value in - let body = + let bind_params = Expr.bind_parameters_to_args_no_simplification in + let bind_depth ~my_depth ~rec_info ~body = let bound = Bound_pattern.singleton (VB.create my_depth Name_mode.normal) in Let.create bound (Named.create_rec_info rec_info) ~body ~free_names_of_body:Unknown |> Expr.create_let in - Expr.apply_renaming - (Expr.bind_parameters_to_args_no_simplification ~params ~args ~body) - perm + let apply_renaming = Expr.apply_renaming in + Inlining_helpers.make_inlined_body ~callee ~params ~args ~my_closure ~my_depth + ~rec_info ~body ~exn_continuation ~return_continuation + ~apply_exn_continuation ~apply_return_continuation ~bind_params ~bind_depth + ~apply_renaming let wrap_inlined_body_for_exn_support ~extra_args ~apply_exn_continuation ~apply_return_continuation ~result_arity ~make_inlined_body = - (* We need to add a wrapper for the exception handler, so that exceptions - coming from the inlined body go through the wrapper and are re-raised with - the correct extra arguments. - - This means we also need to add a push trap before the inlined body, and a - pop trap after. - - The push trap is simply a matter of jumping to the body, while the pop trap - needs to replace the body's return continuation with a wrapper that pops - then jumps back. *) - (* - * As a result, the application [Apply_expr f (args) «k_exn»] - * is replaced (before the actual inlining) by: - * - * [let_cont_exn k1 (exn: val) = - * Apply_cont k_exn exn extra_args - * in - * let_cont k_pop (args) = Apply_cont k args in - * let_cont k_push () = Apply_expr f (args) «k1» in - * Apply_cont k_push ()] - *) - (* This feels quite heavy, but is necessary because we can rewrite neither the - definition and other uses of k_exn nor the uses of the exception - continuation in the body of f, so we need two distinct exception - continuations; and of course the new exception continuation needs to be - correctly pushed and popped. - - The most annoying part of this is that it introduces trywith blocks that - were not part of the initial program, will not be removed, and might be - useless (if the function never raises). - - Maybe a better solution would be to propagate through dacc a lazy - rewriting, that would add the correct extra args to all uses of the - exception continuation in the body. *) - let wrapper = Continuation.create () in - let body_with_pop = - match (apply_return_continuation : Apply.Result_continuation.t) with - | Never_returns -> - make_inlined_body ~apply_exn_continuation:wrapper - ~apply_return_continuation - | Return apply_return_continuation -> - let pop_wrapper_cont = Continuation.create () in - let pop_wrapper_handler = - let kinded_params = - List.map (fun k -> Variable.create "wrapper_return", k) result_arity - in - let trap_action = - Trap_action.Pop { exn_handler = wrapper; raise_kind = None } - in - let args = List.map (fun (v, _) -> Simple.var v) kinded_params in - let handler = - Apply_cont.create ~trap_action apply_return_continuation ~args - ~dbg:Debuginfo.none - |> Expr.create_apply_cont - in - Continuation_handler.create - (Bound_parameter.List.create kinded_params) - ~handler ~free_names_of_handler:Unknown ~is_exn_handler:false - in - let new_apply_return_continuation = - Apply.Result_continuation.Return pop_wrapper_cont - in - let body = - make_inlined_body ~apply_exn_continuation:wrapper - ~apply_return_continuation:new_apply_return_continuation - in - Let_cont.create_non_recursive pop_wrapper_cont pop_wrapper_handler ~body - ~free_names_of_body:Unknown + let apply_cont_create () ~trap_action cont ~args ~dbg = + Apply_cont.create ~trap_action cont ~args ~dbg |> Expr.create_apply_cont in - let wrapper_handler = - let param = Variable.create "exn" in - let kinded_params = [BP.create param K.With_subkind.any_value] in - let exn_handler = Exn_continuation.exn_handler apply_exn_continuation in - let trap_action = Trap_action.Pop { exn_handler; raise_kind = None } in + let let_cont_create () cont ~handler_params ~handler ~body ~is_exn_handler = let handler = - (* Backtrace building functions expect compiler-generated raises not to - have any debug info *) - Apply_cont.create ~trap_action - (Exn_continuation.exn_handler apply_exn_continuation) - ~args:(Simple.var param :: List.map fst extra_args) - ~dbg:Debuginfo.none - |> Expr.create_apply_cont - in - Continuation_handler.create kinded_params ~handler - ~free_names_of_handler:Unknown ~is_exn_handler:true - in - let body_with_push = - (* Wrap the body between push and pop of the wrapper handler *) - let push_wrapper_cont = Continuation.create () in - let handler = body_with_pop in - let push_wrapper_handler = - Continuation_handler.create [] ~handler ~free_names_of_handler:Unknown - ~is_exn_handler:false - in - let trap_action = Trap_action.Push { exn_handler = wrapper } in - let body = - Apply_cont.create ~trap_action push_wrapper_cont ~args:[] - ~dbg:Debuginfo.none - |> Expr.create_apply_cont + Continuation_handler.create handler_params ~handler:(handler ()) + ~free_names_of_handler:Unknown ~is_exn_handler in - Let_cont.create_non_recursive push_wrapper_cont push_wrapper_handler ~body + Let_cont.create_non_recursive cont handler ~body:(body ()) ~free_names_of_body:Unknown in - Let_cont.create_non_recursive wrapper wrapper_handler ~body:body_with_push - ~free_names_of_body:Unknown + Inlining_helpers.wrap_inlined_body_for_exn_support () ~extra_args + ~apply_exn_continuation ~apply_return_continuation ~result_arity + ~make_inlined_body ~apply_cont_create ~let_cont_create let inline dacc ~apply ~unroll_to function_decl = let callee = Apply.callee apply in @@ -210,14 +105,14 @@ let inline dacc ~apply ~unroll_to function_decl = ~my_depth ~free_names_of_body:_ -> - let make_inlined_body = + let make_inlined_body () = make_inlined_body ~callee ~unroll_to ~params ~args ~my_closure ~my_depth ~rec_info ~body ~exn_continuation ~return_continuation in let expr = match Exn_continuation.extra_args apply_exn_continuation with | [] -> - make_inlined_body + make_inlined_body () ~apply_exn_continuation: (Exn_continuation.exn_handler apply_exn_continuation) ~apply_return_continuation diff --git a/middle_end/flambda2/terms/inlining_helpers.ml b/middle_end/flambda2/terms/inlining_helpers.ml new file mode 100644 index 00000000000..6cc51c866c5 --- /dev/null +++ b/middle_end/flambda2/terms/inlining_helpers.ml @@ -0,0 +1,131 @@ +(**************************************************************************) +(* *) +(* OCaml *) +(* *) +(* Pierre Chambart, OCamlPro *) +(* Mark Shinwell and Leo White, Jane Street Europe *) +(* *) +(* Copyright 2013--2020 OCamlPro SAS *) +(* Copyright 2014--2020 Jane Street Group LLC *) +(* *) +(* All rights reserved. This file is distributed under the terms of *) +(* the GNU Lesser General Public License version 2.1, with the *) +(* special exception on linking described in the file LICENSE. *) +(* *) +(**************************************************************************) + +[@@@ocaml.warning "+a-4-30-40-41-42"] + +let make_inlined_body ~callee ~params ~args ~my_closure ~my_depth ~rec_info + ~body ~exn_continuation ~return_continuation ~apply_exn_continuation + ~apply_return_continuation ~bind_params ~bind_depth ~apply_renaming = + let perm = Renaming.empty in + let perm = + match (apply_return_continuation : Flambda.Apply.Result_continuation.t) with + | Return k -> Renaming.add_continuation perm return_continuation k + | Never_returns -> perm + in + let perm = + Renaming.add_continuation perm exn_continuation apply_exn_continuation + in + let body = + bind_params ~params:(my_closure :: params) ~args:(callee :: args) ~body + in + let body = bind_depth ~my_depth ~rec_info ~body in + apply_renaming body perm + +let wrap_inlined_body_for_exn_support acc ~extra_args ~apply_exn_continuation + ~apply_return_continuation ~result_arity ~make_inlined_body + ~apply_cont_create ~let_cont_create = + (* We need to add a wrapper for the exception handler, so that exceptions + coming from the inlined body go through the wrapper and are re-raised with + the correct extra arguments. + + This means we also need to add a push trap before the inlined body, and a + pop trap after. + + The push trap is simply a matter of jumping to the body, while the pop trap + needs to replace the body's return continuation with a wrapper that pops + then jumps back. *) + (* + * As a result, the application [Apply_expr f (args) «k_exn»] + * is replaced (before the actual inlining) by: + * + * [let_cont_exn k1 (exn: val) = + * Apply_cont k_exn exn extra_args + * in + * let_cont k_pop (args) = Apply_cont k args in + * let_cont k_push () = Apply_expr f (args) «k1» in + * Apply_cont k_push ()] + *) + (* This feels quite heavy, but is necessary because we can rewrite neither the + definition and other uses of k_exn nor the uses of the exception + continuation in the body of f, so we need two distinct exception + continuations; and of course the new exception continuation needs to be + correctly pushed and popped. + + The most annoying part of this is that it introduces trywith blocks that + were not part of the initial program, will not be removed, and might be + useless (if the function never raises). + + Maybe a better solution would be to propagate through dacc a lazy + rewriting, that would add the correct extra args to all uses of the + exception continuation in the body. *) + let wrapper = Continuation.create () in + let body_with_pop acc = + match (apply_return_continuation : Flambda.Apply.Result_continuation.t) with + | Never_returns -> + make_inlined_body acc ~apply_exn_continuation:wrapper + ~apply_return_continuation + | Return apply_return_continuation -> + let pop_wrapper_cont = Continuation.create () in + let new_apply_return_continuation = + Flambda.Apply.Result_continuation.Return pop_wrapper_cont + in + let body acc = + make_inlined_body acc ~apply_exn_continuation:wrapper + ~apply_return_continuation:new_apply_return_continuation + in + let kinded_params = + List.map (fun k -> Variable.create "wrapper_return", k) result_arity + in + let trap_action = + Trap_action.Pop { exn_handler = wrapper; raise_kind = None } + in + let args = List.map (fun (v, _) -> Simple.var v) kinded_params in + let handler acc = + apply_cont_create acc ~trap_action apply_return_continuation ~args + ~dbg:Debuginfo.none + in + let_cont_create acc pop_wrapper_cont + ~handler_params:(Bound_parameter.List.create kinded_params) + ~handler ~body ~is_exn_handler:false + in + let param = Variable.create "exn" in + let wrapper_handler_params = + [Bound_parameter.create param Flambda_kind.With_subkind.any_value] + in + let exn_handler = Exn_continuation.exn_handler apply_exn_continuation in + let trap_action = Trap_action.Pop { exn_handler; raise_kind = None } in + let wrapper_handler acc = + (* Backtrace building functions expect compiler-generated raises not to have + any debug info *) + apply_cont_create acc ~trap_action + (Exn_continuation.exn_handler apply_exn_continuation) + ~args:(Simple.var param :: List.map fst extra_args) + ~dbg:Debuginfo.none + in + let body_with_push acc = + (* Wrap the body between push and pop of the wrapper handler *) + let push_wrapper_cont = Continuation.create () in + let push_wrapper_handler = body_with_pop in + let trap_action = Trap_action.Push { exn_handler = wrapper } in + let body acc = + apply_cont_create acc ~trap_action push_wrapper_cont ~args:[] + ~dbg:Debuginfo.none + in + let_cont_create acc push_wrapper_cont ~handler_params:[] + ~handler:push_wrapper_handler ~body ~is_exn_handler:false + in + let_cont_create acc wrapper ~handler_params:wrapper_handler_params + ~handler:wrapper_handler ~body:body_with_push ~is_exn_handler:true diff --git a/middle_end/flambda2/terms/inlining_helpers.mli b/middle_end/flambda2/terms/inlining_helpers.mli new file mode 100644 index 00000000000..e67d6bd43a6 --- /dev/null +++ b/middle_end/flambda2/terms/inlining_helpers.mli @@ -0,0 +1,70 @@ +(**************************************************************************) +(* *) +(* OCaml *) +(* *) +(* Pierre Chambart, OCamlPro *) +(* Mark Shinwell and Leo White, Jane Street Europe *) +(* *) +(* Copyright 2013--2019 OCamlPro SAS *) +(* Copyright 2014--2019 Jane Street Group LLC *) +(* *) +(* All rights reserved. This file is distributed under the terms of *) +(* the GNU Lesser General Public License version 2.1, with the *) +(* special exception on linking described in the file LICENSE. *) +(* *) +(**************************************************************************) + +[@@@ocaml.warning "+a-4-30-40-41-42"] + +val make_inlined_body : + callee:Simple.t -> + params:'param list -> + args:Simple.List.t -> + my_closure:'param -> + my_depth:Variable.t -> + rec_info:Rec_info_expr.t -> + body:'expr_with_acc -> + exn_continuation:Continuation.t -> + return_continuation:Continuation.t -> + apply_exn_continuation:Continuation.t -> + apply_return_continuation:Flambda.Apply.Result_continuation.t -> + bind_params: + (params:'param list -> + args:Simple.List.t -> + body:'expr_with_acc -> + 'expr_with_acc) -> + bind_depth: + (my_depth:Variable.t -> + rec_info:Rec_info_expr.t -> + body:'expr_with_acc -> + 'expr_with_acc) -> + apply_renaming:('expr_with_acc -> Renaming.t -> 'expr_with_acc) -> + 'expr_with_acc + +val wrap_inlined_body_for_exn_support : + 'acc -> + extra_args:(Simple.t * Flambda_kind.With_subkind.t) list -> + apply_exn_continuation:Exn_continuation.t -> + apply_return_continuation:Flambda.Apply.Result_continuation.t -> + result_arity:Flambda_arity.With_subkinds.t -> + make_inlined_body: + ('acc -> + apply_exn_continuation:Continuation.t -> + apply_return_continuation:Flambda.Apply.Result_continuation.t -> + 'expr_with_acc) -> + apply_cont_create: + ('acc -> + trap_action:Trap_action.t -> + Continuation.t -> + args:Simple.List.t -> + dbg:Debuginfo.t -> + 'expr_with_acc) -> + let_cont_create: + ('acc -> + Continuation.t -> + handler_params:Bound_parameter.List.t -> + handler:('acc -> 'expr_with_acc) -> + body:('acc -> 'expr_with_acc) -> + is_exn_handler:bool -> + 'expr_with_acc) -> + 'expr_with_acc