diff --git a/engine/Cargo.lock b/engine/Cargo.lock index 944c80717..ca8a1933e 100644 --- a/engine/Cargo.lock +++ b/engine/Cargo.lock @@ -77,6 +77,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.14" @@ -960,6 +966,7 @@ dependencies = [ "clap", "colored", "console_log", + "criterion", "dashmap", "derive_more", "dissimilar", @@ -1276,6 +1283,12 @@ dependencies = [ "either", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.1.2" @@ -1328,6 +1341,33 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -1497,6 +1537,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -1531,6 +1607,12 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -2169,6 +2251,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2828,6 +2920,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is-wsl" version = "0.4.0" @@ -3405,6 +3508,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + [[package]] name = "open" version = "5.3.0" @@ -3657,6 +3766,34 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polling" version = "2.8.0" @@ -4947,6 +5084,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.8.0" diff --git a/engine/Cargo.toml b/engine/Cargo.toml index e454bcbbd..aad679c5a 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -111,4 +111,4 @@ lto = false inherits = "dev" [profile.release] -lto = true +lto = true \ No newline at end of file diff --git a/engine/baml-lib/baml-core/src/ir/ir_helpers/mod.rs b/engine/baml-lib/baml-core/src/ir/ir_helpers/mod.rs index 7b285773a..1f9f33231 100644 --- a/engine/baml-lib/baml-core/src/ir/ir_helpers/mod.rs +++ b/engine/baml-lib/baml-core/src/ir/ir_helpers/mod.rs @@ -606,24 +606,24 @@ impl IRHelper for IntermediateRepr { .collect::>>>( )?; - let item_types: Vec<&FieldType> = mapped_fields - .values() - .map(|i| &i.meta().1) - .dedup() - .collect(); - let items_type = match item_types.len() { - 0 => None, - 1 => Some(item_types[0].clone()), - _ => Some(FieldType::Union( - item_types.into_iter().map(|t| t.clone()).collect(), - )), - }; - if let Some((key_ty, value_ty)) = map_types(self, &field_type) { - let expected_type = FieldType::Map(Box::new(key_ty.clone()), Box::new(value_ty.clone())); - if !self.is_subtype(&expected_type, &field_base_type) { - anyhow::bail!("Could not unify {:?} with {:?}", expected_type, field_base_type); - } - } + // let item_types: Vec<&FieldType> = mapped_fields + // .values() + // .map(|i| &i.meta().1) + // .dedup() + // .collect(); + // let items_type = match item_types.len() { + // 0 => None, + // 1 => Some(item_types[0].clone()), + // _ => Some(FieldType::Union( + // item_types.into_iter().map(|t| t.clone()).collect(), + // )), + // }; + // if let Some((key_ty, value_ty)) = map_types(self, &field_type) { + // let expected_type = FieldType::Map(Box::new(key_ty.clone()), Box::new(value_ty.clone())); + // if !self.is_subtype(&expected_type, &field_base_type) { + // anyhow::bail!("Could not unify {:?} with {:?}", expected_type, field_base_type); + // } + // } Ok(BamlValueWithMeta::Map(mapped_fields, (meta, field_type))) } @@ -640,21 +640,21 @@ impl IRHelper for IntermediateRepr { }) .collect::>>()?; // dbg!(&new_items); - let item_types: Vec<&FieldType> = - new_items.iter().map(|i| &i.meta().1).dedup().collect(); - let items_type = match item_types.len() { - 0 => None, - 1 => Some(item_types[0].clone()), - _ => Some(FieldType::Union( - item_types.into_iter().map(|t| t.clone()).collect(), - )), - }; - if let Some(ty) = items_type { - let expected_type = FieldType::List(Box::new(ty)); - if !self.is_subtype(&expected_type, &field_base_type) { - anyhow::bail!("Could not unify {:?} with {:?}", expected_type, field_base_type); - } - } + // let item_types: Vec<&FieldType> = + // new_items.iter().map(|i| &i.meta().1).dedup().collect(); + // let items_type = match item_types.len() { + // 0 => None, + // 1 => Some(item_types[0].clone()), + // _ => Some(FieldType::Union( + // item_types.into_iter().map(|t| t.clone()).collect(), + // )), + // }; + // if let Some(ty) = items_type { + // let expected_type = FieldType::List(Box::new(ty)); + // if !self.is_subtype(&expected_type, &field_base_type) { + // anyhow::bail!("Could not unify {:?} with {:?}", expected_type, field_base_type); + // } + // } Ok(BamlValueWithMeta::List(new_items, (meta, field_type))) } @@ -838,7 +838,7 @@ impl IRHelper for IntermediateRepr { /// should declare as the `item_type` in the case of unions that /// admit multiple different children. (Perhaps a union of all the /// child-having variants?). -fn item_type( +fn item_type( ir: &IntermediateRepr, field_type: &FieldType, baml_child_value: &BamlValueWithMeta, @@ -857,9 +857,8 @@ fn item_type( FieldType::Union(variants) => variants .into_iter() .find(|variant| { - ir - .distribute_type_with_meta(baml_child_value.clone(), variant.clone().to_owned()) - .is_ok() + typecheck_value_with_meta(ir, baml_child_value, variant) + }) .map(|v| v.clone()), FieldType::Tuple(_) => None, @@ -868,6 +867,96 @@ fn item_type( res } +fn typecheck_value_with_meta(ir: &IntermediateRepr, value: &BamlValueWithMeta, field_type: &FieldType) -> bool { + let field_base_type = ir.distribute_metadata(&field_type).0; + match value { + BamlValueWithMeta::String(s, meta) => { + let literal_type = FieldType::Literal(LiteralValue::String(s.clone())); + let primitive_type = FieldType::Primitive(TypeValue::String); + + ir.is_subtype(&literal_type, &field_base_type) || ir.is_subtype(&primitive_type, &field_base_type) + + }, + BamlValueWithMeta::Int(i, meta) => + ir.is_subtype(&FieldType::Literal(LiteralValue::Int(*i)), &field_base_type), + BamlValueWithMeta::Float(f, meta) => ir.is_subtype(&FieldType::Primitive(TypeValue::Float), &field_base_type), + + BamlValueWithMeta::Bool(b, meta) => { + let literal_type = FieldType::Literal(LiteralValue::Bool(*b)); + let primitive_type = FieldType::Primitive(TypeValue::Bool); + + ir.is_subtype(&literal_type, &field_base_type) || ir.is_subtype(&primitive_type, &field_base_type) + }, + + BamlValueWithMeta::Null(meta) => true, + + // TODO: Handle enums and literal keys. + BamlValueWithMeta::Map(pairs, meta) => { + true + // TODO! + }, + + BamlValueWithMeta::List(items, meta) => { + let items_ok = items.iter().map(|i| item_type(ir, &field_type, i).map_or(true, |item_ty| typecheck_value_with_meta(ir, i, &item_ty))).all(|x| x); + items_ok + } + + BamlValueWithMeta::Media(m, meta) => + ir.is_subtype( + &FieldType::Primitive(TypeValue::Media(m.media_type)), + &field_base_type, + ), + + BamlValueWithMeta::Enum(name, val, meta) => + ir.is_subtype(&FieldType::Enum(name.clone()), &field_base_type), + + BamlValueWithMeta::Class(name, fields, meta) => { + // // Classes not present in the IR may be dynamically generated. + // // In this case, all types will be inferred, rather than distributed + // // from the `field_type` parameter. + + // TODO + true + + // if ir.find_class(&name).is_err() { + // return distribute_infer_class(self, &name, fields, meta); + // } + // if !self.is_subtype(&FieldType::Class(name.clone()), &field_base_type) { + // anyhow::bail!("Could not unify Class {} with {:?}", name, field_base_type); + // } else { + // let class_type = &self.find_class(&name)?.item.elem; + // let class_fields: BamlMap = class_type + // .static_fields + // .iter() + // .map(|field_node| { + // ( + // field_node.elem.name.clone(), + // field_node.elem.r#type.elem.clone(), + // ) + // }) + // .collect(); + // let mapped_fields = fields + // .into_iter() + // .map(|(k, v)| { + // let field_type = match class_fields.get(k.as_str()) { + // Some(ft) => ft.clone(), + // None => infer_type_with_meta(&v).unwrap_or(UNIT_TYPE), + // }; + // let mapped_field = self.distribute_type_with_meta(v, field_type)?; + // Ok((k, mapped_field)) + // }) + // .collect::>>>( + // )?; + // Ok(BamlValueWithMeta::Class( + // name, + // mapped_fields, + // (meta, field_type), + // )) + // } + } + } +} + /// Like item_type, but specialized for maps. fn map_types<'ir, 'a>( ir: &'ir IntermediateRepr, diff --git a/engine/baml-lib/baml-types/src/baml_value.rs b/engine/baml-lib/baml-types/src/baml_value.rs index 29af2bb97..f6e772dce 100644 --- a/engine/baml-lib/baml-types/src/baml_value.rs +++ b/engine/baml-lib/baml-types/src/baml_value.rs @@ -480,49 +480,47 @@ impl BamlValueWithMeta { /// /// The baml value calling `zip_meta` is the "primary" one, whose value /// data will live on in the returned baml value. - pub fn zip_meta(self, other: BamlValueWithMeta) -> Result> + pub fn zip_meta(self, other: &BamlValueWithMeta) -> Result> where T: std::fmt::Debug { let other_meta: U = other.meta().clone(); - let error_msg = format!("Could not unify {:?} with {:?}.", self, other); + let error_msg = String::new(); // format!("Could not unify {:?} with {:?}.", self, other); let ret = match (self, other) { (BamlValueWithMeta::Null(meta1), _) => { Result::<_,_>::Ok(BamlValueWithMeta::Null((meta1, other_meta))) }, - (BamlValueWithMeta::String(s1, meta1), BamlValueWithMeta::String(s2, meta2)) if s1 == s2 => Ok(BamlValueWithMeta::String(s1, (meta1, meta2))), + (BamlValueWithMeta::String(s1, meta1), BamlValueWithMeta::String(_s2, _)) if true => Ok(BamlValueWithMeta::String(s1, (meta1, other_meta))), (BamlValueWithMeta::String(_,_), _) => anyhow::bail!("Unification error"), - (BamlValueWithMeta::Int(s1, meta1), BamlValueWithMeta::Int(s2, meta2)) if s1 == s2 => Ok(BamlValueWithMeta::Int(s1, (meta1, meta2))), + (BamlValueWithMeta::Int(s1, meta1), BamlValueWithMeta::Int(_s2, _)) if true => Ok(BamlValueWithMeta::Int(s1, (meta1, other_meta))), (BamlValueWithMeta::Int(_,_), _) => anyhow::bail!("Unification error"), - (BamlValueWithMeta::Float(s1, meta1), BamlValueWithMeta::Float(s2, meta2)) if s1 == s2 => Ok(BamlValueWithMeta::Float(s1, (meta1, meta2))), + (BamlValueWithMeta::Float(s1, meta1), BamlValueWithMeta::Float(_s2, _)) if true => Ok(BamlValueWithMeta::Float(s1, (meta1, other_meta))), (BamlValueWithMeta::Float(_,_), _) => anyhow::bail!("Unification error"), - (BamlValueWithMeta::Bool(s1, meta1), BamlValueWithMeta::Bool(s2, meta2)) if s1 == s2 => Ok(BamlValueWithMeta::Bool(s1, (meta1, meta2))), + (BamlValueWithMeta::Bool(s1, meta1), BamlValueWithMeta::Bool(_s2, _)) if true => Ok(BamlValueWithMeta::Bool(s1, (meta1, other_meta))), (BamlValueWithMeta::Bool(_,_), _) => anyhow::bail!("Unification error"), - (BamlValueWithMeta::Map(s1, meta1), BamlValueWithMeta::Map(s2, meta2)) => { - // s1.sort_unstable_keys(); - // s2.sort_unstable_keys(); + (BamlValueWithMeta::Map(s1, meta1), BamlValueWithMeta::Map(s2, _)) => { let map_result = s1.into_iter().zip(s2).map(|((k1,v1), (_k2,v2))| { v1.zip_meta(v2).map(|res| (k1, res)) }).collect::>>()?; - Ok(BamlValueWithMeta::Map(map_result, (meta1, meta2))) + Ok(BamlValueWithMeta::Map(map_result, (meta1, other_meta))) }, (BamlValueWithMeta::Map(_,_), _) => anyhow::bail!("Unification error"), - (BamlValueWithMeta::List(l1, meta1), BamlValueWithMeta::List(l2, meta2)) => { + (BamlValueWithMeta::List(l1, meta1), BamlValueWithMeta::List(l2, _)) => { let list_result = l1.into_iter().zip(l2).map(|(item1, item2)| { item1.zip_meta(item2) }).collect::>>()?; - Ok( BamlValueWithMeta::List(list_result, (meta1, meta2))) + Ok( BamlValueWithMeta::List(list_result, (meta1, other_meta))) } (BamlValueWithMeta::List(_,_), _) => anyhow::bail!("Unification error"), - (BamlValueWithMeta::Media(m1, meta1), BamlValueWithMeta::Media(m2, meta2)) if m1 == m2 => { - Ok(BamlValueWithMeta::Media(m1, (meta1, meta2))) + (BamlValueWithMeta::Media(m1, meta1), BamlValueWithMeta::Media(_m2, _)) if true => { + Ok(BamlValueWithMeta::Media(m1, (meta1, other_meta))) } (BamlValueWithMeta::Media(_, _), _) => anyhow::bail!("Unification error"), - (BamlValueWithMeta::Enum(x1, y1, meta1), BamlValueWithMeta::Enum(x2, y2, meta2)) if x1 == x2 && y1 == y2 => { - Ok(BamlValueWithMeta::Enum(x1, y1, (meta1, meta2))) + (BamlValueWithMeta::Enum(x1, y1, meta1), BamlValueWithMeta::Enum(_x2, _y2, _)) if true => { + Ok(BamlValueWithMeta::Enum(x1, y1, (meta1, other_meta))) } (BamlValueWithMeta::Enum(_, _, _), _) => anyhow::bail!("Unification error"), - (BamlValueWithMeta::Class(name1, fields1, meta1), BamlValueWithMeta::Class(name2, fields2, meta2)) if name1 == name2 => { + (BamlValueWithMeta::Class(name1, fields1, meta1), BamlValueWithMeta::Class(_name2, fields2, _)) if true => { // TODO: We can remove a `clone` by checking that the fields // are ordered the same way between the two classes, then consuming // both classs' fields in parallel. @@ -531,9 +529,9 @@ impl BamlValueWithMeta { // }).collect::>>()?; let map_result = fields1.into_iter().map(|(k1, v1)| { let v2 = fields2.get(&k1).context("Missing expected key")?; - v1.zip_meta(v2.clone()).map(|r| (k1, r)) + v1.zip_meta(v2).map(|r| (k1, r)) }).collect::>>()?; - Ok(BamlValueWithMeta::Class(name1, map_result, (meta1, meta2))) + Ok(BamlValueWithMeta::Class(name1, map_result, (meta1, other_meta))) } (BamlValueWithMeta::Class(_, _, _), _) => anyhow::bail!("Unification error"), }; diff --git a/engine/baml-lib/jsonish/src/deserializer/semantic_streaming.rs b/engine/baml-lib/jsonish/src/deserializer/semantic_streaming.rs index aa99b086e..d674eb4b9 100644 --- a/engine/baml-lib/jsonish/src/deserializer/semantic_streaming.rs +++ b/engine/baml-lib/jsonish/src/deserializer/semantic_streaming.rs @@ -64,18 +64,12 @@ fn process_node( value: BamlValueWithMeta<(CompletionState, &FieldType)>, allow_partials: bool, ) -> Result>, StreamingError> { - let (completion_state, field_type) = value.meta(); + let (completion_state, field_type) = value.meta().clone(); let (base_type, (_, streaming_behavior)) = ir.distribute_metadata(field_type); let must_be_done = required_done(ir, field_type) && allow_partials; - // eprintln!("Working on {value:?}"); - // eprintln!(" completion: {completion_state:?}"); - // eprintln!(" field_type: {field_type:?}"); - // eprintln!(" mustbedone: {}", must_be_done); - - if must_be_done && !(completion_state == &CompletionState::Complete) { - // eprintln!(" Aborting because incomplete"); + if must_be_done && !(completion_state == CompletionState::Complete) { return Err(StreamingError::IncompleteDoneValue); } @@ -84,7 +78,6 @@ fn process_node( } else { None }; - // eprintln!(" new_meta: {:?}", new_meta); let new_value = match value { BamlValueWithMeta::String(s, _) => Ok(BamlValueWithMeta::String(s, new_meta)), @@ -100,10 +93,31 @@ fn process_node( .collect(), new_meta, )), - BamlValueWithMeta::Class(ref class_name, ref fields, _) => { + BamlValueWithMeta::Class(ref class_name, fields, _) => { + let field_names: HashSet = fields.keys().into_iter().map(|s| s.to_string()).collect(); let needed_fields: HashSet = needed_fields(ir, field_type, allow_partials)?; + // let missing_needed_fields = needed_fields.difference(&new_field_names); + let missing_needed_fields: Vec = needed_fields.iter().filter(|field_name| !fields.contains_key(*field_name)).map(|s| s.to_string()).collect(); + let unneeded_fields = field_names.difference(&needed_fields); + + let nulls_for_unneeded_fields = unneeded_fields + .into_iter() + .filter_map(|unneeded_field_name| { + let field = fields.get(unneeded_field_name).expect("This field is guaranteed to be in the field set"); + let use_state = type_streaming_behavior(ir, field.meta().1).state; + let field_stream_state = if use_state { + Some(CompletionState::Incomplete) + } else { + None + }; + Some(( + unneeded_field_name.clone(), + BamlValueWithMeta::Null(field_stream_state), + )) + }) + .collect::>>(); + let mut new_fields = fields - .clone() .into_iter() .filter_map(|(field_name, field_value)| { process_node(ir, field_value, allow_partials) @@ -111,38 +125,8 @@ fn process_node( .map(|v| (field_name, v)) }) .collect::>>(); - let new_field_names = new_fields - .iter() - .filter_map(|(field_name, field_value)| match field_value { - BamlValueWithMeta::Null(_) => None, - _ => Some(field_name.clone()), - }) - .collect(); - let missing_needed_fields = needed_fields.difference(&new_field_names); - let nulls_for_unneeded_fields = fields - .iter() - .filter_map(|(field_name, field)| { - if needed_fields.contains(field_name) || new_fields.get(field_name).is_some() { - None - } else { - let use_state = type_streaming_behavior(ir, field.meta().1).state; - let field_stream_state = if use_state { - Some(CompletionState::Incomplete) - } else { - None - }; - Some(( - field_name.clone(), - BamlValueWithMeta::Null(field_stream_state), - )) - } - }) - .collect::>>(); - // dbg!(&nulls_for_unneeded_fields); - // dbg!(&new_fields); new_fields.extend(nulls_for_unneeded_fields); - // dbg!(&new_fields); let res = BamlValueWithMeta::Class( class_name.clone(), new_fields, @@ -151,7 +135,7 @@ fn process_node( // if class_name == "Person" { // eprintln!("In PERSON: Result: {:?}", res); // } - if missing_needed_fields.clone().count() == 0 { + if missing_needed_fields.clone().len() == 0 { Ok(res) } else { // eprintln!(" Missing needed fields: {missing_needed_fields:?}"); diff --git a/engine/baml-lib/jsonish/src/lib.rs b/engine/baml-lib/jsonish/src/lib.rs index 0470bda28..5ff1cd6f0 100644 --- a/engine/baml-lib/jsonish/src/lib.rs +++ b/engine/baml-lib/jsonish/src/lib.rs @@ -1,5 +1,4 @@ -#[cfg(test)] -mod tests; +pub mod tests; use anyhow::Result; use indexmap::IndexMap; diff --git a/engine/baml-lib/jsonish/src/tests/animation.rs b/engine/baml-lib/jsonish/src/tests/animation.rs index 72b1dd148..0084398f1 100644 --- a/engine/baml-lib/jsonish/src/tests/animation.rs +++ b/engine/baml-lib/jsonish/src/tests/animation.rs @@ -1,3 +1,4 @@ +#[cfg(test)] use crate::{from_str, tests::parsed_value_to_response}; use baml_types::{FieldType, StreamingBehavior}; use internal_baml_core::ir::repr::make_test_ir; diff --git a/engine/baml-lib/jsonish/src/tests/mod.rs b/engine/baml-lib/jsonish/src/tests/mod.rs index 417025ba4..ecb82ba6a 100644 --- a/engine/baml-lib/jsonish/src/tests/mod.rs +++ b/engine/baml-lib/jsonish/src/tests/mod.rs @@ -61,7 +61,7 @@ fn load_test_ir(file_content: &str) -> IntermediateRepr { IntermediateRepr::from_parser_database(&schema.db, schema.configuration).unwrap() } -fn render_output_format( +pub fn render_output_format( ir: &IntermediateRepr, output: &FieldType, env_values: &EvaluationContext<'_>, diff --git a/engine/baml-runtime/Cargo.toml b/engine/baml-runtime/Cargo.toml index fdd11b554..b49cafcd8 100644 --- a/engine/baml-runtime/Cargo.toml +++ b/engine/baml-runtime/Cargo.toml @@ -92,6 +92,7 @@ tracing = { version = "0.1.40", features = ["valuable"] } tracing-subscriber = { version = "0.3.18", features = ["json", "env-filter","valuable"] } thiserror = "2.0.1" log-once = "0.4.1" +criterion = "0.5.1" [target.'cfg(target_arch = "wasm32")'.dependencies] @@ -150,6 +151,7 @@ skip-integ-tests = [] [dev-dependencies] assert_cmd = "2" console_log = "1" +criterion = "0.5.1" dissimilar = "1.0.4" expect-test = "1.1.0" indoc.workspace = true @@ -158,3 +160,8 @@ rstest = "0.22.0" wasm-bindgen-test = "0.3.42" walkdir = "2.5.0" wasm-logger = "0.2.0" + +[[ bench ]] +name = "bench" +path = "benches/bench.rs" +harness = false diff --git a/engine/baml-runtime/benches/lib.rs b/engine/baml-runtime/benches/lib.rs new file mode 100644 index 000000000..d33fc0100 --- /dev/null +++ b/engine/baml-runtime/benches/lib.rs @@ -0,0 +1 @@ +pub mod sap_parser_benchmark; \ No newline at end of file diff --git a/engine/baml-runtime/benches/sap_parser_benchmark.rs b/engine/baml-runtime/benches/sap_parser_benchmark.rs new file mode 100644 index 000000000..e69de29bb diff --git a/engine/baml-runtime/src/internal/llm_client/mod.rs b/engine/baml-runtime/src/internal/llm_client/mod.rs index 3d99970a6..b7eda3c52 100644 --- a/engine/baml-runtime/src/internal/llm_client/mod.rs +++ b/engine/baml-runtime/src/internal/llm_client/mod.rs @@ -61,8 +61,8 @@ pub fn parsed_value_to_response( // into a final value. // Node that we set the StreamState to `None` unless `allow_partials`. let response_value = baml_value_with_streaming - .zip_meta(value_with_response_checks)? - .zip_meta(meta_flags)? + .zip_meta(&value_with_response_checks)? + .zip_meta(&meta_flags)? .map_meta(|((x, y), z)| (z.clone(), y.clone(), if allow_partials { x.clone() } else { None } )); Ok(ResponseBamlValue(response_value)) } diff --git a/engine/language_client_python/src/types/function_results.rs b/engine/language_client_python/src/types/function_results.rs index c595308e7..395d5c99f 100644 --- a/engine/language_client_python/src/types/function_results.rs +++ b/engine/language_client_python/src/types/function_results.rs @@ -80,7 +80,7 @@ fn pythonize_strict( enum_module: &Bound<'_, PyModule>, cls_module: &Bound<'_, PyModule>, ) -> PyResult { - // eprintln!("pythonize_strict parsed: {:?}", parsed); + eprintln!("pythonize_strict parsed: {:?}", parsed); let meta = parsed.0.meta().clone(); let py_value_without_constraints = match parsed.0 { BamlValueWithMeta::String(val, _) => val.into_py_any(py), diff --git a/integ-tests/baml_src/test-files/functions/output/class.baml b/integ-tests/baml_src/test-files/functions/output/class.baml index c8ef29ba4..ea29a779c 100644 --- a/integ-tests/baml_src/test-files/functions/output/class.baml +++ b/integ-tests/baml_src/test-files/functions/output/class.baml @@ -1,5 +1,5 @@ class TestOutputClass { - prop1 string @description("A long string with about 200 words") @stream.done @stream.with_state + prop1 string @description("A long string with about 200 words") prop2 int } diff --git a/integ-tests/python/tests/test_functions.py b/integ-tests/python/tests/test_functions.py index a6fdcdc50..55732f0c6 100644 --- a/integ-tests/python/tests/test_functions.py +++ b/integ-tests/python/tests/test_functions.py @@ -363,6 +363,7 @@ async def test_json_type_alias_cycle(self): res = await b.JsonTypeAliasCycle(data) assert res == data assert res["json"]["object"]["list"] == [1, 2, 3] + assert False class MyCustomClass(NamedArgsSingleClass): @@ -382,38 +383,38 @@ async def accepts_subclass_of_baml_type(): @pytest.mark.asyncio async def test_should_work_for_all_outputs(): a = "a" # dummy - res = await b.FnOutputBool(a) - assert res == True + # res = await b.FnOutputBool(a) + # assert res == True - integer = await b.FnOutputInt(a) - assert integer == 5 + # integer = await b.FnOutputInt(a) + # assert integer == 5 - literal_integer = await b.FnOutputLiteralInt(a) - assert literal_integer == 5 + # literal_integer = await b.FnOutputLiteralInt(a) + # assert literal_integer == 5 - literal_bool = await b.FnOutputLiteralBool(a) - assert literal_bool == False + # literal_bool = await b.FnOutputLiteralBool(a) + # assert literal_bool == False - literal_string = await b.FnOutputLiteralString(a) - assert literal_string == "example output" + # literal_string = await b.FnOutputLiteralString(a) + # assert literal_string == "example output" list = await b.FnOutputClassList(a) # Broken assert len(list) > 0 assert len(list[0].prop1) > 0 - classWEnum = await b.FnOutputClassWithEnum(a) - assert classWEnum.prop2 in ["ONE", "TWO"] + # classWEnum = await b.FnOutputClassWithEnum(a) + # assert classWEnum.prop2 in ["ONE", "TWO"] - classs = await b.FnOutputClass(a) - assert classs.prop1 is not None - assert classs.prop2 == 540 + # classs = await b.FnOutputClass(a) + # assert classs.prop1 is not None + # assert classs.prop2 == 540 - enumList = await b.FnEnumListOutput(a) - assert len(enumList) == 2 + # enumList = await b.FnEnumListOutput(a) + # assert len(enumList) == 2 - myEnum = await b.FnEnumOutput(a) - # As no check is added for myEnum, adding a simple assert to ensure the call was made - assert myEnum is not None + # myEnum = await b.FnEnumOutput(a) + # # As no check is added for myEnum, adding a simple assert to ensure the call was made + # assert myEnum is not None @pytest.mark.asyncio diff --git a/integ-tests/typescript/baml_client/partial_types.ts b/integ-tests/typescript/baml_client/partial_types.ts index 3aa79925a..b1bc5ed5b 100644 --- a/integ-tests/typescript/baml_client/partial_types.ts +++ b/integ-tests/typescript/baml_client/partial_types.ts @@ -459,7 +459,7 @@ export interface TestClassWithEnum { } export interface TestOutputClass { - prop1: string + prop1?: (string | null) prop2?: (number | null) } diff --git a/integ-tests/typescript/test-report.html b/integ-tests/typescript/test-report.html index 5f5da0f20..4fc85d981 100644 --- a/integ-tests/typescript/test-report.html +++ b/integ-tests/typescript/test-report.html @@ -257,4 +257,5 @@ font-size: 1rem; padding: 0 0.5rem; } -

Test Report

Started: 2025-01-09 13:22:56
Suites (2)
1 passed
0 failed
1 pending
Tests (91)
1 passed
0 failed
90 pending
Trace Test
traceSync with Return
pending
0s
Trace Test
traceSync with no return
pending
0s
Trace Test
traceAsync with Return
pending
0s
Trace Test
traceAsync with no return
pending
0s
Integ tests
should handle invalid AWS region gracefully
pending
0s
Integ tests
should handle invalid AWS access keygracefully
pending
0s
Integ tests
should handle invalid AWS profile gracefully
pending
0s
Integ tests
should handle invalid AWS session token gracefully
pending
0s
Integ tests > should work for all inputs
single bool
pending
0s
Integ tests > should work for all inputs
single string list
pending
0s
Integ tests > should work for all inputs
return literal union
pending
0s
Integ tests > should work for all inputs
optional list and map
pending
0s
Integ tests > should work for all inputs
single class
pending
0s
Integ tests > should work for all inputs
multiple classes
pending
0s
Integ tests > should work for all inputs
single enum list
pending
0s
Integ tests > should work for all inputs
single float
pending
0s
Integ tests > should work for all inputs
single int
pending
0s
Integ tests > should work for all inputs
single literal int
pending
0s
Integ tests > should work for all inputs
single literal bool
pending
0s
Integ tests > should work for all inputs
single literal string
pending
0s
Integ tests > should work for all inputs
single class with literal prop
pending
0s
Integ tests > should work for all inputs
single class with literal union prop
pending
0s
Integ tests > should work for all inputs
single optional string
pending
0s
Integ tests > should work for all inputs
single map string to string
pending
0s
Integ tests > should work for all inputs
single map string to class
pending
0s
Integ tests > should work for all inputs
single map string to map
pending
0s
Integ tests > should work for all inputs
enum key in map
pending
0s
Integ tests > should work for all inputs
literal string union key in map
pending
0s
Integ tests > should work for all inputs
single literal string key in map
pending
0s
Integ tests > should work for all inputs
primitive union alias
pending
0s
Integ tests > should work for all inputs
map alias
pending
0s
Integ tests > should work for all inputs
alias union
pending
0s
Integ tests > should work for all inputs
alias pointing to recursive class
pending
0s
Integ tests > should work for all inputs
class pointing to alias that points to recursive class
pending
0s
Integ tests > should work for all inputs
recursive class with alias indirection
pending
0s
Integ tests > should work for all inputs
merge alias attributes
pending
0s
Integ tests > should work for all inputs
simple recursive map alias
pending
0s
Integ tests > should work for all inputs
simple recursive map alias
pending
0s
Integ tests > should work for all inputs
recursive alias cycles
pending
0s
Integ tests > should work for all inputs
json type alias cycle
pending
0s
Integ tests
should work for all outputs
passed
13.654s
Integ tests
works with retries1
pending
0s
Integ tests
works with retries2
pending
0s
Integ tests
works with fallbacks
pending
0s
Integ tests
should work with image from url
pending
0s
Integ tests
should work with image from base 64
pending
0s
Integ tests
should work with audio base 64
pending
0s
Integ tests
should work with audio from url
pending
0s
Integ tests
should support streaming in OpenAI
pending
0s
Integ tests
should support streaming in Gemini
pending
0s
Integ tests
should support AWS
pending
0s
Integ tests
should support streaming in AWS
pending
0s
Integ tests
should allow overriding the region
pending
0s
Integ tests
should support OpenAI shorthand
pending
0s
Integ tests
should support OpenAI shorthand streaming
pending
0s
Integ tests
should support anthropic shorthand
pending
0s
Integ tests
should support anthropic shorthand streaming
pending
0s
Integ tests
should support streaming without iterating
pending
0s
Integ tests
should support streaming in Claude
pending
0s
Integ tests
should support azure
pending
0s
Integ tests
should support azure streaming
pending
0s
Integ tests
should fail if azure is not configured
pending
0s
Integ tests
should support vertex
pending
0s
Integ tests
supports tracing sync
pending
0s
Integ tests
supports tracing async
pending
0s
Integ tests
should work with dynamic types single
pending
0s
Integ tests
should work with dynamic types enum
pending
0s
Integ tests
should work with dynamic literals
pending
0s
Integ tests
should work with dynamic types class
pending
0s
Integ tests
should work with dynamic inputs class
pending
0s
Integ tests
should work with dynamic inputs list
pending
0s
Integ tests
should work with dynamic output map
pending
0s
Integ tests
should work with dynamic output union
pending
0s
Integ tests
should work with nested classes
pending
0s
Integ tests
should work with dynamic client
pending
0s
Integ tests
should work with 'onLogEvent'
pending
0s
Integ tests
should work with a sync client
pending
0s
Integ tests
should raise an error when appropriate
pending
0s
Integ tests
should raise a BAMLValidationError
pending
0s
Integ tests
should reset environment variables correctly
pending
0s
Integ tests
should use aliases when serializing input objects - classes
pending
0s
Integ tests
should use aliases when serializing, but still have original keys in jinja
pending
0s
Integ tests
should use aliases when serializing input objects - enums
pending
0s
Integ tests
should use aliases when serializing input objects - lists
pending
0s
Integ tests
constraints: should handle checks in return types
pending
0s
Integ tests
constraints: should handle checks in returned unions
pending
0s
Integ tests
constraints: should handle block-level checks
pending
0s
Integ tests
constraints: should handle nested-block-level checks
pending
0s
Integ tests
simple recursive type
pending
0s
Integ tests
mutually recursive type
pending
0s
should support semantic streaming
pending
0s
\ No newline at end of file +

Test Report

Started: 2025-01-09 21:28:50
Suites (2)
0 passed
1 failed
1 pending
Tests (91)
0 passed
1 failed
90 pending
Trace Test
traceSync with Return
pending
0s
Trace Test
traceSync with no return
pending
0s
Trace Test
traceAsync with Return
pending
0s
Trace Test
traceAsync with no return
pending
0s
Integ tests
should handle invalid AWS region gracefully
pending
0s
Integ tests
should handle invalid AWS access keygracefully
pending
0s
Integ tests
should handle invalid AWS profile gracefully
pending
0s
Integ tests
should handle invalid AWS session token gracefully
pending
0s
Integ tests > should work for all inputs
single bool
pending
0s
Integ tests > should work for all inputs
single string list
pending
0s
Integ tests > should work for all inputs
return literal union
pending
0s
Integ tests > should work for all inputs
optional list and map
pending
0s
Integ tests > should work for all inputs
single class
pending
0s
Integ tests > should work for all inputs
multiple classes
pending
0s
Integ tests > should work for all inputs
single enum list
pending
0s
Integ tests > should work for all inputs
single float
pending
0s
Integ tests > should work for all inputs
single int
pending
0s
Integ tests > should work for all inputs
single literal int
pending
0s
Integ tests > should work for all inputs
single literal bool
pending
0s
Integ tests > should work for all inputs
single literal string
pending
0s
Integ tests > should work for all inputs
single class with literal prop
pending
0s
Integ tests > should work for all inputs
single class with literal union prop
pending
0s
Integ tests > should work for all inputs
single optional string
pending
0s
Integ tests > should work for all inputs
single map string to string
pending
0s
Integ tests > should work for all inputs
single map string to class
pending
0s
Integ tests > should work for all inputs
single map string to map
pending
0s
Integ tests > should work for all inputs
enum key in map
pending
0s
Integ tests > should work for all inputs
literal string union key in map
pending
0s
Integ tests > should work for all inputs
single literal string key in map
pending
0s
Integ tests > should work for all inputs
primitive union alias
pending
0s
Integ tests > should work for all inputs
map alias
pending
0s
Integ tests > should work for all inputs
alias union
pending
0s
Integ tests > should work for all inputs
alias pointing to recursive class
pending
0s
Integ tests > should work for all inputs
class pointing to alias that points to recursive class
pending
0s
Integ tests > should work for all inputs
recursive class with alias indirection
pending
0s
Integ tests > should work for all inputs
merge alias attributes
pending
0s
Integ tests > should work for all inputs
simple recursive map alias
pending
0s
Integ tests > should work for all inputs
simple recursive map alias
pending
0s
Integ tests > should work for all inputs
recursive alias cycles
pending
0s
Integ tests > should work for all inputs
json type alias cycle
pending
0s
Integ tests
should work for all outputs
failed
4.785s
TypeError: Cannot read properties of null (reading 'length')
+    at Object.length (/Users/greghale/code/baml/integ-tests/typescript/tests/integ-tests.test.ts:312:26)
Integ tests
works with retries1
pending
0s
Integ tests
works with retries2
pending
0s
Integ tests
works with fallbacks
pending
0s
Integ tests
should work with image from url
pending
0s
Integ tests
should work with image from base 64
pending
0s
Integ tests
should work with audio base 64
pending
0s
Integ tests
should work with audio from url
pending
0s
Integ tests
should support streaming in OpenAI
pending
0s
Integ tests
should support streaming in Gemini
pending
0s
Integ tests
should support AWS
pending
0s
Integ tests
should support streaming in AWS
pending
0s
Integ tests
should allow overriding the region
pending
0s
Integ tests
should support OpenAI shorthand
pending
0s
Integ tests
should support OpenAI shorthand streaming
pending
0s
Integ tests
should support anthropic shorthand
pending
0s
Integ tests
should support anthropic shorthand streaming
pending
0s
Integ tests
should support streaming without iterating
pending
0s
Integ tests
should support streaming in Claude
pending
0s
Integ tests
should support azure
pending
0s
Integ tests
should support azure streaming
pending
0s
Integ tests
should fail if azure is not configured
pending
0s
Integ tests
should support vertex
pending
0s
Integ tests
supports tracing sync
pending
0s
Integ tests
supports tracing async
pending
0s
Integ tests
should work with dynamic types single
pending
0s
Integ tests
should work with dynamic types enum
pending
0s
Integ tests
should work with dynamic literals
pending
0s
Integ tests
should work with dynamic types class
pending
0s
Integ tests
should work with dynamic inputs class
pending
0s
Integ tests
should work with dynamic inputs list
pending
0s
Integ tests
should work with dynamic output map
pending
0s
Integ tests
should work with dynamic output union
pending
0s
Integ tests
should work with nested classes
pending
0s
Integ tests
should work with dynamic client
pending
0s
Integ tests
should work with 'onLogEvent'
pending
0s
Integ tests
should work with a sync client
pending
0s
Integ tests
should raise an error when appropriate
pending
0s
Integ tests
should raise a BAMLValidationError
pending
0s
Integ tests
should reset environment variables correctly
pending
0s
Integ tests
should use aliases when serializing input objects - classes
pending
0s
Integ tests
should use aliases when serializing, but still have original keys in jinja
pending
0s
Integ tests
should use aliases when serializing input objects - enums
pending
0s
Integ tests
should use aliases when serializing input objects - lists
pending
0s
Integ tests
constraints: should handle checks in return types
pending
0s
Integ tests
constraints: should handle checks in returned unions
pending
0s
Integ tests
constraints: should handle block-level checks
pending
0s
Integ tests
constraints: should handle nested-block-level checks
pending
0s
Integ tests
simple recursive type
pending
0s
Integ tests
mutually recursive type
pending
0s
should support semantic streaming
pending
0s
\ No newline at end of file