Rectorization to introduce the separation between Model metadata and deployment

demo.py

@@ -17,7 +17,7 @@
 print(account.usages)
 
 # Make a request
-request = Request(model="ai21/j1-large", prompt="Life is like a box of", echo_prompt=True)
+request = Request(model_deployment="ai21/j2-large", prompt="Life is like a box of", echo_prompt=True)
 request_result: RequestResult = service.make_request(auth, request)
 print(request_result.completions[0].text)
 
@@ -28,12 +28,12 @@
 print(request_result.completions[0].text)
 
 # How to get the embedding for some text
-request = Request(model="openai/text-similarity-ada-001", prompt="Life is like a box of", embedding=True)
+request = Request(model_deployment="openai/text-similarity-ada-002", prompt="Life is like a box of", embedding=True)
 request_result = service.make_request(auth, request)
 print(request_result.embedding)
 
 # Tokenize
-request = TokenizationRequest(tokenizer="ai21/j1-jumbo", text="Tokenize me please.")
+request = TokenizationRequest(tokenizer="ai21/j2-jumbo", text="Tokenize me please.")
 tokenization_request_result: TokenizationRequestResult = service.tokenize(auth, request)
 print(f"Number of tokens: {len(tokenization_request_result.tokens)}")
 

scripts/compute_request_limits.py

@@ -57,7 +57,7 @@ def try_request(
 
     try:
         request = Request(
-            model=model_name,
+            model_deployment=model_name,
             prompt=prefix + " ".join(["hello"] * (sequence_length - num_tokens_prefix - num_tokens_suffix)) + suffix,
             max_tokens=num_tokens,
         )
@@ -287,7 +287,7 @@ def main():
     print("client successfully created")
 
     print("Making short request...")
-    request = Request(model=args.model_name, prompt=args.prefix + "hello" + args.suffix, max_tokens=1)
+    request = Request(model_deployment=args.model_name, prompt=args.prefix + "hello" + args.suffix, max_tokens=1)
     response = client.make_request(request)
     if not response.success:
         raise ValueError("Request failed")

src/helm/benchmark/adaptation/adapter_spec.py

@@ -73,7 +73,11 @@ class AdapterSpec:
 
     # Decoding parameters (inherited by `Request`)
 
-    # Model to make the request to (need to fill in)
+    # Model deployment to make the request to (need to fill in)
+    model_deployment: str = ""
+
+    # DEPRECATED: old model field, kept for backward compatibility
+    # TODO: Remove this once we do not wish to support backward compatibility anymore.
     model: str = ""
 
     # Temperature to use

src/helm/benchmark/adaptation/adapters/adapter.py

@@ -21,7 +21,7 @@ class Adapter(ABC):
     def __init__(self, adapter_spec: AdapterSpec, tokenizer_service: TokenizerService):
         self.adapter_spec: AdapterSpec = adapter_spec
         self.window_service: WindowService = WindowServiceFactory.get_window_service(
-            adapter_spec.model, tokenizer_service
+            adapter_spec.model_deployment, tokenizer_service
         )
 
     @abstractmethod

src/helm/benchmark/adaptation/adapters/binary_ranking_adapter.py

@@ -49,7 +49,7 @@ def generate_requests(
                 reference_index=reference_index,
             )
             request = Request(
-                model=self.adapter_spec.model,
+                model_deployment=self.adapter_spec.model_deployment,
                 prompt=prompt.text,
                 num_completions=self.adapter_spec.num_outputs,
                 temperature=self.adapter_spec.temperature,

src/helm/benchmark/adaptation/adapters/generation_adapter.py

@@ -38,7 +38,7 @@ def generate_requests(
             training_instances, eval_instance, include_output=False, reference_index=None
         )
         request = Request(
-            model=self.adapter_spec.model,
+            model_deployment=self.adapter_spec.model_deployment,
             prompt=prompt.text,
             num_completions=self.adapter_spec.num_outputs,
             temperature=self.adapter_spec.temperature,

src/helm/benchmark/adaptation/adapters/language_modeling_adapter.py

@@ -113,7 +113,7 @@ def _generate_requests(self, eval_instance: Instance) -> List[RequestState]:
             self.window_service.encode(prefix_token).tokens, tokens[:first_seq_len], max_request_length, text
         )
         request = Request(
-            model=self.adapter_spec.model,
+            model_deployment=self.adapter_spec.model_deployment,
             prompt=prompt_text,
             num_completions=1,
             temperature=0,
@@ -161,7 +161,7 @@ def _generate_requests(self, eval_instance: Instance) -> List[RequestState]:
             )
 
             request = Request(
-                model=self.adapter_spec.model,
+                model_deployment=self.adapter_spec.model_deployment,
                 prompt=prompt_text,
                 num_completions=1,
                 temperature=0,

src/helm/benchmark/adaptation/adapters/multimodal/generation_multimodal_adapter.py

@@ -28,7 +28,7 @@ def generate_requests(
         )
 
         request = Request(
-            model=self.adapter_spec.model,
+            model_deployment=self.adapter_spec.model_deployment,
             multimodal_prompt=prompt.multimedia_object,
             num_completions=self.adapter_spec.num_outputs,
             temperature=self.adapter_spec.temperature,

src/helm/benchmark/adaptation/adapters/multimodal/in_context_learning_multimodal_adapter.py

@@ -26,7 +26,7 @@ def generate_requests(
         )
 
         request = Request(
-            model=self.adapter_spec.model,
+            model_deployment=self.adapter_spec.model_deployment,
             multimodal_prompt=prompt.multimedia_object,
             num_completions=self.adapter_spec.num_outputs,
             temperature=self.adapter_spec.temperature,

src/helm/benchmark/adaptation/adapters/multimodal/test_in_context_learning_multimodal_adapter.py

@@ -21,7 +21,7 @@ def teardown_method(self, _):
 
     def test_construct_prompt(self):
         adapter_spec: AdapterSpec = AdapterSpec(
-            model="simple/model1",
+            model_deployment="simple/model1",
             method=ADAPT_GENERATION_MULTIMODAL,
             global_prefix="[START]",
             instructions="Please answer the following question about the images.",
@@ -90,7 +90,7 @@ def test_construct_prompt(self):
 
     def test_construct_prompt_multi_label(self):
         adapter_spec: AdapterSpec = AdapterSpec(
-            model="simple/model1",
+            model_deployment="simple/model1",
             method=ADAPT_GENERATION_MULTIMODAL,
             global_prefix="[START]",
             instructions="Please answer the following question about the images.",
@@ -170,7 +170,7 @@ def test_construct_prompt_idefics_instruct_example(self):
         Constructing the same prompt from this example: https://huggingface.co/blog/idefics
         """
         adapter_spec: AdapterSpec = AdapterSpec(
-            model="simple/model1",
+            model_deployment="simple/model1",
             method=ADAPT_GENERATION_MULTIMODAL,
             input_prefix="User: ",
             input_suffix="<end_of_utterance>",

src/helm/benchmark/adaptation/adapters/multiple_choice_joint_adapter.py

@@ -54,7 +54,7 @@ def generate_requests(
             for reference_index, reference in enumerate(eval_instance.references)
         )
         request = Request(
-            model=self.adapter_spec.model,
+            model_deployment=self.adapter_spec.model_deployment,
             prompt=prompt.text,
             num_completions=1,
             top_k_per_token=self.adapter_spec.num_outputs,

src/helm/benchmark/adaptation/adapters/multiple_choice_separate_adapter.py

@@ -40,7 +40,7 @@ def construct_request_state(
         request_mode: str = "original",
     ) -> RequestState:
         request = Request(
-            model=self.adapter_spec.model,
+            model_deployment=self.adapter_spec.model_deployment,
             prompt=prompt.text,
             num_completions=1,
             temperature=0,

src/helm/benchmark/adaptation/adapters/test_generation_adapter.py

@@ -32,7 +32,7 @@ def test_adapt(self):
 
     def test_construct_prompt(self):
         adapter_spec = AdapterSpec(
-            model="openai/davinci",
+            model_deployment="openai/davinci",
             method=ADAPT_GENERATION,
             input_prefix="",
             input_suffix="",
@@ -59,7 +59,11 @@ def test_construct_prompt(self):
 
     def test_construct_prompt_with_truncation(self):
         adapter_spec = AdapterSpec(
-            model="openai/davinci", method=ADAPT_GENERATION, input_prefix="", output_prefix="", max_tokens=100
+            model_deployment="openai/davinci",
+            method=ADAPT_GENERATION,
+            input_prefix="",
+            output_prefix="",
+            max_tokens=100,
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         correct_reference = Reference(Output(text=""), tags=[CORRECT_TAG])
@@ -80,7 +84,7 @@ def test_construct_prompt_with_truncation(self):
         assert prompt_text.count("eval") == 1948
 
     def test_sample_examples_without_references(self):
-        adapter_spec = AdapterSpec(method=ADAPT_GENERATION, model="openai/ada", max_train_instances=1)
+        adapter_spec = AdapterSpec(method=ADAPT_GENERATION, model_deployment="openai/ada", max_train_instances=1)
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         all_train_instances = [
             Instance(Input(text="prompt1"), references=[]),
@@ -92,7 +96,7 @@ def test_sample_examples_without_references(self):
         assert len(examples) == 1
 
     def test_sample_examples_open_ended_generation(self):
-        adapter_spec = AdapterSpec(method=ADAPT_GENERATION, model="openai/ada", max_train_instances=3)
+        adapter_spec = AdapterSpec(method=ADAPT_GENERATION, model_deployment="openai/ada", max_train_instances=3)
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
 
         all_train_instances: List[Instance] = [
@@ -106,7 +110,7 @@ def test_sample_examples_open_ended_generation(self):
         assert seed0_examples != seed1_examples, "Examples should differ when changing the seed"
 
     def test_sample_examples_open_ended_generation_stress(self):
-        adapter_spec = AdapterSpec(method=ADAPT_GENERATION, model="openai/ada", max_train_instances=5)
+        adapter_spec = AdapterSpec(method=ADAPT_GENERATION, model_deployment="openai/ada", max_train_instances=5)
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
 
         all_train_instances: List[Instance] = [
@@ -146,7 +150,7 @@ def test_sample_examples_open_ended_generation_stress(self):
 
     def test_multiple_correct_reference(self):
         adapter_spec = AdapterSpec(
-            method=ADAPT_GENERATION, model="openai/ada", max_train_instances=2, sample_train=False
+            method=ADAPT_GENERATION, model_deployment="openai/ada", max_train_instances=2, sample_train=False
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         train_instances = [
@@ -191,7 +195,11 @@ def test_multiple_correct_reference(self):
 
     def test_multiple_correct_reference_multi_label(self):
         adapter_spec = AdapterSpec(
-            method=ADAPT_GENERATION, model="openai/ada", max_train_instances=2, multi_label=True, sample_train=False
+            method=ADAPT_GENERATION,
+            model_deployment="openai/ada",
+            max_train_instances=2,
+            multi_label=True,
+            sample_train=False,
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         train_instances = [

src/helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py

@@ -15,7 +15,7 @@ def test_construct_language_modeling_prompt(self):
         adapter_spec = AdapterSpec(
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
-            model="openai/davinci",
+            model_deployment="openai/davinci",
             output_prefix="",
             max_tokens=0,
         )
@@ -38,7 +38,7 @@ def test_fits_tokens_within_context_window(self):
         adapter_spec = AdapterSpec(
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
-            model="openai/curie",
+            model_deployment="openai/curie",
             output_prefix="",
             max_tokens=0,
         )
@@ -69,7 +69,7 @@ def test_prompt_truncated(self):
         adapter_spec = AdapterSpec(
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
-            model="anthropic/claude-v1.3",
+            model_deployment="anthropic/claude-v1.3",
             output_prefix="",
             max_tokens=0,
         )
@@ -104,7 +104,7 @@ def test_prompt_truncated(self):
         adapter_spec_2_ = AdapterSpec(
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
-            model="anthropic/claude-v1.3",
+            model_deployment="anthropic/claude-v1.3",
             output_prefix="",
             max_tokens=2000,
         )

src/helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py

@@ -7,7 +7,9 @@
 
 class TestMultipleChoiceJointAdapter(TestAdapter):
     def test_sample_examples(self):
-        adapter_spec = AdapterSpec(method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", max_train_instances=4)
+        adapter_spec = AdapterSpec(
+            method=ADAPT_MULTIPLE_CHOICE_JOINT, model_deployment="openai/ada", max_train_instances=4
+        )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         all_train_instances = [
             Instance(Input(text="say no"), references=[Reference(Output(text="no"), tags=[CORRECT_TAG])]),
@@ -27,13 +29,17 @@ def test_sample_examples(self):
         assert examples[3].input.text == "say yes3"
 
     def test_sample_examples_no_train_instances(self):
-        adapter_spec = AdapterSpec(method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", max_train_instances=2)
+        adapter_spec = AdapterSpec(
+            method=ADAPT_MULTIPLE_CHOICE_JOINT, model_deployment="openai/ada", max_train_instances=2
+        )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         examples = adapter.sample_examples(all_train_instances=[], seed=0)
         assert len(examples) == 0
 
     def test_sample_examples_greater_max_train_instances(self):
-        adapter_spec = AdapterSpec(method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", max_train_instances=10)
+        adapter_spec = AdapterSpec(
+            method=ADAPT_MULTIPLE_CHOICE_JOINT, model_deployment="openai/ada", max_train_instances=10
+        )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         all_train_instances = [
             Instance(Input(text="say no"), references=[Reference(Output(text="no"), tags=[CORRECT_TAG])]),
@@ -46,7 +52,10 @@ def test_sample_examples_greater_max_train_instances(self):
 
     def test_multiple_correct_reference(self):
         adapter_spec = AdapterSpec(
-            method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", max_train_instances=10, sample_train=False
+            method=ADAPT_MULTIPLE_CHOICE_JOINT,
+            model_deployment="openai/ada",
+            max_train_instances=10,
+            sample_train=False,
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         train_instances = [
@@ -101,7 +110,7 @@ def test_multiple_correct_reference(self):
     def test_multiple_correct_reference_multi_label(self):
         adapter_spec = AdapterSpec(
             method=ADAPT_MULTIPLE_CHOICE_JOINT,
-            model="openai/ada",
+            model_deployment="openai/ada",
             max_train_instances=10,
             multi_label=True,
             sample_train=False,

src/helm/benchmark/config_registry.py

@@ -0,0 +1,14 @@
+from helm.benchmark.model_deployment_registry import register_deployments_if_not_already_registered
+from helm.benchmark.model_metadata_registry import register_metadatas_if_not_already_registered
+from helm.benchmark.tokenizer_config_registry import register_tokenizers_if_not_already_registered
+
+HELM_REGISTERED: bool = False
+
+
+def register_helm_configurations():
+    global HELM_REGISTERED
+    if not HELM_REGISTERED:
+        register_metadatas_if_not_already_registered()
+        register_tokenizers_if_not_already_registered()
+        register_deployments_if_not_already_registered()
+        HELM_REGISTERED = True

src/helm/benchmark/huggingface_registration.py

@@ -1,13 +1,22 @@
 import os
 from typing import Optional
+from datetime import date
 
 from helm.benchmark.model_deployment_registry import (
     ClientSpec,
     ModelDeployment,
     WindowServiceSpec,
     register_model_deployment,
 )
+from helm.benchmark.model_metadata_registry import (
+    get_model_metadata,
+    ModelMetadata,
+    register_model_metadata,
+    TEXT_MODEL_TAG,
+    FULL_FUNCTIONALITY_TEXT_MODEL_TAG,
+)
 from helm.benchmark.tokenizer_config_registry import TokenizerConfig, TokenizerSpec, register_tokenizer_config
+from helm.common.hierarchical_logger import hlog
 
 
 def register_huggingface_model(
@@ -30,6 +39,27 @@ def register_huggingface_model(
             args=object_spec_args,
         ),
     )
+
+    # We check if the model is already registered because we don't want to
+    # overwrite the model metadata if it's already registered.
+    # If it's not registered, we register it, as otherwise an error would be thrown
+    # when we try to register the model deployment.
+    try:
+        _ = get_model_metadata(model_name=helm_model_name)
+    except ValueError:
+        register_model_metadata(
+            ModelMetadata(
+                name=helm_model_name,
+                creator_organization_name="Unknown",
+                display_name=helm_model_name,
+                description=helm_model_name,
+                access="open",
+                release_date=date.today(),
+                tags=[TEXT_MODEL_TAG, FULL_FUNCTIONALITY_TEXT_MODEL_TAG],
+            )
+        )
+        hlog(f"Registered default metadata for model {helm_model_name}")
+
     register_model_deployment(model_deployment)
     tokenizer_config = TokenizerConfig(
         name=helm_model_name,