get multiple providers working for meta-reference (for inference + safety)

This commit is contained in:
Ashwin Bharambe 2024-11-04 16:33:42 -08:00
parent 60800bc09b
commit 6c7ea6e904
10 changed files with 136 additions and 95 deletions

View file

@ -73,16 +73,20 @@ def pytest_addoption(parser):
def pytest_generate_tests(metafunc):
if "inference_model" in metafunc.fixturenames:
metafunc.parametrize(
"inference_model",
[pytest.param(metafunc.config.getoption("--inference-model"), id="")],
indirect=True,
)
safety_model = metafunc.config.getoption("--safety-model")
if "safety_model" in metafunc.fixturenames:
metafunc.parametrize(
"safety_model",
[pytest.param(metafunc.config.getoption("--safety-model"), id="")],
[pytest.param(safety_model, id="")],
indirect=True,
)
if "inference_model" in metafunc.fixturenames:
inference_model = metafunc.config.getoption("--inference-model")
models = list(set({inference_model, safety_model}))
metafunc.parametrize(
"inference_model",
[pytest.param(models, id="")],
indirect=True,
)
if "agents_stack" in metafunc.fixturenames: