From 48d7080cbc2999ac3d57b87157919b2cdaa72bbb Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Wed, 12 Mar 2025 10:03:54 -0700 Subject: [PATCH] factor in diff review comments --- .../scoring/basic/scoring_fn/bfcl/__init__.py | 0 .../basic/scoring_fn/bfcl_scoring_fn.py | 4 +- .../scoring/basic/utils/bfcl/__init__.py | 5 ++ .../{scoring_fn => utils}/bfcl/ast_parser.py | 5 +- .../{scoring_fn => utils}/bfcl/checker.py | 58 ++----------------- .../{scoring_fn => utils}/bfcl/tree_sitter.py | 5 +- 6 files changed, 19 insertions(+), 58 deletions(-) delete mode 100644 llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/__init__.py create mode 100644 llama_stack/providers/inline/scoring/basic/utils/bfcl/__init__.py rename llama_stack/providers/inline/scoring/basic/{scoring_fn => utils}/bfcl/ast_parser.py (98%) rename llama_stack/providers/inline/scoring/basic/{scoring_fn => utils}/bfcl/checker.py (95%) rename llama_stack/providers/inline/scoring/basic/{scoring_fn => utils}/bfcl/tree_sitter.py (88%) diff --git a/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/__init__.py b/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl_scoring_fn.py b/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl_scoring_fn.py index 34bbc0157..4be5ac120 100644 --- a/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl_scoring_fn.py +++ b/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl_scoring_fn.py @@ -11,8 +11,8 @@ from llama_stack.apis.scoring_functions import ScoringFnParams from llama_stack.providers.utils.scoring.base_scoring_fn import RegisteredBaseScoringFn from .fn_defs.bfcl import bfcl -from .bfcl.ast_parser import decode_ast -from .bfcl.checker import ast_checker, is_empty_output +from ..utils.bfcl.ast_parser import decode_ast +from ..utils.bfcl.checker import ast_checker, is_empty_output import json import re diff --git a/llama_stack/providers/inline/scoring/basic/utils/bfcl/__init__.py b/llama_stack/providers/inline/scoring/basic/utils/bfcl/__init__.py new file mode 100644 index 000000000..756f351d8 --- /dev/null +++ b/llama_stack/providers/inline/scoring/basic/utils/bfcl/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. diff --git a/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/ast_parser.py b/llama_stack/providers/inline/scoring/basic/utils/bfcl/ast_parser.py similarity index 98% rename from llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/ast_parser.py rename to llama_stack/providers/inline/scoring/basic/utils/bfcl/ast_parser.py index 595dfc95a..ca53acb37 100644 --- a/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/ast_parser.py +++ b/llama_stack/providers/inline/scoring/basic/utils/bfcl/ast_parser.py @@ -1,5 +1,8 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. import ast from .tree_sitter import get_parser diff --git a/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/checker.py b/llama_stack/providers/inline/scoring/basic/utils/bfcl/checker.py similarity index 95% rename from llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/checker.py rename to llama_stack/providers/inline/scoring/basic/utils/bfcl/checker.py index 48abfcf15..320d0992d 100644 --- a/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/checker.py +++ b/llama_stack/providers/inline/scoring/basic/utils/bfcl/checker.py @@ -1,6 +1,8 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. - +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. import json import re import time @@ -18,47 +20,6 @@ class NoAPIKeyError(Exception): REAL_TIME_MATCH_ALLOWED_DIFFERENCE = 0.2 -UNDERSCORE_TO_DOT = [ - "gpt-4o-2024-08-06-FC", - "gpt-4o-2024-05-13-FC", - "gpt-4o-mini-2024-07-18-FC", - "gpt-4-turbo-2024-04-09-FC", - "gpt-4-1106-preview-FC", - "gpt-4-0125-preview-FC", - "gpt-4-0613-FC", - "gpt-3.5-turbo-0125-FC", - "claude-3-opus-20240229-FC", - "claude-3-sonnet-20240229-FC", - "claude-3-haiku-20240307-FC", - "claude-3-5-sonnet-20240620-FC", - "open-mistral-nemo-2407-FC-Any", - "open-mistral-nemo-2407-FC-Auto", - "open-mixtral-8x22b-FC-Any", - "open-mixtral-8x22b-FC-Auto", - "mistral-large-2407-FC", - "mistral-large-2407-FC-Any", - "mistral-large-2407-FC-Auto", - "mistral-small-2402-FC-Any", - "mistral-small-2402-FC-Auto", - "mistral-small-2402-FC", - "gemini-1.0-pro", - "gemini-1.5-pro-preview-0409", - "gemini-1.5-pro-preview-0514", - "gemini-1.5-flash-preview-0514", - "meetkai/functionary-small-v3.1-FC", - "meetkai/functionary-small-v3.2-FC", - "meetkai/functionary-medium-v3.1-FC", - "NousResearch/Hermes-2-Pro-Llama-3-8B", - "NousResearch/Hermes-2-Pro-Llama-3-70B", - "NousResearch/Hermes-2-Pro-Mistral-7B", - "NousResearch/Hermes-2-Theta-Llama-3-8B", - "NousResearch/Hermes-2-Theta-Llama-3-70B", - "command-r-plus-FC", - "command-r-plus-FC-optimized", - "THUDM/glm-4-9b-chat", - "ibm-granite/granite-20b-functioncalling", - "yi-large-fc", -] JAVA_TYPE_CONVERSION = { "byte": int, @@ -133,16 +94,6 @@ def get_possible_answer_type(possible_answer: list): return None -def convert_func_name(function_name, model_name: str): - model_name_escaped = model_name.replace("_", "/") - if "." in function_name: - if model_name_escaped in UNDERSCORE_TO_DOT: - # OAI does not support "." in the function name so we replace it with "_". ^[a-zA-Z0-9_-]{1,64}$ is the regex for the name. - # This happens for OpenAI, Mistral, and Google models - return re.sub(r"\.", "_", function_name) - return function_name - - def type_checker( param: str, value, @@ -401,7 +352,6 @@ def simple_function_checker( "error_type": "simple_function_checker:unclear", } - func_name = convert_func_name(func_name, model_name) # Check if function name matches if func_name not in model_output: diff --git a/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/tree_sitter.py b/llama_stack/providers/inline/scoring/basic/utils/bfcl/tree_sitter.py similarity index 88% rename from llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/tree_sitter.py rename to llama_stack/providers/inline/scoring/basic/utils/bfcl/tree_sitter.py index 607287f29..ed97ee360 100644 --- a/llama_stack/providers/inline/scoring/basic/scoring_fn/bfcl/tree_sitter.py +++ b/llama_stack/providers/inline/scoring/basic/utils/bfcl/tree_sitter.py @@ -1,5 +1,8 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# This software may be used and distributed according to the terms of the Llama 2 Community License Agreement. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. """ Tree-sitter changes its API with unfortunate frequency. Modules that need it should