Skip to content

Commit

Permalink
[Update] Update bunnyllama3
Browse files Browse the repository at this point in the history
Update bunnyllama3.py
  • Loading branch information
junming-yang committed Jul 17, 2024
2 parents a32622a + f141a3e commit aae54e6
Showing 1 changed file with 5 additions and 8 deletions.
13 changes: 5 additions & 8 deletions vlmeval/vlm/bunnyllama3.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ def __init__(self, model_path='BAAI/Bunny-v1_1-Llama-3-8B-V', **kwargs):
self.kwargs = kwargs

def use_custom_prompt(self, dataset):
if listinstr(['MCQ', 'Y/N'], DATASET_TYPE(dataset)) or listinstr(
['mathvista', 'ocrvqa', 'textvqa', 'chartqa', 'docvqa'], dataset.lower()):
if listinstr(['MCQ', 'Y/N'], DATASET_TYPE(dataset)) or listinstr(['mathvista'], dataset.lower()):
return True
else:
return False
Expand All @@ -42,7 +41,7 @@ def build_prompt(self, line, dataset):
prompt = line['question']

if DATASET_TYPE(dataset) == 'MCQ':
if listinstr(['mmmu'], dataset):
if listinstr(['mmmu'], dataset.lower()):
hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None
assert hint is None

Expand Down Expand Up @@ -82,18 +81,18 @@ def build_prompt(self, line, dataset):
options_prompt += f'{key}. {item}\n'

prompt += question + options_prompt
if listinstr(['cn', 'ccbench'], dataset):
if listinstr(['cn', 'ccbench'], dataset.lower()):
prompt += '请直接回答选项字母。'
else:
prompt += "Answer with the option's letter from the given choices directly."
elif DATASET_TYPE(dataset) == 'Y/N':
if listinstr(['mme'], dataset):
if listinstr(['mme'], dataset.lower()):
if not listinstr(
['code_reasoning', 'commonsense_reasoning', 'numerical_calculation', 'text_translation'],
line['category']):
prompt = prompt.replace(' Please answer yes or no.',
'\nAnswer the question using a single word or phrase.')
elif listinstr(['pope'], dataset):
elif listinstr(['pope'], dataset.lower()):
prompt = prompt.replace(' Please answer yes or no.',
'\nAnswer the question using a single word or phrase.')
elif listinstr(['mathvista'], dataset.lower()):
Expand All @@ -103,8 +102,6 @@ def build_prompt(self, line, dataset):
if match.group(4) is not None:
prompt += '\n' + match.group(4).rstrip('\n')
prompt += '\n' + match.group(1)
elif listinstr(['ocrvqa', 'textvqa', 'chartqa', 'docvqa'], dataset.lower()):
prompt += '\nAnswer the question using a single word or phrase.'
else:
raise ValueError(
f"Bunny doesn't implement a custom prompt for {dataset}. It should use the default prompt, but didn't.")
Expand Down

0 comments on commit aae54e6

Please sign in to comment.