chris/MiniGPT-4
1
0
Fork 0
mirror of https://github.com/Vision-CAIR/MiniGPT-4.git synced 2025-04-04 01:50:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

update evaluation

Browse Source
This commit is contained in:
junchen14 2023-11-01 11:39:18 +03:00
parent 76cad1ebfd
commit 45c07e5603
5 changed files with 33 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
eval_configs/minigptv2_benchmark_evaluation.yaml
View File

@ -25,61 +25,53 @@ evaluation_datasets:
refcoco:
eval_file_path: /ibex/project/c2133/minigpt4_v2_dataset
img_path: /ibex/ai/reference/CV/COCO/cocoapi/data/2014/images/jpeg/train
save_path: results/refcoco.json
max_new_tokens: 20
batch_size: 10
refcocog:
eval_file_path: /path/to/eval/annotation/path
img_path: /path/to/eval/image/path
save_path: /path/to/save/path
max_new_tokens: 20
batch_size: 10
refcoco+:
eval_file_path: /path/to/eval/annotation/path
img_path: /path/to/eval/image/path
save_path: /path/to/save/path
max_new_tokens: 20
batch_size: 10
gqa:
eval_file_path: /path/to/eval/annotation/path
img_path: /path/to/eval/image/path
save_path: /path/to/save/path
max_new_tokens: 20
batch_size: 10
okvqa:
eval_file_path: /path/to/eval/annotation/path
img_path: /path/to/eval/image/path
save_path: /path/to/save/path
eval_file_path: /ibex/project/c2133/blip_dataset/ok_vqa_copy
img_path: /ibex/ai/reference/CV/COCO/cocoapi/data/2017/images/jpeg/train
max_new_tokens: 20
batch_size: 10
vizwiz:
eval_file_path: /path/to/eval/annotation/path
img_path: /path/to/eval/image/path
save_path: /path/to/save/path
eval_file_path: /ibex/project/c2133/minigpt4_v2_dataset/vizwiz/val.json
img_path: /ibex/project/c2133/minigpt4_v2_dataset/vizwiz/val
max_new_tokens: 20
batch_size: 10
iconvqa:
eval_file_path: /path/to/eval/annotation/path
img_path: /path/to/eval/image/path
save_path: /path/to/save/path
max_new_tokens: 20
batch_size: 10
vsr:
eval_file_path: cambridgeltl/vsr_zeroshot
img_path: /path/to/eval/image/path
save_path: /path/to/save/path
max_new_tokens: 20
batch_size: 10
hm:
eval_file_path: /path/to/eval/annotation/path
img_path: /path/to/eval/image/path
save_path: /path/to/save/path
eval_file_path: /ibex/project/c2133/minigpt4_v2_dataset/hateful_meme/dev.jsonl
img_path: /ibex/project/c2133/minigpt4_v2_dataset/hateful_meme
max_new_tokens: 20
batch_size: 10
run:
task: image_text_pretrain
name: minigptv2_evaluation
save_path: /path/to/save/folder_path

2
eval_configs/minigptv2_eval.yaml
View File

@ -3,7 +3,7 @@ model:
model_type: pretrain
max_txt_len: 500
end_sym: "</s>"
low_resource: False
low_resource: True
prompt_template: '[INST] {} [/INST]'
llama_model: "/ibex/project/c2133/llama_v2/llama-2-7b-chat-pytorch_update"
ckpt: ""

10
eval_scripts/EVAL_README.md
View File

@ -61,9 +61,9 @@ export PYTHONPATH=$PYTHONPATH:/path/to/directory/of/MiniGPT-4
Set **llama_model** to the path of LLaMA model.
Set **ckpt** to the path of our pretrained model.
Set **eval_file_path** to the path of the annotation files for the evaluation data.
Set **img_path** to the path of the images.
Set **save_path** to the path of saving evaluation output.
Set **eval_file_path** to the path of the annotation files for each evaluation data.
Set **img_path** to the img_path for each evaluation dataset.
Set **save_path** to the save_path for evch evaluation dataset.
in [minigpt4/eval_configs/minigptv2_benchmark_evaluation.yaml](../minigpt4/eval_configs/minigptv2_benchmark_evaluation.yaml)
@ -80,7 +80,7 @@ dataset names:
```
torchrun --master-port ${port} --nproc_per_node 1 eval_ref.py \
--cfg-path ${cfg_path} --dataset dataset_name --resample
--cfg-path ${cfg_path} --dataset refcoco,refcoco+,refcocog --resample
```
@ -96,7 +96,7 @@ dataset names:
```
torchrun --master-port ${port} --nproc_per_node 1 eval_vqa.py \
--cfg-path ${cfg_path} --dataset dataset_name
--cfg-path ${cfg_path} --dataset okvqa,vizwiz,iconvqa,gqa,vsr,hm
```

7
eval_scripts/eval_ref.py
View File

@ -39,6 +39,7 @@ conv_temp.system = ""
#
model.eval()
save_path = cfg.run_cfg.save_path
@ -49,7 +50,6 @@ for dataset in args.dataset:
img_path = cfg.evaluation_datasets_cfg[dataset]["img_path"]
batch_size = cfg.evaluation_datasets_cfg[dataset]["batch_size"]
max_new_tokens = cfg.evaluation_datasets_cfg[dataset]["max_new_tokens"]
save_path = cfg.evaluation_datasets_cfg[dataset]["save_path"]
with open(os.path.join(eval_file_path,f"{dataset}/{dataset}_{split}.json"), 'r') as f:
refcoco = json.load(f)
@ -87,8 +87,9 @@ for dataset in args.dataset:
if len(resamples) == 0:
break
with open(os.path.join(save_path,f"{args.dataset}_{split}.json"),'w') as f:
file_save_path = os.path.join(save_path,f"{args.dataset}_{split}.json")
with open(file_save_path,'w') as f:
json.dump(minigpt4_predict, f)
count=0

34
eval_scripts/eval_vqa.py
View File

@ -35,7 +35,7 @@ model, vis_processor = init_model(args)
conv_temp = CONV_VISION_minigptv2.copy()
conv_temp.system = ""
model.eval()
save_path = cfg.run_cfg.save_path
if 'okvqa' in args.dataset:
@ -44,10 +44,10 @@ if 'okvqa' in args.dataset:
img_path = cfg.evaluation_datasets_cfg["okvqa"]["img_path"]
batch_size = cfg.evaluation_datasets_cfg["okvqa"]["batch_size"]
max_new_tokens = cfg.evaluation_datasets_cfg["okvqa"]["max_new_tokens"]
save_path = cfg.evaluation_datasets_cfg["okvqa"]["save_path"]
# evaluation_annntation_path = os.path.join(eval_file_path, "okvqa_test_split.json")
with open(eval_file_path) as f:
evaluation_annntation_path = os.path.join(eval_file_path, "okvqa_test_split.json")
with open(evaluation_annntation_path) as f:
ok_vqa_test_split = json.load(f)
data = OKVQAEvalData(ok_vqa_test_split, vis_processor, img_path)
@ -65,14 +65,15 @@ if 'okvqa' in args.dataset:
result['question_id'] = int(question_id)
minigpt4_predict.append(result)
with open(save_path,'w') as f:
file_save_path= os.join(save_path,"okvqa.json")
with open(file_save_path,'w') as f:
json.dump(minigpt4_predict, f)
annFile = os.path.join(eval_file_path,"mscoco_val2014_annotations_clean.json")
quesFile = os.path.join(eval_file_path,"OpenEnded_mscoco_val2014_questions_clean.json" )
vqa = VQA(annFile, quesFile)
vqaRes = vqa.loadRes(save_path, quesFile)
vqaRes = vqa.loadRes(file_save_path, quesFile)
vqaEval = VQAEval(vqa, vqaRes, n=2)
vqaEval.evaluate()
@ -84,7 +85,6 @@ if 'vizwiz' in args.dataset:
img_path = cfg.evaluation_datasets_cfg["vizwiz"]["img_path"]
batch_size = cfg.evaluation_datasets_cfg["vizwiz"]["batch_size"]
max_new_tokens = cfg.evaluation_datasets_cfg["vizwiz"]["max_new_tokens"]
save_path = cfg.evaluation_datasets_cfg["vizwiz"]["save_path"]
vizwiz = json.load(open(eval_file_path, 'r'))
@ -109,7 +109,8 @@ if 'vizwiz' in args.dataset:
acc = min(count/3.0, 1.0)
total_acc.append(acc)
with open(save_path,'w') as f:
file_save_path = os.path.join(save_path, "vizwiz.json")
with open(file_save_path,'w') as f:
json.dump(minigpt4_predict, f)
print('vizwiz Acc: ', np.average(total_acc)* 100.0, flush=True)
@ -120,8 +121,6 @@ if 'iconvqa' in args.dataset:
img_path = cfg.evaluation_datasets_cfg["iconvqa"]["img_path"]
batch_size = cfg.evaluation_datasets_cfg["iconvqa"]["batch_size"]
max_new_tokens = cfg.evaluation_datasets_cfg["iconvqa"]["max_new_tokens"]
save_path = cfg.evaluation_datasets_cfg["iconvqa"]["save_path"]
iconqa_text_val = json.load(open(eval_file_path,"r"))
@ -150,7 +149,6 @@ if 'gqa' in args.dataset:
img_path = cfg.evaluation_datasets_cfg["gqa"]["img_path"]
batch_size = cfg.evaluation_datasets_cfg["gqa"]["batch_size"]
max_new_tokens = cfg.evaluation_datasets_cfg["gqa"]["max_new_tokens"]
save_path = cfg.evaluation_datasets_cfg["gqa"]["save_path"]
gqa = json.load(open(eval_file_path))
data = GQAEvalData(gqa, vis_processor, img_path)
@ -172,7 +170,8 @@ if 'gqa' in args.dataset:
total+=1
print('gqa val:', count / total * 100, flush=True)
with open(save_path,'w') as f:
file_save_path = os.path.join(save_path, "gqa.json")
with open(file_save_path,'w') as f:
json.dump(minigpt4_predict, f)
if 'vsr' in args.dataset:
@ -180,7 +179,6 @@ if 'vsr' in args.dataset:
img_path = cfg.evaluation_datasets_cfg["vsr"]["img_path"]
batch_size = cfg.evaluation_datasets_cfg["vsr"]["batch_size"]
max_new_tokens = cfg.evaluation_datasets_cfg["vsr"]["max_new_tokens"]
save_path = cfg.evaluation_datasets_cfg["vsr"]["save_path"]
annotation = load_dataset("cambridgeltl/vsr_zeroshot", split='test')
data = VSREvalData(annotation, vis_processor, img_path)
@ -203,7 +201,8 @@ if 'vsr' in args.dataset:
count+=1
total+=1
print('vsr test:', count / total * 100, flush=True)
with open(save_path,'w') as f:
file_save_path = os.path.join(save_path,"vsr.json")
with open(file_save_path,'w') as f:
json.dump(minigpt4_predict, f)
if 'hm' in args.dataset:
@ -212,7 +211,6 @@ if 'hm' in args.dataset:
img_path = cfg.evaluation_datasets_cfg["hm"]["img_path"]
batch_size = cfg.evaluation_datasets_cfg["hm"]["batch_size"]
max_new_tokens = cfg.evaluation_datasets_cfg["hm"]["max_new_tokens"]
save_path = cfg.evaluation_datasets_cfg["hm"]["save_path"]
annotation = []
with open(eval_file_path, 'r') as jsonl_file:
@ -247,8 +245,8 @@ if 'hm' in args.dataset:
if answer == label:
count+=1
total+=1
print('hm val:', count / total * 100, flush=True)
with open(save_path,'w') as f:
file_save_path = os.path.join(save_path, "hm.json")
with open(file_save_path,'w') as f:
json.dump(minigpt4_predict, f)