diff --git a/eval_configs/minigptv2_benchmark_evaluation.yaml b/eval_configs/minigptv2_benchmark_evaluation.yaml index c7c738b..4fcb91b 100644 --- a/eval_configs/minigptv2_benchmark_evaluation.yaml +++ b/eval_configs/minigptv2_benchmark_evaluation.yaml @@ -25,61 +25,53 @@ evaluation_datasets: refcoco: eval_file_path: /ibex/project/c2133/minigpt4_v2_dataset img_path: /ibex/ai/reference/CV/COCO/cocoapi/data/2014/images/jpeg/train - save_path: results/refcoco.json max_new_tokens: 20 batch_size: 10 refcocog: eval_file_path: /path/to/eval/annotation/path img_path: /path/to/eval/image/path - save_path: /path/to/save/path max_new_tokens: 20 batch_size: 10 refcoco+: eval_file_path: /path/to/eval/annotation/path img_path: /path/to/eval/image/path - save_path: /path/to/save/path max_new_tokens: 20 batch_size: 10 gqa: eval_file_path: /path/to/eval/annotation/path img_path: /path/to/eval/image/path - save_path: /path/to/save/path max_new_tokens: 20 batch_size: 10 okvqa: - eval_file_path: /path/to/eval/annotation/path - img_path: /path/to/eval/image/path - save_path: /path/to/save/path + eval_file_path: /ibex/project/c2133/blip_dataset/ok_vqa_copy + img_path: /ibex/ai/reference/CV/COCO/cocoapi/data/2017/images/jpeg/train max_new_tokens: 20 batch_size: 10 vizwiz: - eval_file_path: /path/to/eval/annotation/path - img_path: /path/to/eval/image/path - save_path: /path/to/save/path + eval_file_path: /ibex/project/c2133/minigpt4_v2_dataset/vizwiz/val.json + img_path: /ibex/project/c2133/minigpt4_v2_dataset/vizwiz/val max_new_tokens: 20 batch_size: 10 iconvqa: eval_file_path: /path/to/eval/annotation/path img_path: /path/to/eval/image/path - save_path: /path/to/save/path max_new_tokens: 20 batch_size: 10 vsr: eval_file_path: cambridgeltl/vsr_zeroshot img_path: /path/to/eval/image/path - save_path: /path/to/save/path max_new_tokens: 20 batch_size: 10 hm: - eval_file_path: /path/to/eval/annotation/path - img_path: /path/to/eval/image/path - save_path: /path/to/save/path + eval_file_path: /ibex/project/c2133/minigpt4_v2_dataset/hateful_meme/dev.jsonl + img_path: /ibex/project/c2133/minigpt4_v2_dataset/hateful_meme max_new_tokens: 20 batch_size: 10 run: task: image_text_pretrain name: minigptv2_evaluation + save_path: /path/to/save/folder_path diff --git a/eval_configs/minigptv2_eval.yaml b/eval_configs/minigptv2_eval.yaml index 00c21ed..1fa5d7b 100644 --- a/eval_configs/minigptv2_eval.yaml +++ b/eval_configs/minigptv2_eval.yaml @@ -3,7 +3,7 @@ model: model_type: pretrain max_txt_len: 500 end_sym: "" - low_resource: False + low_resource: True prompt_template: '[INST] {} [/INST]' llama_model: "/ibex/project/c2133/llama_v2/llama-2-7b-chat-pytorch_update" ckpt: "" diff --git a/eval_scripts/EVAL_README.md b/eval_scripts/EVAL_README.md index 7db1997..d12e436 100644 --- a/eval_scripts/EVAL_README.md +++ b/eval_scripts/EVAL_README.md @@ -61,9 +61,9 @@ export PYTHONPATH=$PYTHONPATH:/path/to/directory/of/MiniGPT-4 Set **llama_model** to the path of LLaMA model. Set **ckpt** to the path of our pretrained model. -Set **eval_file_path** to the path of the annotation files for the evaluation data. -Set **img_path** to the path of the images. -Set **save_path** to the path of saving evaluation output. +Set **eval_file_path** to the path of the annotation files for each evaluation data. +Set **img_path** to the img_path for each evaluation dataset. +Set **save_path** to the save_path for evch evaluation dataset. in [minigpt4/eval_configs/minigptv2_benchmark_evaluation.yaml](../minigpt4/eval_configs/minigptv2_benchmark_evaluation.yaml) @@ -80,7 +80,7 @@ dataset names: ``` torchrun --master-port ${port} --nproc_per_node 1 eval_ref.py \ - --cfg-path ${cfg_path} --dataset dataset_name --resample + --cfg-path ${cfg_path} --dataset refcoco,refcoco+,refcocog --resample ``` @@ -96,7 +96,7 @@ dataset names: ``` torchrun --master-port ${port} --nproc_per_node 1 eval_vqa.py \ - --cfg-path ${cfg_path} --dataset dataset_name + --cfg-path ${cfg_path} --dataset okvqa,vizwiz,iconvqa,gqa,vsr,hm ``` diff --git a/eval_scripts/eval_ref.py b/eval_scripts/eval_ref.py index b538d9c..28d55a2 100644 --- a/eval_scripts/eval_ref.py +++ b/eval_scripts/eval_ref.py @@ -39,6 +39,7 @@ conv_temp.system = "" # model.eval() +save_path = cfg.run_cfg.save_path @@ -49,7 +50,6 @@ for dataset in args.dataset: img_path = cfg.evaluation_datasets_cfg[dataset]["img_path"] batch_size = cfg.evaluation_datasets_cfg[dataset]["batch_size"] max_new_tokens = cfg.evaluation_datasets_cfg[dataset]["max_new_tokens"] - save_path = cfg.evaluation_datasets_cfg[dataset]["save_path"] with open(os.path.join(eval_file_path,f"{dataset}/{dataset}_{split}.json"), 'r') as f: refcoco = json.load(f) @@ -87,8 +87,9 @@ for dataset in args.dataset: if len(resamples) == 0: break - - with open(os.path.join(save_path,f"{args.dataset}_{split}.json"),'w') as f: + + file_save_path = os.path.join(save_path,f"{args.dataset}_{split}.json") + with open(file_save_path,'w') as f: json.dump(minigpt4_predict, f) count=0 diff --git a/eval_scripts/eval_vqa.py b/eval_scripts/eval_vqa.py index 68ca951..9e40c43 100644 --- a/eval_scripts/eval_vqa.py +++ b/eval_scripts/eval_vqa.py @@ -35,7 +35,7 @@ model, vis_processor = init_model(args) conv_temp = CONV_VISION_minigptv2.copy() conv_temp.system = "" model.eval() - +save_path = cfg.run_cfg.save_path if 'okvqa' in args.dataset: @@ -44,10 +44,10 @@ if 'okvqa' in args.dataset: img_path = cfg.evaluation_datasets_cfg["okvqa"]["img_path"] batch_size = cfg.evaluation_datasets_cfg["okvqa"]["batch_size"] max_new_tokens = cfg.evaluation_datasets_cfg["okvqa"]["max_new_tokens"] - save_path = cfg.evaluation_datasets_cfg["okvqa"]["save_path"] + - # evaluation_annntation_path = os.path.join(eval_file_path, "okvqa_test_split.json") - with open(eval_file_path) as f: + evaluation_annntation_path = os.path.join(eval_file_path, "okvqa_test_split.json") + with open(evaluation_annntation_path) as f: ok_vqa_test_split = json.load(f) data = OKVQAEvalData(ok_vqa_test_split, vis_processor, img_path) @@ -65,14 +65,15 @@ if 'okvqa' in args.dataset: result['question_id'] = int(question_id) minigpt4_predict.append(result) - with open(save_path,'w') as f: + file_save_path= os.join(save_path,"okvqa.json") + with open(file_save_path,'w') as f: json.dump(minigpt4_predict, f) annFile = os.path.join(eval_file_path,"mscoco_val2014_annotations_clean.json") quesFile = os.path.join(eval_file_path,"OpenEnded_mscoco_val2014_questions_clean.json" ) vqa = VQA(annFile, quesFile) - vqaRes = vqa.loadRes(save_path, quesFile) + vqaRes = vqa.loadRes(file_save_path, quesFile) vqaEval = VQAEval(vqa, vqaRes, n=2) vqaEval.evaluate() @@ -84,7 +85,6 @@ if 'vizwiz' in args.dataset: img_path = cfg.evaluation_datasets_cfg["vizwiz"]["img_path"] batch_size = cfg.evaluation_datasets_cfg["vizwiz"]["batch_size"] max_new_tokens = cfg.evaluation_datasets_cfg["vizwiz"]["max_new_tokens"] - save_path = cfg.evaluation_datasets_cfg["vizwiz"]["save_path"] vizwiz = json.load(open(eval_file_path, 'r')) @@ -109,7 +109,8 @@ if 'vizwiz' in args.dataset: acc = min(count/3.0, 1.0) total_acc.append(acc) - with open(save_path,'w') as f: + file_save_path = os.path.join(save_path, "vizwiz.json") + with open(file_save_path,'w') as f: json.dump(minigpt4_predict, f) print('vizwiz Acc: ', np.average(total_acc)* 100.0, flush=True) @@ -120,8 +121,6 @@ if 'iconvqa' in args.dataset: img_path = cfg.evaluation_datasets_cfg["iconvqa"]["img_path"] batch_size = cfg.evaluation_datasets_cfg["iconvqa"]["batch_size"] max_new_tokens = cfg.evaluation_datasets_cfg["iconvqa"]["max_new_tokens"] - save_path = cfg.evaluation_datasets_cfg["iconvqa"]["save_path"] - iconqa_text_val = json.load(open(eval_file_path,"r")) @@ -150,7 +149,6 @@ if 'gqa' in args.dataset: img_path = cfg.evaluation_datasets_cfg["gqa"]["img_path"] batch_size = cfg.evaluation_datasets_cfg["gqa"]["batch_size"] max_new_tokens = cfg.evaluation_datasets_cfg["gqa"]["max_new_tokens"] - save_path = cfg.evaluation_datasets_cfg["gqa"]["save_path"] gqa = json.load(open(eval_file_path)) data = GQAEvalData(gqa, vis_processor, img_path) @@ -172,7 +170,8 @@ if 'gqa' in args.dataset: total+=1 print('gqa val:', count / total * 100, flush=True) - with open(save_path,'w') as f: + file_save_path = os.path.join(save_path, "gqa.json") + with open(file_save_path,'w') as f: json.dump(minigpt4_predict, f) if 'vsr' in args.dataset: @@ -180,7 +179,6 @@ if 'vsr' in args.dataset: img_path = cfg.evaluation_datasets_cfg["vsr"]["img_path"] batch_size = cfg.evaluation_datasets_cfg["vsr"]["batch_size"] max_new_tokens = cfg.evaluation_datasets_cfg["vsr"]["max_new_tokens"] - save_path = cfg.evaluation_datasets_cfg["vsr"]["save_path"] annotation = load_dataset("cambridgeltl/vsr_zeroshot", split='test') data = VSREvalData(annotation, vis_processor, img_path) @@ -203,7 +201,8 @@ if 'vsr' in args.dataset: count+=1 total+=1 print('vsr test:', count / total * 100, flush=True) - with open(save_path,'w') as f: + file_save_path = os.path.join(save_path,"vsr.json") + with open(file_save_path,'w') as f: json.dump(minigpt4_predict, f) if 'hm' in args.dataset: @@ -212,7 +211,6 @@ if 'hm' in args.dataset: img_path = cfg.evaluation_datasets_cfg["hm"]["img_path"] batch_size = cfg.evaluation_datasets_cfg["hm"]["batch_size"] max_new_tokens = cfg.evaluation_datasets_cfg["hm"]["max_new_tokens"] - save_path = cfg.evaluation_datasets_cfg["hm"]["save_path"] annotation = [] with open(eval_file_path, 'r') as jsonl_file: @@ -247,8 +245,8 @@ if 'hm' in args.dataset: if answer == label: count+=1 total+=1 - + print('hm val:', count / total * 100, flush=True) - - with open(save_path,'w') as f: + file_save_path = os.path.join(save_path, "hm.json") + with open(file_save_path,'w') as f: json.dump(minigpt4_predict, f)