mirror of
https://github.com/Vision-CAIR/MiniGPT-4.git
synced 2025-04-04 18:10:47 +00:00
185 lines
6.6 KiB
Python
185 lines
6.6 KiB
Python
import argparse
|
|
import os
|
|
import random
|
|
import requests
|
|
from io import BytesIO
|
|
|
|
import numpy as np
|
|
from PIL import Image
|
|
import torch
|
|
import torch.backends.cudnn as cudnn
|
|
import gradio as gr
|
|
|
|
import minigpt4.tasks as tasks
|
|
from minigpt4.common.config import Config
|
|
from minigpt4.common.dist_utils import get_rank, init_distributed_mode
|
|
from minigpt4.common.logger import setup_logger
|
|
from minigpt4.common.optims import (
|
|
LinearWarmupCosineLRScheduler,
|
|
LinearWarmupStepLRScheduler,
|
|
)
|
|
from minigpt4.common.registry import registry
|
|
from minigpt4.common.utils import now
|
|
|
|
# imports modules for registration
|
|
from minigpt4.datasets.builders import *
|
|
from minigpt4.models import *
|
|
from minigpt4.processors import *
|
|
from minigpt4.runners import *
|
|
from minigpt4.tasks import *
|
|
|
|
|
|
parser = argparse.ArgumentParser(description="Demo")
|
|
parser.add_argument("--cfg-path", required=False, default='textvqa_eval.yaml', help="path to configuration file.")
|
|
parser.add_argument("--ckpt_path", required=False, help="path to configuration file.")
|
|
parser.add_argument("--lora_r", type=int, default=64, help="path to configuration file.")
|
|
parser.add_argument("--lora_alpha", type=int, default=16, help="path to configuration file.")
|
|
parser.add_argument("--name", type=str)
|
|
parser.add_argument("--img_path", type=str)
|
|
parser.add_argument("--eval_file_path", type=str)
|
|
parser.add_argument(
|
|
"--options",
|
|
nargs="+",
|
|
help="override some settings in the used config, the key-value pair "
|
|
"in xxx=yyy format will be merged into config file (deprecate), "
|
|
"change to --cfg-options instead.",
|
|
)
|
|
|
|
|
|
print('Initializing Chat')
|
|
args = parser.parse_args()
|
|
cfg = Config(args)
|
|
|
|
ckpt_list = [args.ckpt_path]
|
|
|
|
print('evaluating config:', args.cfg_path)
|
|
print('evaluating checkpoint:', args.ckpt_path)
|
|
|
|
|
|
|
|
import json
|
|
import numpy as np
|
|
|
|
f = open(f'{args.eval_file_path}/textocr/TextOCR_0.1_val.json', 'r')
|
|
textcap_dataset_val = json.load(f)
|
|
|
|
all_img_ids = list(textcap_dataset_val['imgs'].keys())
|
|
ann_ids = list(textcap_dataset_val['anns'].keys())
|
|
|
|
|
|
for ckpt in ckpt_list:
|
|
cfg.model_cfg.ckpt = ckpt
|
|
cfg.model_cfg.lora_r=args.lora_r
|
|
cfg.model_cfg.lora_alpha=args.lora_alpha
|
|
|
|
model_config = cfg.model_cfg
|
|
model_cls = registry.get_model_class(model_config.arch)
|
|
model = model_cls.from_config(model_config).to('cuda:0')
|
|
model.eval()
|
|
print('Initialization Finished')
|
|
|
|
vis_processor_cfg = cfg.datasets_cfg.coco_vqa.vis_processor.eval
|
|
vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg)
|
|
|
|
text_processor_cfg = cfg.datasets_cfg.coco_vqa.text_processor.eval
|
|
text_processor = registry.get_processor_class(text_processor_cfg.name).from_config(text_processor_cfg)
|
|
task = tasks.setup_task(cfg)
|
|
|
|
from collections import Counter
|
|
|
|
def find_most_frequent_item(items):
|
|
counter = Counter(items)
|
|
most_common = counter.most_common(1)
|
|
if most_common:
|
|
return most_common[0][0]
|
|
else:
|
|
return None
|
|
from minigpt4.datasets.datasets.vqa_datasets import VQADataset, VQAEvalDataset
|
|
|
|
class textVQAEvalDataset(VQADataset):
|
|
def __init__(self, vis_processor, text_processor, vis_root=None, ann_paths=None, use_ocr=False):
|
|
# super().__init__(vis_processor, text_processor, vis_root, ann_paths)
|
|
|
|
from datasets import load_dataset
|
|
self.annotation = load_dataset("textvqa", split="validation",cache_dir=f"{args.eval_file_path}/textvqa_cache")
|
|
self.vis_processor = vis_processor
|
|
self.text_processor = text_processor
|
|
self.instruction_pool = [
|
|
'[vqa] Question: {} Short answer:',
|
|
]
|
|
|
|
self.use_ocr = use_ocr
|
|
|
|
|
|
def __getitem__(self, index):
|
|
ann = self.annotation[index]
|
|
image_path = ann['flickr_original_url']
|
|
image = ann["image"].convert("RGB")
|
|
|
|
img_id = ann['image_id']
|
|
sel_anns = [textcap_dataset_val['anns'][ann] for ann in ann_ids if ann.startswith(img_id)]
|
|
sel_caps = [ann['utf8_string'] for ann in sel_anns]
|
|
sel_caps = [cap for cap in sel_caps if cap not in ['\n', '.']]
|
|
captions = " ".join(sel_caps)
|
|
|
|
image = self.vis_processor(image)
|
|
question = self.text_processor(ann["question"])
|
|
captions = self.text_processor(captions)
|
|
|
|
instruction = random.choice(self.instruction_pool).format(question)
|
|
|
|
if self.use_ocr and img_id in all_img_ids:
|
|
instruction = "<s>[INST] <Img><ImageHere></Img> OCR tokens: {}. {} [/INST]".format(captions, instruction)
|
|
else:
|
|
instruction = "<s>[INST] <Img><ImageHere></Img> {} [/INST]".format(instruction)
|
|
|
|
|
|
answers = find_most_frequent_item(ann["answers"])
|
|
|
|
return {
|
|
"image": image,
|
|
"text_input": question,
|
|
"answer": answers,
|
|
'image_path': image_path,
|
|
"instruction_input": instruction,
|
|
"question_id": ann["question_id"],
|
|
# "instance_id": ann["instance_id"],
|
|
}
|
|
|
|
dataset = textVQAEvalDataset(vis_processor, text_processor, use_ocr=cfg.run_cfg.use_ocr)
|
|
dataloader = torch.utils.data.DataLoader(dataset, batch_size=cfg.run_cfg.batch_size_eval, num_workers=cfg.run_cfg.num_workers)
|
|
|
|
result_dir = cfg.run_cfg.output_dir
|
|
if not os.path.exists(result_dir):
|
|
os.makedirs(result_dir)
|
|
|
|
epoch = cfg.model_cfg.ckpt.split('/')[-1].split('.')[0].split('_')[1]
|
|
exp_id = cfg.model_cfg.ckpt.split('/')[-2] + "_" + epoch
|
|
|
|
val_result = task.evaluation(model, dataloader, cuda_enabled=True)
|
|
task.after_evaluation(val_result, split_name=f'{args.name}_textvqa_val', result_dir=result_dir)
|
|
|
|
|
|
anno_files = f'{args.eval_file_path}/TextVQA/gt_ann.json'
|
|
ques_files = f'{args.eval_file_path}/TextVQA/gt_ques.json'
|
|
|
|
from minigpt4.common.vqa_tools.vqa import VQA
|
|
from minigpt4.common.vqa_tools.vqa_eval import VQAEval
|
|
import logging, json, os
|
|
|
|
result_file = '{}/{}_textvqa_val.json'.format(result_dir, args.name)
|
|
vqa = VQA(anno_files, ques_files)
|
|
vqa_result = vqa.loadRes(
|
|
resFile=result_file, quesFile=ques_files
|
|
)
|
|
|
|
# create vqaEval object by taking vqa and vqaRes
|
|
# n is precision of accuracy (number of places after decimal), default is 2
|
|
vqa_scorer = VQAEval(vqa, vqa_result, n=2)
|
|
logging.info("Start VQA evaluation.")
|
|
vqa_scorer.evaluate()
|
|
|
|
# print accuracies
|
|
overall_acc = vqa_scorer.accuracy["overall"]
|
|
print('textvqa val acc: ', overall_acc, flush=True)
|