mirror of
https://github.com/Vision-CAIR/MiniGPT-4.git
synced 2025-04-17 11:10:46 +00:00
61 lines
1.7 KiB
Python
Executable File
61 lines
1.7 KiB
Python
Executable File
"""
|
|
Copyright (c) 2022, salesforce.com, inc.
|
|
All rights reserved.
|
|
SPDX-License-Identifier: BSD-3-Clause
|
|
For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
|
|
"""
|
|
|
|
import os
|
|
import json
|
|
|
|
from PIL import Image
|
|
|
|
from minigpt4.datasets.datasets.vqa_datasets import VQADataset
|
|
|
|
from collections import OrderedDict
|
|
import random
|
|
|
|
class __DisplMixin:
|
|
def displ_item(self, index):
|
|
sample, ann = self.__getitem__(index), self.annotation[index]
|
|
|
|
return OrderedDict(
|
|
{
|
|
"file": ann["image"],
|
|
"question": ann["question"],
|
|
"question_id": ann["question_id"],
|
|
"answers": "; ".join(ann["answer"]),
|
|
"image": sample["image"],
|
|
}
|
|
)
|
|
|
|
|
|
class GQADataset(VQADataset, __DisplMixin):
|
|
def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
|
|
super().__init__(vis_processor, text_processor, vis_root, ann_paths)
|
|
self.instruction_pool =[
|
|
"[vqa] {}",
|
|
"[vqa] Based on the image, respond to this question with a short answer: {}"
|
|
]
|
|
|
|
def __getitem__(self, index):
|
|
ann = self.annotation[index]
|
|
|
|
image_path = os.path.join(self.vis_root, ann["image"])
|
|
image = Image.open(image_path).convert("RGB")
|
|
|
|
image = self.vis_processor(image)
|
|
question = self.text_processor(ann["question"])
|
|
|
|
instruction = random.choice(self.instruction_pool).format(question)
|
|
instruction = "<Img><ImageHere></Img> {} ".format(instruction)
|
|
|
|
answers = self.text_processor(ann["answer"])
|
|
|
|
return {
|
|
"image": image,
|
|
"instruction_input": instruction,
|
|
"answer": answers,
|
|
}
|
|
|