chris/MiniGPT-4
1
0
Fork 0
mirror of https://github.com/Vision-CAIR/MiniGPT-4.git synced 2025-04-04 18:10:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

0401 cls cross/route/gate/route (universal)

Browse Source
This commit is contained in:
wanghanzi 2024-04-01 12:15:47 +08:00
parent 2057032a63
commit 4631c70d76
60 changed files with 1207 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
minigpt4/datasets/datasets/aok_vqa_datasets.py
View File

@ -89,8 +89,8 @@ class AOKVQADataset(VQADataset, __DisplMixin):
return {
"image": data['image'],
"image_id": data["image_id"],
# "q_input": q_input,
"q_input": llm_input,
"q_input": q_input,
# "q_input": llm_input,
"llm_input": llm_input,
"text_input": question,
"text_output": answer,
@ -164,8 +164,8 @@ class AOKVQAEvalDataset(VQAEvalDataset, __DisplMixin):
"correct_choice_idx": correct_choice_idx_list,
"direct_answers": direct_answers_list,
"llm_input": llm_input_list,
"q_input": llm_input_list,
# "q_input": q_input_list,
# "q_input": llm_input_list,
"q_input": q_input_list,
"gt_answers": gt_answers_list,
"source": source_list,
}
@ -208,13 +208,13 @@ class AOKVQAEvalDataset(VQAEvalDataset, __DisplMixin):
correct_choice_idx = None
correct_answer = direct_answers
llm_input = ann.get("llm_input",random.choice(self.instruction_pool).format(question))
llm_input = ann.get("llm_input",random.choice(self.instruction_pool).format(question, ", ".join(choices)))
# llm_input = random.choice(self.instruction_pool).format(question, ", ".join(choices))
return {
"image": image,
# "q_input": question,
"q_input": llm_input,
"q_input": question,
# "q_input": llm_input,
"llm_input": llm_input,
"text_input": question,
"question_id": ann["question_id"],

6
minigpt4/datasets/datasets/coco_vqa_datasets.py
View File

@ -96,8 +96,8 @@ class COCOVQADataset(VQADataset, __DisplMixin):
"image": data['image'],
"image_id": data["image_id"],
"question_id": data["question_id"],
# "q_input": q_input,
"q_input": llm_input,
"q_input": q_input,
# "q_input": llm_input,
"llm_input": llm_input,
"text_input": question,
"text_output": answer,
@ -181,7 +181,7 @@ class COCOVQAEvalDataset(VQAEvalDataset, __DisplMixin):
"question_id": ann["question_id"],
# "instance_id": ann["instance_id"],
# "question": question,
"q_input": llm_input,
# "q_input": llm_input,
"q_input": q_input,
"llm_input": llm_input,
"text_input": question,

8
minigpt4/datasets/datasets/gqa_datasets.py
View File

@ -75,8 +75,8 @@ class GQADataset(VQADataset, __DisplMixin):
# "text_output": ann["fullAnswer"],
"text_output": answers,
# "instruction_input": instruction,
"q_input": llm_input,
# "q_input": q_input,
# "q_input": llm_input,
"q_input": q_input,
"llm_input": llm_input,
"gt_answers": answers,
"source": "gqa",
@ -150,8 +150,8 @@ class GQAEvalDataset(VQAEvalDataset, __DisplMixin):
"gt_answers": answer,
"fullAnswer": fullAnswer,
"text_output": answer,
# "q_input": q_input,
"q_input": llm_input,
"q_input": q_input,
# "q_input": llm_input,
"llm_input": llm_input,
"question_id": ann["question_id"],
# "instance_id": ann["instance_id"],

8
minigpt4/datasets/datasets/ok_vqa_datasets.py
View File

@ -98,8 +98,8 @@ class OKVQADataset(VQADataset, __DisplMixin):
"image_id": data["image_id"],
"question_id": data["question_id"],
# "instruction_input": instruction,
"q_input": llm_input,
# "q_input": q_input,
# "q_input": llm_input,
"q_input": q_input,
"llm_input": llm_input,
"text_input": question,
"text_output": answer,
@ -181,8 +181,8 @@ class OKVQAEvalDataset(VQAEvalDataset, __DisplMixin):
'image_path': image_path,
"question_id": ann["question_id"],
"question": question,
# "q_input": q_input,
"q_input": llm_input,
"q_input": q_input,
# "q_input": llm_input,
"llm_input": llm_input,
"text_input": question,
"source": 'okvqa',

15
minigpt4/eval_scripts/eval_vqa.py
View File

@ -29,10 +29,21 @@ def list_of_str(arg):
def eval_parser():
parser = argparse.ArgumentParser(description="Demo")
parser.add_argument(
"--device",
default=0,
help="device to run the model",
)
parser.add_argument(
"--cfg-path",
default="/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/minigpt4/projects/qformer_moe_vicuna/eval/vqa_benchmark_evaluation.yaml",
help="path to configuration file.")
parser.add_argument(
"--dataset",
default=['vizwiz','hm'],
type=list_of_str,
help="dataset to evaluate",
)
parser.add_argument(
"--options",
nargs="+",
@ -58,7 +69,6 @@ def init_model(cfg, device):
model_cls = registry.get_model_class(model_config.arch)
model = model_cls.from_config(model_config).to(device)
# import pudb; pudb.set_trace()
key = list(cfg.datasets_cfg.keys())[0]
vis_processor_cfg = cfg.datasets_cfg.get(key).vis_processor.train
vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg)
@ -68,13 +78,12 @@ def init_model(cfg, device):
return model, vis_processor, text_processor
parser = eval_parser()
parser.add_argument("--dataset", type=list_of_str, default=['vizwiz','hm'], help="dataset to evaluate")
args = parser.parse_args()
cfg = Config(args)
setup_seeds(cfg)
print(cfg._convert_node_to_json(cfg.config))
setup_logger()
device = torch.device("cuda:5" if torch.cuda.is_available() else "cpu")
device = torch.device("cuda:{}".format(args.device) if torch.cuda.is_available() else "cpu")
model, vis_processor, _ = init_model(cfg, device)
model.eval()

2
minigpt4/models/QformerRouteMoECLS.py
View File

@ -530,7 +530,7 @@ class BertLayer(nn.Module):
self.seq_len_dim,
attention_output[:, query_length:, :],
)
cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size]
cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size]
# add moe query ffn
moe_ffn_attention_input = query_attention_output[:, :query_length, :] # [bz, query_length+seq_len, 768]

2
minigpt4/models/QformerRouteMoECLSLN.py
View File

@ -524,7 +524,7 @@ class BertLayer(nn.Module):
self.seq_len_dim,
attention_output[:, query_length:, :],
)
cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size]
cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size]
# add moe query ffn
moe_ffn_attention_input = query_attention_output[:, :query_length, :] # [bz, query_length+seq_len, 768]

2
minigpt4/models/QformerRouteMoELNUni.py
View File

@ -524,7 +524,7 @@ class BertLayer(nn.Module):
self.seq_len_dim,
attention_output[:, query_length:, :],
)
cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size]
cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size]
# add moe query ffn
# query_attention_output size: [bz, query_length+seq_len, 768]

2
minigpt4/models/QformerRouteMoEUni.py
View File

@ -531,7 +531,7 @@ class BertLayer(nn.Module):
self.seq_len_dim,
attention_output[:, query_length:, :],
)
cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size]
cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size]
# add moe query ffn
# query_attention_output size: [bz, query_length+seq_len, 768]

59
minigpt4/models/moe/beam_search_universal.py
View File

@ -22,7 +22,7 @@ class UniRouteMoELayer(nn.Module):
self.route_method = route_method
if self.route_method == "pre-route-uni":
self.gate = nn.Linear(hidden_size, num_experts, bias=False).float()
elif self.route_method in ["post-route-uni"]:
elif self.route_method in ["post-route-uni",'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
gate = nn.Linear(hidden_size, 1, bias=False).float()
self.gate = gate
@ -38,7 +38,7 @@ class UniRouteMoELayer(nn.Module):
def beam_search(self, current_scores_log, beam_scores, expert_route, batch_size):
if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni']:
if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni','uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
# current_scores_log torch.Size([bz, num_experts-1])
assert beam_scores==None and expert_route==None
current_scores = torch.exp(current_scores_log)
@ -143,7 +143,32 @@ class UniRouteMoELayer(nn.Module):
# import pdb;pdb.set_trace()
return candidate_output, beam_scores, expert_route, beam_idx, importance_loss
def forward_post_route_uni(self, x, beam_scores, expert_route, use_log=True):
def calculate_cls_gate_score(self, cls_hidden, output_x):
if self.route_method == 'uni-cls-route':
# cls_hidden = [bz, 768]
gate_score = self.gate(cls_hidden) # bz, 1
elif self.route_method == 'uni-cls-query-route': # add cls_hiddin on query_token mean pool hidden
mean_output = torch.mean(output_x, dim=1) # bz, 768
gate_score = self.gate(mean_output+cls_hidden) # bz, 1
elif self.route_method == 'uni-cls-cross-route':
# cls_hidden as Q, output_x as K, V calculate scaled dot-product attention between Q and K and V
# cls_hidden: bz, 768
# output_x: bz, 32, 768
Q = cls_hidden.unsqueeze(1) # bz, 1, 768
K = output_x # bz, 32, 768
V = output_x # bz, 32, 768
# scaled dot-product attention
QK = torch.matmul(Q, K.transpose(-1, -2)) / (K.size(-1) ** 0.5) # bz, 1, 32
QK = F.softmax(QK, dim=-1) # bz, 1, 32
gate_score = torch.matmul(QK, V) # bz, 1, 768
gate_score = gate_score.squeeze(1) # bz, 768
gate_score = self.gate(gate_score) # bz, 1
return gate_score
def forward_route_uni(self, x, beam_scores, expert_route, use_log=True, cls_hidden=None):
if beam_scores == None:
batch_size = x.shape[0]
@ -155,8 +180,6 @@ class UniRouteMoELayer(nn.Module):
x_masked, x_uniexpert = x[select_expert],x[select_universal]
num_tokens = x.shape[1]
import pdb; pdb.set_trace()
def forward_expert(input_x, expert_idx):
output_x = self.experts[expert_idx].forward(input_x)
return output_x
@ -168,8 +191,14 @@ class UniRouteMoELayer(nn.Module):
logits_gate_lst = list()
for expert_idx in range(self.num_route_experts): # num_expert-1
output_x = forward_expert(x_masked, expert_idx)
output_x_aver = torch.mean(output_x, dim=1)
gate_score = self.gate(output_x_aver)
if self.route_method == 'post-route-uni':
output_x_aver = torch.mean(output_x, dim=1)
gate_score = self.gate(output_x_aver)
elif self.route_method in ['uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route'] and cls_hidden is not None:
gate_score = self.calculate_cls_gate_score(cls_hidden, output_x)
logits_gate_lst.append(gate_score)
outputs.append(output_x.unsqueeze(0))
@ -186,14 +215,12 @@ class UniRouteMoELayer(nn.Module):
# beam_scores torch.Size([bz*(num_beam-1)]), expert_route torch.Size([bz*(num_beam-1), layer_n])
current_select_expert = expert_route[:,-1] # torch.Size([bz*(num_beam-1)])
import pdb; pdb.set_trace()
if self.layer_judge == 'first':
replicated_tensor = candidate_output_raw.unsqueeze(2).expand(self.num_route_experts, batch_size, self.num_route_beam, num_tokens, self.hidden_size)
candidate_output_raw = replicated_tensor.contiguous().view(self.num_route_experts, -1, num_tokens, self.hidden_size) # [bz*num_beams, 32,768]
current_scores_t = current_scores.unsqueeze(1).expand(batch_size, self.num_route_beam, self.num_route_experts)
current_scores = current_scores_t.contiguous().view(-1, self.num_route_experts) # [bz*(num_beams-1), num_experts-1]
import pdb; pdb.set_trace()
candidate_output = candidate_output_raw.permute(1, 0, 2, 3)[beam_idx] # torch.Size([8, 2, 32, 768])
expert_select_matrix = F.one_hot(current_select_expert, self.num_route_experts)
if self.weight_type == 'ffn_prob':
@ -203,8 +230,6 @@ class UniRouteMoELayer(nn.Module):
output = candidate_output * expert_select_matrix.unsqueeze(-1).unsqueeze(-1)
experts_output = torch.sum(output, dim=1) # [bz*num_beams-1, 32, 768]
import pdb; pdb.set_trace()
####################
### universal expert
####################
@ -220,26 +245,26 @@ class UniRouteMoELayer(nn.Module):
output.append(combine_tmp)
final_output = torch.cat(output) # [bz*num_beam, 32 ,768]
import pdb; pdb.set_trace()
# import pdb; pdb.set_trace()
return final_output, beam_scores, expert_route, beam_idx, importance_loss
def forward(self, x, attention_mask, beam_scores, expert_route, use_log=True):
def forward(self, x, attention_mask, beam_scores, expert_route, cls_hidden):
"""
if first_layer: x [bz, 32, 768]
else: x [bz*num_beams, 32, 768]
"""
if self.route_method == 'pre-route-uni':
candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_pre_route(x, beam_scores, expert_route, use_log=True)
elif self.route_method in ['post-route-uni']:
candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_post_route_uni(x, beam_scores, expert_route, use_log=True)
elif self.route_method in ['post-route-uni', 'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_route_uni(x, beam_scores, expert_route, use_log=True, cls_hidden=cls_hidden)
import pdb;pdb.set_trace()
return candidate_output, beam_scores, expert_route, beam_idx, importance_loss
if __name__ == '__main__':
import sys
@ -305,7 +330,7 @@ if __name__ == '__main__':
num_experts=config.moebert_expert_num,
num_beams=config.moebert_num_beams,
layer_judge = layer_judge,
route_method = "post-route-uni",
route_method = "uni-cls-cross-rout[e",
weight_type="ffn_prob"
)
layer_output = experts_post(x2, None, beam_scores2, expert_route2, False)

47
minigpt4/models/moe/uniroute_moe_layer.py
View File

@ -19,9 +19,9 @@ class UniRouteMoELayer(nn.Module):
self.weight_type = weight_type
self.route_method = route_method
if self.route_method == "pre-route-uni":
self.gate = nn.Linear(hidden_size, num_experts, bias=False).float()
elif self.route_method in ["post-route-uni"]:
if self.route_method in ['pre-route-uni', 'uni-cls-gate-route']:
self.gate = nn.Linear(hidden_size, self.num_route_experts, bias=False).float()
elif self.route_method in ["post-route-uni",'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
gate = nn.Linear(hidden_size, 1, bias=False).float()
self.gate = gate
@ -35,9 +35,8 @@ class UniRouteMoELayer(nn.Module):
# Compute coefficient of variation (i.e. std/mean) squared.
return (std_importance_per_expert / mean_importance_per_expert)**2
def beam_search(self, current_scores_log, beam_scores, expert_route, batch_size):
if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni']:
if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni','uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route','uni-cls-gate-route']:
# current_scores_log torch.Size([bz, num_experts-1])
assert beam_scores==None and expert_route==None
current_scores = torch.exp(current_scores_log)
@ -80,7 +79,6 @@ class UniRouteMoELayer(nn.Module):
return beam_scores, expert_route, beam_idx
def forward_gate(self, x):
"""
TODO: Pre forward gate
@ -166,8 +164,16 @@ class UniRouteMoELayer(nn.Module):
gate_score = self.gate(gate_score) # bz, 1
return gate_score
def adjust_cls_hidden(self, cls_hidden, output_x):
if cls_hidden.shape[0]/self.num_beams == output_x.shape[0]/self.num_route_beam:
cls_hidden_lst = list()
for i in range(cls_hidden.shape[0]):
if i % self.num_beams != 0:
cls_hidden_lst.append(cls_hidden[i,:])
cls_hidden = torch.stack(cls_hidden_lst)
return cls_hidden
def forward_route_uni(self, x, beam_scores, expert_route, use_log=True, cls_hidden=None):
def forward_route_uni(self, x, beam_scores, expert_route, cls_hidden=None):
if beam_scores == None:
batch_size = x.shape[0]
@ -186,6 +192,9 @@ class UniRouteMoELayer(nn.Module):
####################
### route expert
####################
if cls_hidden is not None:
cls_hidden = self.adjust_cls_hidden(cls_hidden, x_masked)
outputs = list()
logits_gate_lst = list()
for expert_idx in range(self.num_route_experts): # num_expert-1
@ -194,20 +203,24 @@ class UniRouteMoELayer(nn.Module):
if self.route_method == 'post-route-uni':
output_x_aver = torch.mean(output_x, dim=1)
gate_score = self.gate(output_x_aver)
logits_gate_lst.append(gate_score)
elif self.route_method in ['uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route'] and cls_hidden is not None:
gate_score = self.calculate_cls_gate_score(cls_hidden, output_x)
logits_gate_lst.append(gate_score)
logits_gate_lst.append(gate_score)
outputs.append(output_x.unsqueeze(0))
candidate_output_raw = torch.cat(outputs) # torch.Size([num_expert-1, bz*(num_beam-1), 32, 768])
logits_gate = torch.cat(logits_gate_lst,dim=1)# torch.Size([bz*(num_beam-1), num_expert-1])
current_scores = F.softmax(logits_gate, dim=-1) # torch.Size([bz*(num_beam-1), num_expert-1])
if use_log:
current_scores_log = torch.log(current_scores) # 取log之后可以直接相加 torch.Size([bz*(num_beam-1), num_expert-1])
if self.route_method == 'uni-cls-gate-route':
# universal expert with cls_hidden state into nn.Linear(768,num_experts-1)
logits_gate = self.gate(cls_hidden)
current_scores = F.softmax(logits_gate, dim=-1) # torch.Size([bz*(num_beam-1), num_expert-1])
else:
current_scores_log = current_scores
logits_gate = torch.cat(logits_gate_lst,dim=1)# torch.Size([bz*(num_beam-1), num_expert-1])
current_scores = F.softmax(logits_gate, dim=-1) # torch.Size([bz*(num_beam-1), num_expert-1])
current_scores_log = torch.log(current_scores) # 取log之后可以直接相加 torch.Size([bz*(num_beam-1), num_expert-1])
importance_loss = self._importance_auxiliary_loss(current_scores)
beam_scores, expert_route, beam_idx = self.beam_search(current_scores_log, beam_scores, expert_route, current_scores_log.shape[0])
@ -254,9 +267,9 @@ class UniRouteMoELayer(nn.Module):
else: x [bz*num_beams, 32, 768]
"""
if self.route_method == 'pre-route-uni':
candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_pre_route(x, beam_scores, expert_route, use_log=True)
elif self.route_method in ['post-route-uni', 'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_route_uni(x, beam_scores, expert_route, use_log=True, cls_hidden=cls_hidden)
candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_pre_route(x, beam_scores, expert_route)
elif self.route_method in ['post-route-uni', 'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route','uni-cls-gate-route']:
candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_route_uni(x, beam_scores, expert_route, cls_hidden=cls_hidden)
return candidate_output, beam_scores, expert_route, beam_idx, importance_loss

143
minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval.yaml Normal file
View File

@ -0,0 +1,143 @@
# Copyright (c) 2022, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
model:
arch: blip2_vicuna_instruct
model_type: vicuna7b_pretrain
load_pretrained: True
load_finetuned: True
vit_model: eva_clip_g
pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
finetuned: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/checkpoint_7.pth"
q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
# vit encoder
image_size: 224
drop_path_rate: 0
use_grad_checkpoint: False
vit_precision: "fp16"
# Q-Former
num_query_token: 32
qformer_text_input: True
# vicuna
llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1"
prompt: ""
max_txt_len: 256
max_output_txt_len: 256
# freeze
freeze_vit: True
freeze_llm: True
freeze_qformer: False
freeze_t5_proj: False
# moe
general_version: 'uni_route_moe'
moebert_route_method: 'uni-cls-cross-route'
moebert_load_balance: 0.01
moebert_expert_num: 4
moebert_num_beams: 4
moe_weight_type: 'ffn_prob'
use_balance_loss: False
ln_position: "out"
datasets:
# gqa:
# type: balanced_sft_raw_eval
# batch_size: 32
# vis_processor:
# eval:
# name: "blip2_image_eval"
# image_size: 224
# text_processor:
# eval:
# name: "blip_caption"
# coco_vqa: # 658104
# type: vqa_v2_eval
# batch_size: 32
# vis_processor:
# eval:
# name: "blip2_image_eval"
# image_size: 224
# text_processor:
# eval:
# name: "blip_caption"
coco_caption: # 414113 train
type: coco_cap_eval
batch_size: 32
vis_processor:
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
eval:
name: "blip_caption"
# ok_vqa: # train, valid (9009, 5046)
# type: ok_vqa_eval
# batch_size: 32
# vis_processor:
# eval:
# name: "blip2_image_eval"
# image_size: 224
# text_processor:
# eval:
# name: "blip_caption"
# aok_vqa: # train: 17056, val: 1145
# batch_size: 32
# vis_processor:
# eval:
# name: "blip2_image_eval"
# image_size: 224
# text_processor:
# eval:
# name: "blip_caption"
textcaps_caption: # train: 109765, val: 15830
batch_size: 32
vis_processor:
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
eval:
name: "blip_caption"
run:
task: instruction_tuning
# optimizer
lr_sched: "linear_warmup_cosine_lr"
init_lr: 5e-5
min_lr: 1e-6
warmup_lr: 1e-6
log_freq: 5
save_freq: 1500
weight_decay: 0.05
max_epoch: 10
num_workers: 4
warmup_steps: 600
iters_per_epoch: 3000
seed: 42
output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/eval/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/"
amp: True
resume_ckpt_path: null
evaluate: True
test_splits: ["val"]
# test_splits: ["test"]
device: "cuda"
world_size: 1
dist_url: "env://"
distributed: True

113
minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval1.yaml Normal file
View File

@ -0,0 +1,113 @@
# Copyright (c) 2022, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
model:
arch: blip2_vicuna_instruct
model_type: vicuna7b_pretrain
load_pretrained: True
load_finetuned: True
vit_model: eva_clip_g
pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
finetuned: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/checkpoint_7.pth"
q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
# vit encoder
image_size: 224
drop_path_rate: 0
use_grad_checkpoint: False
vit_precision: "fp16"
# Q-Former
num_query_token: 32
qformer_text_input: True
# vicuna
llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1"
prompt: ""
max_txt_len: 256
max_output_txt_len: 256
# freeze
freeze_vit: True
freeze_llm: True
freeze_qformer: False
freeze_t5_proj: False
# moe
general_version: 'uni_route_moe'
moebert_route_method: 'uni-cls-cross-route'
moebert_load_balance: 0.01
moebert_expert_num: 4
moebert_num_beams: 4
moe_weight_type: 'ffn_prob'
use_balance_loss: False
ln_position: "out"
datasets:
coco_caption: # 414113 train
type: coco_cap_eval
batch_size: 32
vis_processor:
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
eval:
name: "blip_caption"
textcaps_caption: # train: 109765, val: 15830
batch_size: 32
vis_processor:
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
eval:
name: "blip_caption"
evaluation_datasets:
vizwiz:
eval_file_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/VizWiz/val.json
img_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/VizWiz/val
max_new_tokens: 20
batch_size: 32
hm:
eval_file_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/hm_data/dev_seen.jsonl
img_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/hm_data/
max_new_tokens: 20
batch_size: 32
run:
task: instruction_tuning
# optimizer
lr_sched: "linear_warmup_cosine_lr"
init_lr: 5e-5
min_lr: 1e-6
warmup_lr: 1e-6
log_freq: 5
save_freq: 1500
weight_decay: 0.05
max_epoch: 10
num_workers: 4
warmup_steps: 600
iters_per_epoch: 3000
seed: 42
output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/eval/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/"
amp: True
resume_ckpt_path: null
evaluate: True
# test_splits: ["val"]
test_splits: ["test"]
device: "cuda"
world_size: 1
dist_url: "env://"
distributed: True

89
minigpt4/projects/qformer_moe_cls_vicuna/eval/mix_vqa_coco_vicuna_eval_coco_vqa_test.yaml Normal file
View File

@ -0,0 +1,89 @@
# Copyright (c) 2022, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
model:
arch: blip2_vicuna_instruct
model_type: vicuna7b_pretrain
load_pretrained: True
load_finetuned: True
vit_model: eva_clip_g
pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
finetuned: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/checkpoint_7.pth"
q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
# vit encoder
image_size: 224
drop_path_rate: 0
use_grad_checkpoint: False
vit_precision: "fp16"
# Q-Former
num_query_token: 32
qformer_text_input: True
# vicuna
llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1"
prompt: ""
max_txt_len: 256
max_output_txt_len: 256
# freeze
freeze_vit: True
freeze_llm: True
freeze_qformer: False
freeze_t5_proj: False
# moe
general_version: 'uni_route_moe'
moebert_route_method: 'uni-cls-cross-route'
moebert_load_balance: 0.01
moebert_expert_num: 4
moebert_num_beams: 4
moe_weight_type: 'ffn_prob'
use_balance_loss: False
ln_position: "out"
datasets:
coco_vqa: # 658104
type: vqa_v2_eval
batch_size: 48
vis_processor:
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
eval:
name: "blip_caption"
run:
task: instruction_tuning
# optimizer
lr_sched: "linear_warmup_cosine_lr"
init_lr: 5e-5
min_lr: 1e-6
warmup_lr: 1e-6
log_freq: 5
save_freq: 1500
weight_decay: 0.05
max_epoch: 10
num_workers: 4
warmup_steps: 600
iters_per_epoch: 3000
seed: 42
output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/eval/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/"
amp: True
resume_ckpt_path: null
evaluate: True
test_splits: ["test"]
device: "cuda"
world_size: 1
dist_url: "env://"
distributed: True

11
minigpt4/projects/qformer_moe_route_universal_vicuna/train/mix6_qformer_moe_route_uni_blip2_vicuna7b_0328.yaml → minigpt4/projects/qformer_moe_cls_vicuna/train/mix6_qformer_moe_route_cls_blip2_vicuna7b_0329.yaml
View File

@ -35,9 +35,10 @@ model:
freeze_t5_proj: False
# moe
# general_version: 'cls_route_moe'
general_version: 'uni_route_moe'
moebert_route_method: "post-route-uni"
moebert_load_balance: 0.05
moebert_route_method: 'uni-cls-cross-route'
moebert_load_balance: 0.01
moebert_expert_num: 3
moebert_num_beams: 3
moe_weight_type: 'ffn_prob'
@ -154,13 +155,13 @@ run:
save_freq: 1500
weight_decay: 0.05
max_epoch: 8
max_epoch: 10
num_workers: 4
warmup_steps: 600
iters_per_epoch: 5000
seed: 42
output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0328/"
output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330/"
amp: True
resume_ckpt_path: null
@ -170,7 +171,7 @@ run:
valid_splits: ["val"]
wandb_log: False
# job_name: mix6_uni_route_post_5e5_3ex3b_005_10ep_0319
# job_name: mix6_uni_cls_route_post_5e5_3ex3b_005_8ep_0330
device: "cuda"
world_size: 1

180
minigpt4/projects/qformer_moe_route_universal_vicuna/train/mix6_qformer_moe_route_uni_blip2_vicuna7b_0329.yaml Normal file
View File

@ -0,0 +1,180 @@
# Copyright (c) 2022, salesforce.com, inc.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
model:
arch: blip2_vicuna_instruct
model_type: vicuna7b_pretrain
load_pretrained: True
load_finetuned: False
vit_model: eva_clip_g
pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
# vit encoder
image_size: 224
drop_path_rate: 0
use_grad_checkpoint: False
vit_precision: "fp16"
# Q-Former
num_query_token: 32
qformer_text_input: True
# vicuna7b
llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1"
prompt: ""
max_txt_len: 256
max_output_txt_len: 256
# freeze
freeze_vit: True
freeze_llm: True
freeze_qformer: False
freeze_t5_proj: False
# moe
general_version: 'base'
# general_version: 'uni_route_moe'
# moebert_route_method: "post-route-uni"
# moebert_load_balance: 0.05
# moebert_expert_num: 3
# moebert_num_beams: 3
# moe_weight_type: 'ffn_prob'
# use_balance_loss: False
# ln_position: "out"
datasets:
gqa: # train: 943000, 12578, 12578)
type: balanced_sft_raw
batch_size: 32
vis_processor:
train:
name: "blip2_image_train"
image_size: 224
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
train:
name: "blip_caption"
eval:
name: "blip_caption"
sample_ratio: 10
ok_vqa: # train, valid (9009, 5046)
batch_size: 32
vis_processor:
train:
name: "blip2_image_train"
image_size: 224
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
train:
name: "blip_caption"
eval:
name: "blip_caption"
sample_ratio: 1
coco_vqa: # 658104
batch_size: 32
vis_processor:
train:
name: "blip2_image_train"
image_size: 224
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
train:
name: "blip_caption"
eval:
name: "blip_caption"
sample_ratio: 9
coco_caption: # 414113 train
batch_size: 32
vis_processor:
train:
name: "blip2_image_train"
image_size: 224
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
train:
name: "blip_caption"
eval:
name: "blip_caption"
sample_ratio: 7
aok_vqa: # train: 17056, val: 1145
batch_size: 32
vis_processor:
train:
name: "blip2_image_train"
image_size: 224
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
train:
name: "blip_caption"
eval:
name: "blip_caption"
sample_ratio: 2
# sample_ratio: 1
textcaps_caption: # train: 109765, val: 15830
batch_size: 32
vis_processor:
train:
name: "blip2_image_train"
image_size: 224
eval:
name: "blip2_image_eval"
image_size: 224
text_processor:
train:
name: "blip_caption"
eval:
name: "blip_caption"
# sample_ratio: 3
sample_ratio: 4
run:
task: instruction_tuning
lr_sched: "linear_warmup_cosine_lr"
init_lr: 5e-5
min_lr: 1e-6
warmup_lr: 1e-6
log_freq: 5
save_freq: 1500
weight_decay: 0.05
max_epoch: 8
num_workers: 4
warmup_steps: 600
iters_per_epoch: 5000
seed: 40
# output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0328/"
output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_base_lr5e5_textinqf_8epo_0329/"
amp: True
resume_ckpt_path: null
evaluate: False
train_splits: ["train"]
valid_splits: ["val"]
wandb_log: False
# job_name: mix6_uni_route_post_5e5_3ex3b_005_10ep_0319
device: "cuda"
world_size: 1
dist_url: "env://"
distributed: True

11
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
56.320559707425666,54.43,73.75,71.61572052401746,64.02907005786079
58.936237875655905,56.23,75.25,71.09170305676857,65.37698523310611
60.99538877404993,55.3,76.4,73.01310043668123,66.42712230268279
61.146446175862614,58.62,76.58,73.1004366812227,67.36172071427133
62.5934170774368,58.5,77.5,73.88646288209607,68.11996998988322
62.41850850691684,57.47,78.24,73.7117903930131,67.96007472498249
62.99888694546033,58.08,78.88,74.235807860262,68.54867370143059
63.77802512323104,58.72,79.34,74.06113537117905,68.97479012360252
63.74622356495468,58.5,79.51,74.235807860262,68.99800785630417
63.825727460645574,58.41,79.63,74.41048034934498,69.06905195249763
1 gqa okvqa vqav2 aokvqa avg
2 56.320559707425666 54.43 73.75 71.61572052401746 64.02907005786079
3 58.936237875655905 56.23 75.25 71.09170305676857 65.37698523310611
4 60.99538877404993 55.3 76.4 73.01310043668123 66.42712230268279
5 61.146446175862614 58.62 76.58 73.1004366812227 67.36172071427133
6 62.5934170774368 58.5 77.5 73.88646288209607 68.11996998988322
7 62.41850850691684 57.47 78.24 73.7117903930131 67.96007472498249
8 62.99888694546033 58.08 78.88 74.235807860262 68.54867370143059
9 63.77802512323104 58.72 79.34 74.06113537117905 68.97479012360252
10 63.74622356495468 58.5 79.51 74.235807860262 68.99800785630417
11 63.825727460645574 58.41 79.63 74.41048034934498 69.06905195249763

8
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,8 @@
gqa,okvqa,vqav2,aokvqa,avg
57.40976307839084,53.92,74.0,71.26637554585152,64.14903465606059
60.136746700588326,56.03,75.27,71.70305676855895,65.78495086728682
60.99538877404993,56.27,76.52,73.27510917030567,66.7651244860889
61.94943552234059,57.39,76.95,74.06113537117905,67.58764272337991
62.55366512959135,58.17,77.83,73.53711790393012,68.02269575838037
63.26920019080935,57.8,78.7,75.54585152838428,68.8287629297984
63.46000954046749,58.15,79.09,74.58515283842794,68.82129059472385
1 gqa okvqa vqav2 aokvqa avg
2 57.40976307839084 53.92 74.0 71.26637554585152 64.14903465606059
3 60.136746700588326 56.03 75.27 71.70305676855895 65.78495086728682
4 60.99538877404993 56.27 76.52 73.27510917030567 66.7651244860889
5 61.94943552234059 57.39 76.95 74.06113537117905 67.58764272337991
6 62.55366512959135 58.17 77.83 73.53711790393012 68.02269575838037
7 63.26920019080935 57.8 78.7 75.54585152838428 68.8287629297984
8 63.46000954046749 58.15 79.09 74.58515283842794 68.82129059472385

7
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,7 @@
gqa,okvqa,vqav2,aokvqa,avg
56.948640483383684,53.97,73.92,71.17903930131004,64.00441994617344
60.27985371283193,54.86,75.51,71.79039301310044,65.6100616814831
61.138495786293525,56.33,76.33,72.66375545851528,66.61556281120221
61.631419939577036,58.13,76.89,73.1877729257642,67.45979821633532
62.513913181745906,57.35,77.96,72.5764192139738,67.60008309892993
63.126093178565746,57.36,78.63,74.14847161572052,68.31614119857156
1 gqa okvqa vqav2 aokvqa avg
2 56.948640483383684 53.97 73.92 71.17903930131004 64.00441994617344
3 60.27985371283193 54.86 75.51 71.79039301310044 65.6100616814831
4 61.138495786293525 56.33 76.33 72.66375545851528 66.61556281120221
5 61.631419939577036 58.13 76.89 73.1877729257642 67.45979821633532
6 62.513913181745906 57.35 77.96 72.5764192139738 67.60008309892993
7 63.126093178565746 57.36 78.63 74.14847161572052 68.31614119857156

7
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,7 @@
gqa,okvqa,vqav2,aokvqa,avg
57.06789632692002,54.44,73.8,71.09170305676857,64.09989984592214
59.85848306567022,56.06,75.25,72.75109170305677,65.97989369218175
60.717125139131824,55.3,76.37,73.1877729257642,66.39372451622401
61.34520591508984,57.32,76.94,73.62445414847161,67.30741501589037
62.6649705835586,58.16,77.93,72.5764192139738,67.8328474493831
62.943234218476704,59.1,78.55,74.32314410480349,68.72909458082005
1 gqa okvqa vqav2 aokvqa avg
2 57.06789632692002 54.44 73.8 71.09170305676857 64.09989984592214
3 59.85848306567022 56.06 75.25 72.75109170305677 65.97989369218175
4 60.717125139131824 55.3 76.37 73.1877729257642 66.39372451622401
5 61.34520591508984 57.32 76.94 73.62445414847161 67.30741501589037
6 62.6649705835586 58.16 77.93 72.5764192139738 67.8328474493831
7 62.943234218476704 59.1 78.55 74.32314410480349 68.72909458082005

8
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,8 @@
gqa,okvqa,vqav2,aokvqa,avg
57.203052949594536,53.68,73.68,70.48034934497817,63.76085057364318
59.09524566703769,54.05,74.96,72.75109170305677,65.21408434252362
61.18619812370806,56.16,76.09,73.53711790393012,66.74332900690955
62.03688980760057,57.59,76.64,72.92576419213974,67.29816349993509
62.47416123390046,57.58,77.69,72.75109170305677,67.62381323423931
63.014787724598506,57.61,78.51,73.97379912663754,68.27714671280901
63.43615837176022,57.94,79.02,73.62445414847161,68.50515313005796
1 gqa okvqa vqav2 aokvqa avg
2 57.203052949594536 53.68 73.68 70.48034934497817 63.76085057364318
3 59.09524566703769 54.05 74.96 72.75109170305677 65.21408434252362
4 61.18619812370806 56.16 76.09 73.53711790393012 66.74332900690955
5 62.03688980760057 57.59 76.64 72.92576419213974 67.29816349993509
6 62.47416123390046 57.58 77.69 72.75109170305677 67.62381323423931
7 63.014787724598506 57.61 78.51 73.97379912663754 68.27714671280901
8 63.43615837176022 57.94 79.02 73.62445414847161 68.50515313005796

11
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
56.726029575449196,54.51,73.9,71.9650655021834,64.27527376940814
59.5802194307521,55.89,75.12,72.83842794759825,65.8571618445876
60.605819685164576,56.41,75.93,74.06113537117905,66.7517387640859
61.43266020034982,57.36,76.65,73.1004366812227,67.13577422039313
62.32310383208777,57.71,77.76,74.235807860262,68.00722792308744
62.80807759580219,58.29,78.22,73.7117903930131,68.25746699720382
63.07839084115122,58.6,78.88,74.235807860262,68.6985496753533
63.722372396247415,58.5,79.24,74.67248908296943,69.03371536980421
63.89728096676737,58.97,79.44,73.88646288209607,69.04843596221586
63.84957862935284,58.62,79.56,73.88646288209607,68.97901037786222
1 gqa okvqa vqav2 aokvqa avg
2 56.726029575449196 54.51 73.9 71.9650655021834 64.27527376940814
3 59.5802194307521 55.89 75.12 72.83842794759825 65.8571618445876
4 60.605819685164576 56.41 75.93 74.06113537117905 66.7517387640859
5 61.43266020034982 57.36 76.65 73.1004366812227 67.13577422039313
6 62.32310383208777 57.71 77.76 74.235807860262 68.00722792308744
7 62.80807759580219 58.29 78.22 73.7117903930131 68.25746699720382
8 63.07839084115122 58.6 78.88 74.235807860262 68.6985496753533
9 63.722372396247415 58.5 79.24 74.67248908296943 69.03371536980421
10 63.89728096676737 58.97 79.44 73.88646288209607 69.04843596221586
11 63.84957862935284 58.62 79.56 73.88646288209607 68.97901037786222

8
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,8 @@
gqa,okvqa,vqav2,aokvqa,avg
56.99634282079822,53.79,73.68,70.82969432314411,63.82400928598558
59.68357449515026,54.46,75.16,72.40174672489083,65.42633030501027
60.939736047066305,55.99,75.88,71.87772925764192,66.17186632617705
62.00508824932422,58.43,76.67,72.48908296943232,67.39854280468914
62.8716807123549,58.22,77.73,73.88646288209607,68.17703589861274
62.99093655589124,58.08,78.53,74.06113537117905,68.41551798176756
63.42820798219113,58.48,78.93,74.235807860262,68.76850396061329
1 gqa okvqa vqav2 aokvqa avg
2 56.99634282079822 53.79 73.68 70.82969432314411 63.82400928598558
3 59.68357449515026 54.46 75.16 72.40174672489083 65.42633030501027
4 60.939736047066305 55.99 75.88 71.87772925764192 66.17186632617705
5 62.00508824932422 58.43 76.67 72.48908296943232 67.39854280468914
6 62.8716807123549 58.22 77.73 73.88646288209607 68.17703589861274
7 62.99093655589124 58.08 78.53 74.06113537117905 68.41551798176756
8 63.42820798219113 58.48 78.93 74.235807860262 68.76850396061329

7
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,7 @@
gqa,okvqa,vqav2,aokvqa,avg
56.932739704245506,54.43,73.9,71.52838427947599,64.19778099593037
60.057242804897434,55.95,75.28,72.31441048034935,65.9004133213117
61.44061058991891,54.26,76.24,73.1877729257642,66.28209587892077
61.806328510096996,57.5,76.83,73.44978165938865,67.3965275423714
62.56956590872953,58.33,77.89,73.7991266375546,68.14717313657103
62.760375258387654,58.37,78.45,74.32314410480349,68.47587984079779
1 gqa okvqa vqav2 aokvqa avg
2 56.932739704245506 54.43 73.9 71.52838427947599 64.19778099593037
3 60.057242804897434 55.95 75.28 72.31441048034935 65.9004133213117
4 61.44061058991891 54.26 76.24 73.1877729257642 66.28209587892077
5 61.806328510096996 57.5 76.83 73.44978165938865 67.3965275423714
6 62.56956590872953 58.33 77.89 73.7991266375546 68.14717313657103
7 62.760375258387654 58.37 78.45 74.32314410480349 68.47587984079779

10
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,10 @@
gqa,okvqa,vqav2,aokvqa,avg
56.940690093814595,54.81,73.85,71.09170305676857,64.17309828764579
60.20034981714104,55.52,75.37,72.5764192139738,65.9166922577787
61.19414851327715,54.56,76.38,73.27510917030567,66.35231442089571
61.217999681984416,57.83,76.88,73.36244541484717,67.3226112742079
62.64111941485133,57.97,77.97,72.92576419213974,67.87672090174777
63.03863889330577,58.54,78.49,72.75109170305677,68.20493264909064
63.61106694228017,58.49,79.1,74.06113537117905,68.8155505783648
63.66671966926379,58.68,79.56,72.92576419213974,68.70812096535089
64.0244871998728,59.23,79.74,73.7117903930131,69.17656939822146
1 gqa okvqa vqav2 aokvqa avg
2 56.940690093814595 54.81 73.85 71.09170305676857 64.17309828764579
3 60.20034981714104 55.52 75.37 72.5764192139738 65.9166922577787
4 61.19414851327715 54.56 76.38 73.27510917030567 66.35231442089571
5 61.217999681984416 57.83 76.88 73.36244541484717 67.3226112742079
6 62.64111941485133 57.97 77.97 72.92576419213974 67.87672090174777
7 63.03863889330577 58.54 78.49 72.75109170305677 68.20493264909064
8 63.61106694228017 58.49 79.1 74.06113537117905 68.8155505783648
9 63.66671966926379 58.68 79.56 72.92576419213974 68.70812096535089
10 64.0244871998728 59.23 79.74 73.7117903930131 69.17656939822146

8
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,8 @@
gqa,okvqa,vqav2,aokvqa,avg
56.78963269200191,53.55,73.54,70.04366812227074,63.48082520356816
59.95388774049929,54.89,75.12,71.52838427947599,65.37306800499383
60.7966290348227,55.93,76.01,72.83842794759825,66.39376424560524
61.84608045794244,56.83,76.7,73.44978165938865,67.20646552933277
62.943234218476704,57.48,77.75,74.235807860262,68.10226051968468
62.67292097312769,57.44,78.43,75.4585152838428,68.50035906424263
63.36460486563842,57.82,78.92,75.10917030567686,68.80344379282882
1 gqa okvqa vqav2 aokvqa avg
2 56.78963269200191 53.55 73.54 70.04366812227074 63.48082520356816
3 59.95388774049929 54.89 75.12 71.52838427947599 65.37306800499383
4 60.7966290348227 55.93 76.01 72.83842794759825 66.39376424560524
5 61.84608045794244 56.83 76.7 73.44978165938865 67.20646552933277
6 62.943234218476704 57.48 77.75 74.235807860262 68.10226051968468
7 62.67292097312769 57.44 78.43 75.4585152838428 68.50035906424263
8 63.36460486563842 57.82 78.92 75.10917030567686 68.80344379282882

8
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,8 @@
gqa,okvqa,vqav2,aokvqa,avg
57.020193989505486,53.7,73.8,70.21834061135371,63.6846336502148
60.33550643981555,54.9,75.12,72.40174672489083,65.68931329117659
60.947686436635394,55.55,76.29,73.01310043668123,66.45019671832915
61.45651136905709,57.24,76.39,73.62445414847161,67.17774137938218
62.65702019398951,57.34,77.73,75.37117903930131,68.2745498083227
63.5156622674511,57.95,78.4,73.62445414847161,68.37252910398068
63.6587692796947,57.77,78.75,74.49781659388647,68.6691464683953
1 gqa okvqa vqav2 aokvqa avg
2 57.020193989505486 53.7 73.8 70.21834061135371 63.6846336502148
3 60.33550643981555 54.9 75.12 72.40174672489083 65.68931329117659
4 60.947686436635394 55.55 76.29 73.01310043668123 66.45019671832915
5 61.45651136905709 57.24 76.39 73.62445414847161 67.17774137938218
6 62.65702019398951 57.34 77.73 75.37117903930131 68.2745498083227
7 63.5156622674511 57.95 78.4 73.62445414847161 68.37252910398068
8 63.6587692796947 57.77 78.75 74.49781659388647 68.6691464683953

11
process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
57.218953728732714,55.12,74.19,71.52838427947599,64.51433450205218
60.0095404674829,55.87,75.27,72.66375545851528,65.95332398149955
60.73302591827,56.65,76.19,73.44978165938865,66.75570189441466
61.42470981078073,58.09,76.81,73.44978165938865,67.44362286754235
62.37875655907139,57.71,78.0,73.53711790393012,67.90646861575038
62.73652408968039,57.99,78.57,73.27510917030567,68.14290831499652
63.26124980124026,58.96,79.16,74.75982532751092,69.03526878218779
63.626967721418346,58.1,79.68,74.235807860262,68.9106938954201
63.77007473366195,58.19,79.87,74.32314410480349,69.03830470961636
63.626967721418346,57.76,79.98,74.49781659388647,68.9661960788262
1 gqa okvqa vqav2 aokvqa avg
2 57.218953728732714 55.12 74.19 71.52838427947599 64.51433450205218
3 60.0095404674829 55.87 75.27 72.66375545851528 65.95332398149955
4 60.73302591827 56.65 76.19 73.44978165938865 66.75570189441466
5 61.42470981078073 58.09 76.81 73.44978165938865 67.44362286754235
6 62.37875655907139 57.71 78.0 73.53711790393012 67.90646861575038
7 62.73652408968039 57.99 78.57 73.27510917030567 68.14290831499652
8 63.26124980124026 58.96 79.16 74.75982532751092 69.03526878218779
9 63.626967721418346 58.1 79.68 74.235807860262 68.9106938954201
10 63.77007473366195 58.19 79.87 74.32314410480349 69.03830470961636
11 63.626967721418346 57.76 79.98 74.49781659388647 68.9661960788262

9
process_log/caption_result/results_coco_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv Normal file
View File

@ -0,0 +1,9 @@
Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr
0.8182082924454812,0.6638925862784945,0.5165455592392635,0.3928161572696676,0.2920462773984222,0.5918020291082554,1.2987178029426083
0.8240854949333832,0.6717746108928253,0.5257706306569868,0.40249821532303487,0.2936994611883999,0.5958284475381215,1.3272042634335437
0.8246398014401575,0.6743122037328803,0.5292565031234792,0.40578728669235764,0.2964349874781921,0.597962455259089,1.3321248494065205
0.8270564620397717,0.6793843847871991,0.5355102159092959,0.41366323151841006,0.2977148403261682,0.6024717705283715,1.3520977723112642
0.8245944565058522,0.6751194271857663,0.5306411803879995,0.40888833957017,0.29642409734712727,0.598579033363294,1.3383802889190928
0.8280804382261628,0.6796044258685356,0.537046439535483,0.4152107936908873,0.2990414503157605,0.602392043880006,1.3531755491021618
0.8299061115859161,0.6814451792563676,0.5379678657467563,0.4156139159677314,0.2992684448837016,0.6027499590797065,1.3538095043242933
0.8276954938807027,0.6800987491517746,0.5374861277345877,0.41554847690232694,0.2996663866671749,0.6026710548192867,1.3561296682313817
1 Bleu_1 Bleu_2 Bleu_3 Bleu_4 METEOR ROUGE_L CIDEr
2 0.8182082924454812 0.6638925862784945 0.5165455592392635 0.3928161572696676 0.2920462773984222 0.5918020291082554 1.2987178029426083
3 0.8240854949333832 0.6717746108928253 0.5257706306569868 0.40249821532303487 0.2936994611883999 0.5958284475381215 1.3272042634335437
4 0.8246398014401575 0.6743122037328803 0.5292565031234792 0.40578728669235764 0.2964349874781921 0.597962455259089 1.3321248494065205
5 0.8270564620397717 0.6793843847871991 0.5355102159092959 0.41366323151841006 0.2977148403261682 0.6024717705283715 1.3520977723112642
6 0.8245944565058522 0.6751194271857663 0.5306411803879995 0.40888833957017 0.29642409734712727 0.598579033363294 1.3383802889190928
7 0.8280804382261628 0.6796044258685356 0.537046439535483 0.4152107936908873 0.2990414503157605 0.602392043880006 1.3531755491021618
8 0.8299061115859161 0.6814451792563676 0.5379678657467563 0.4156139159677314 0.2992684448837016 0.6027499590797065 1.3538095043242933
9 0.8276954938807027 0.6800987491517746 0.5374861277345877 0.41554847690232694 0.2996663866671749 0.6026710548192867 1.3561296682313817

9
process_log/caption_result/results_text_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv Normal file
View File

@ -0,0 +1,9 @@
Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr
0.7309287396242824,0.5494655932407475,0.4041141975269019,0.2921841096479282,0.23361135981392955,0.48650322014232156,0.9966076482929254
0.7382464874712654,0.5571950857230415,0.4121904524343956,0.30143019979529023,0.23826500378661586,0.4926618553461225,1.0232964787151213
0.7350922242488682,0.5559390933443953,0.4113206176871192,0.3009155411187479,0.2374533115810085,0.49274408438489914,1.0099444415721002
0.7369161501554021,0.556728672691215,0.41086927244074317,0.3004344996181908,0.23806683186363936,0.49184279207452697,1.0280928870205412
0.7407251133040751,0.5595661891353809,0.4138229944446712,0.3032027996796593,0.2415473861083878,0.4972778899306893,1.0479529588574925
0.7428951836605517,0.5623862396980985,0.4161405396490859,0.3047709156231164,0.2438504314752837,0.49858613995817086,1.0477488915322588
0.745208695011462,0.5654882473522628,0.4200936310645993,0.308981454910347,0.24371581659378194,0.49914038443933745,1.053786374859757
0.7444798335079819,0.5624985319454049,0.4157842492239835,0.3044642812661473,0.2432655219581268,0.49722674885535495,1.056752744677739
1 Bleu_1 Bleu_2 Bleu_3 Bleu_4 METEOR ROUGE_L CIDEr
2 0.7309287396242824 0.5494655932407475 0.4041141975269019 0.2921841096479282 0.23361135981392955 0.48650322014232156 0.9966076482929254
3 0.7382464874712654 0.5571950857230415 0.4121904524343956 0.30143019979529023 0.23826500378661586 0.4926618553461225 1.0232964787151213
4 0.7350922242488682 0.5559390933443953 0.4113206176871192 0.3009155411187479 0.2374533115810085 0.49274408438489914 1.0099444415721002
5 0.7369161501554021 0.556728672691215 0.41086927244074317 0.3004344996181908 0.23806683186363936 0.49184279207452697 1.0280928870205412
6 0.7407251133040751 0.5595661891353809 0.4138229944446712 0.3032027996796593 0.2415473861083878 0.4972778899306893 1.0479529588574925
7 0.7428951836605517 0.5623862396980985 0.4161405396490859 0.3047709156231164 0.2438504314752837 0.49858613995817086 1.0477488915322588
8 0.745208695011462 0.5654882473522628 0.4200936310645993 0.308981454910347 0.24371581659378194 0.49914038443933745 1.053786374859757
9 0.7444798335079819 0.5624985319454049 0.4157842492239835 0.3044642812661473 0.2432655219581268 0.49722674885535495 1.056752744677739

56
process_log/load_log.py
View File

@ -11,7 +11,53 @@ import pandas as pd
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_4ex4b_2loss_005_top6layer_textinqf_10epo_0319/20240319110/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr6e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0319/20240319105"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_tcap_raw_Qformer_base_lr5e5_10epo_0320_instruct/20240320230/"
path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr3e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0326/20240326134"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr3e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0326/20240326134"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/flant5xxl/base/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328/20240328100/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0328/20240328164"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_base_lr5e5_textinqf_8epo_0329/20240329110"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0330/20240329232/"
# mix5
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330211/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330210/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330211/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330183/"
# mix6 3ex3b uni 005
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330211/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330182/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330173/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330173/"
# mix6 3ex3b uni 001
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330/20240330221/"
# mi6 non-universal
# cls-cross-route-mix6-001-3ex3b-0330-wanghanzi-master-0
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330/20240330221/"
# cls-cross-route-mix6-001-4ex4b-wanghanzi-master-0
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331214/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331214/"
# mix6 4ex4b uni 001
path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331/20240331132/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331144/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331143/"
# cls cross
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331215/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331215/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/"
# 8ex8b uni
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331/20240331150/"
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/"
# post
# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0331/20240331131/"
modes = ['gqa','okvqa','vqav2','aokvqa']
file_name = dict()
data = dict()
@ -25,7 +71,11 @@ for mode in modes:
data[mode] = accs
print(data)
df = pd.DataFrame(data)
df.to_csv("/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/results.csv",index=False)
# calculate average score
df['avg'] = df.mean(axis=1)
name = path.split('/')[-3]
df.to_csv(f"/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/0401_results_{name}.csv",index=False)
print("save to /mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/0401_results_{}.csv".format(name))
print(df)
modes = ['coco_cap','text_cap']
for mode in modes:
@ -35,7 +85,7 @@ for mode in modes:
for line in f:
tmp.append(json.loads(line)["val"])
df1 = pd.DataFrame(tmp)
df1.to_csv("/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/results_{}.csv".format(mode),index=False)
df1.to_csv("/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/caption_result/results_{}_{}.csv".format(mode,name),index=False)
print("\n",df1)

21
process_log/results.csv
View File

@ -1,11 +1,10 @@
gqa,okvqa,vqav2,aokvqa
56.20130386388933,54.9,73.99,70.65502183406113
59.98568929877563,56.83,75.05,71.87772925764192
60.81252981396088,56.97,76.33,72.31441048034935
61.217999681984416,58.72,76.9,72.13973799126637
62.63316902528224,57.77,77.9,73.53711790393012
62.63316902528224,58.7,78.62,72.31441048034935
63.0863412307203,58.44,79.17,73.01310043668123
63.99268564159644,58.3,79.67,73.1004366812227
63.85752901892193,58.26,79.89,73.1877729257642
64.05628875814915,58.36,79.96,73.27510917030567
gqa,okvqa,vqav2,aokvqa,avg
57.14740022261091,54.38,74.13,71.52838427947599,64.29644612552173
59.19860073143583,55.98,75.1,72.5764192139738,65.7137549863524
60.64557163301002,56.56,75.9,73.01310043668123,66.52966801742281
61.3770074733662,57.87,76.6,74.06113537117905,67.47703571113631
62.40260772777866,57.16,77.31,74.32314410480349,67.79893795814553
62.99888694546033,57.96,78.12,73.1877729257642,68.06666496780613
62.69677214183494,59.45,78.75,74.235807860262,68.78314500052424
63.126093178565746,58.74,79.22,74.41048034934498,68.87414338197769
63.30100174908571,58.32,79.5,73.7117903930131,68.7081980355247

1 gqa okvqa vqav2 aokvqa avg
2 56.20130386388933 57.14740022261091 54.9 54.38 73.99 74.13 70.65502183406113 71.52838427947599 64.29644612552173
3 59.98568929877563 59.19860073143583 56.83 55.98 75.05 75.1 71.87772925764192 72.5764192139738 65.7137549863524
4 60.81252981396088 60.64557163301002 56.97 56.56 76.33 75.9 72.31441048034935 73.01310043668123 66.52966801742281
5 61.217999681984416 61.3770074733662 58.72 57.87 76.9 76.6 72.13973799126637 74.06113537117905 67.47703571113631
6 62.63316902528224 62.40260772777866 57.77 57.16 77.9 77.31 73.53711790393012 74.32314410480349 67.79893795814553
7 62.63316902528224 62.99888694546033 58.7 57.96 78.62 78.12 72.31441048034935 73.1877729257642 68.06666496780613
8 63.0863412307203 62.69677214183494 58.44 59.45 79.17 78.75 73.01310043668123 74.235807860262 68.78314500052424
9 63.99268564159644 63.126093178565746 58.3 58.74 79.67 79.22 73.1004366812227 74.41048034934498 68.87414338197769
10 63.85752901892193 63.30100174908571 58.26 58.32 79.89 79.5 73.1877729257642 73.7117903930131 68.7081980355247
64.05628875814915 58.36 79.96 73.27510917030567

9
process_log/results_base.csv Normal file
View File

@ -0,0 +1,9 @@
gqa,okvqa,vqav2,aokvqa,avg
53.46636985212275,48.02,71.28,73.27510917030567,61.5103697556071
55.525520750516776,50.12,72.37,74.32314410480349,63.08466621383006
56.91683892510733,49.66,73.78,75.54585152838428,63.9756726133729
57.616473207187155,52.98,74.3,76.85589519650655,65.43809210092343
58.538718397201464,52.81,74.97,75.9825327510917,65.57531278707329
59.54841787247575,52.71,75.53,76.68122270742359,66.11741014497483
59.60407059945937,53.43,75.81,76.68122270742359,66.38132332672075
59.68357449515026,53.71,75.99,77.117903930131,66.6253696063203
1 gqa okvqa vqav2 aokvqa avg
2 53.46636985212275 48.02 71.28 73.27510917030567 61.5103697556071
3 55.525520750516776 50.12 72.37 74.32314410480349 63.08466621383006
4 56.91683892510733 49.66 73.78 75.54585152838428 63.9756726133729
5 57.616473207187155 52.98 74.3 76.85589519650655 65.43809210092343
6 58.538718397201464 52.81 74.97 75.9825327510917 65.57531278707329
7 59.54841787247575 52.71 75.53 76.68122270742359 66.11741014497483
8 59.60407059945937 53.43 75.81 76.68122270742359 66.38132332672075
9 59.68357449515026 53.71 75.99 77.117903930131 66.6253696063203

18
process_log/results_coco_cap.csv
View File

@ -1,11 +1,9 @@
Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr
0.823223287530701,0.6717046486084919,0.5289578677489075,0.40968911321073265,0.29959733857127835,0.6000811689013733,1.3453305422681376
0.8223586805364383,0.6761940235369243,0.5372021051535613,0.42002107530341415,0.3007070731886293,0.6033562057362387,1.3625435411445188
0.8268533010152048,0.6768505208938964,0.5354603117460063,0.41692252223016474,0.3025309435581166,0.6047665704453451,1.372728338529115
0.8330185365456568,0.6839215475169012,0.543703824076164,0.42514888177425736,0.3056707793309544,0.6059406486352524,1.3861562304957278
0.82851059467086,0.6809176651975785,0.540587141996957,0.4231733852045105,0.30531255145802344,0.6074083433970782,1.3834468097216504
0.826759683808046,0.6786303228237849,0.5393123491853253,0.4218864249035804,0.30435744091261596,0.6061024359922585,1.373555774390539
0.8276490281491424,0.6807227995567375,0.5428624362839755,0.4269084803683001,0.30608939915890326,0.6078544738015538,1.3817698541866794
0.8294752997477592,0.684006846582908,0.5462002712862497,0.43036414820903174,0.3072188868161371,0.6112755876824707,1.3946012917260822
0.8295967394767231,0.6833360908793101,0.5455277195646591,0.4292003829675252,0.3075074477217475,0.6103970688371728,1.392473266614681
0.8311746625208477,0.685154908394367,0.5473453025692251,0.4310962369321893,0.3077283192093123,0.6112527230048272,1.3960092087389069
0.8182082924454812,0.6638925862784945,0.5165455592392635,0.3928161572696676,0.2920462773984222,0.5918020291082554,1.2987178029426083
0.8240854949333832,0.6717746108928253,0.5257706306569868,0.40249821532303487,0.2936994611883999,0.5958284475381215,1.3272042634335437
0.8246398014401575,0.6743122037328803,0.5292565031234792,0.40578728669235764,0.2964349874781921,0.597962455259089,1.3321248494065205
0.8270564620397717,0.6793843847871991,0.5355102159092959,0.41366323151841006,0.2977148403261682,0.6024717705283715,1.3520977723112642
0.8245944565058522,0.6751194271857663,0.5306411803879995,0.40888833957017,0.29642409734712727,0.598579033363294,1.3383802889190928
0.8280804382261628,0.6796044258685356,0.537046439535483,0.4152107936908873,0.2990414503157605,0.602392043880006,1.3531755491021618
0.8299061115859161,0.6814451792563676,0.5379678657467563,0.4156139159677314,0.2992684448837016,0.6027499590797065,1.3538095043242933
0.8276954938807027,0.6800987491517746,0.5374861277345877,0.41554847690232694,0.2996663866671749,0.6026710548192867,1.3561296682313817

1 Bleu_1 Bleu_2 Bleu_3 Bleu_4 METEOR ROUGE_L CIDEr
2 0.823223287530701 0.8182082924454812 0.6717046486084919 0.6638925862784945 0.5289578677489075 0.5165455592392635 0.40968911321073265 0.3928161572696676 0.29959733857127835 0.2920462773984222 0.6000811689013733 0.5918020291082554 1.3453305422681376 1.2987178029426083
3 0.8223586805364383 0.8240854949333832 0.6761940235369243 0.6717746108928253 0.5372021051535613 0.5257706306569868 0.42002107530341415 0.40249821532303487 0.3007070731886293 0.2936994611883999 0.6033562057362387 0.5958284475381215 1.3625435411445188 1.3272042634335437
4 0.8268533010152048 0.8246398014401575 0.6768505208938964 0.6743122037328803 0.5354603117460063 0.5292565031234792 0.41692252223016474 0.40578728669235764 0.3025309435581166 0.2964349874781921 0.6047665704453451 0.597962455259089 1.372728338529115 1.3321248494065205
5 0.8330185365456568 0.8270564620397717 0.6839215475169012 0.6793843847871991 0.543703824076164 0.5355102159092959 0.42514888177425736 0.41366323151841006 0.3056707793309544 0.2977148403261682 0.6059406486352524 0.6024717705283715 1.3861562304957278 1.3520977723112642
6 0.82851059467086 0.8245944565058522 0.6809176651975785 0.6751194271857663 0.540587141996957 0.5306411803879995 0.4231733852045105 0.40888833957017 0.30531255145802344 0.29642409734712727 0.6074083433970782 0.598579033363294 1.3834468097216504 1.3383802889190928
7 0.826759683808046 0.8280804382261628 0.6786303228237849 0.6796044258685356 0.5393123491853253 0.537046439535483 0.4218864249035804 0.4152107936908873 0.30435744091261596 0.2990414503157605 0.6061024359922585 0.602392043880006 1.373555774390539 1.3531755491021618
8 0.8276490281491424 0.8299061115859161 0.6807227995567375 0.6814451792563676 0.5428624362839755 0.5379678657467563 0.4269084803683001 0.4156139159677314 0.30608939915890326 0.2992684448837016 0.6078544738015538 0.6027499590797065 1.3817698541866794 1.3538095043242933
9 0.8294752997477592 0.8276954938807027 0.684006846582908 0.6800987491517746 0.5462002712862497 0.5374861277345877 0.43036414820903174 0.41554847690232694 0.3072188868161371 0.2996663866671749 0.6112755876824707 0.6026710548192867 1.3946012917260822 1.3561296682313817
0.8295967394767231 0.6833360908793101 0.5455277195646591 0.4292003829675252 0.3075074477217475 0.6103970688371728 1.392473266614681
0.8311746625208477 0.685154908394367 0.5473453025692251 0.4310962369321893 0.3077283192093123 0.6112527230048272 1.3960092087389069

11
process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
56.49546827794561,56.81,73.81,70.21834061135371,64.33345222232484
58.97598982350135,54.6,75.05,70.91703056768559,64.88575509779673
61.408809031642555,56.58,76.51,73.36244541484717,66.96531361162243
62.020989028462395,58.58,76.98,72.13973799126637,67.43018175493219
62.346955000795035,56.98,78.14,73.27510917030567,67.68551604277518
63.43615837176022,57.51,78.88,73.1004366812227,68.23164876324573
63.79392590236922,57.77,79.41,73.7117903930131,68.67142907384559
63.722372396247415,58.26,79.93,73.88646288209607,68.94970881958587
63.57131499443472,58.05,80.13,73.36244541484717,68.77844010232047
63.817777071076485,58.43,80.19,73.7117903930131,69.0373918660224
1 gqa okvqa vqav2 aokvqa avg
2 56.49546827794561 56.81 73.81 70.21834061135371 64.33345222232484
3 58.97598982350135 54.6 75.05 70.91703056768559 64.88575509779673
4 61.408809031642555 56.58 76.51 73.36244541484717 66.96531361162243
5 62.020989028462395 58.58 76.98 72.13973799126637 67.43018175493219
6 62.346955000795035 56.98 78.14 73.27510917030567 67.68551604277518
7 63.43615837176022 57.51 78.88 73.1004366812227 68.23164876324573
8 63.79392590236922 57.77 79.41 73.7117903930131 68.67142907384559
9 63.722372396247415 58.26 79.93 73.88646288209607 68.94970881958587
10 63.57131499443472 58.05 80.13 73.36244541484717 68.77844010232047
11 63.817777071076485 58.43 80.19 73.7117903930131 69.0373918660224

9
process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,9 @@
gqa,okvqa,vqav2,aokvqa,avg
56.638575290189216,55.75,73.72,71.09170305676857,64.30006958673944
59.111146446175866,56.38,75.02,73.1877729257642,65.92472984298502
61.43266020034982,56.12,76.5,72.83842794759825,66.72277203698701
62.283351884242336,58.41,77.06,72.40174672489083,67.5387746522833
62.31515344251868,57.59,78.2,73.7117903930131,67.95423595888295
63.42820798219113,57.71,78.77,73.97379912663754,68.47050177720718
63.316902528223885,58.18,79.44,73.97379912663754,68.72767541371536
63.74622356495468,58.59,79.9,74.235807860262,69.11800785630417
1 gqa okvqa vqav2 aokvqa avg
2 56.638575290189216 55.75 73.72 71.09170305676857 64.30006958673944
3 59.111146446175866 56.38 75.02 73.1877729257642 65.92472984298502
4 61.43266020034982 56.12 76.5 72.83842794759825 66.72277203698701
5 62.283351884242336 58.41 77.06 72.40174672489083 67.5387746522833
6 62.31515344251868 57.59 78.2 73.7117903930131 67.95423595888295
7 63.42820798219113 57.71 78.77 73.97379912663754 68.47050177720718
8 63.316902528223885 58.18 79.44 73.97379912663754 68.72767541371536
9 63.74622356495468 58.59 79.9 74.235807860262 69.11800785630417

10
process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,10 @@
gqa,okvqa,vqav2,aokvqa,avg
56.78963269200191,56.29,73.86,70.91703056768559,64.46416581492187
59.66767371601208,54.41,75.21,72.48908296943232,65.4441891713611
61.33725552552075,56.65,76.37,73.62445414847161,66.9954274184981
62.013038638893306,59.54,77.12,73.27510917030567,67.98703695229975
62.927333439338526,57.51,78.17,73.62445414847161,68.05794689695253
63.67467005883288,57.53,78.9,73.01310043668123,68.27944262387852
63.5156622674511,58.04,79.41,74.06113537117905,68.75669940965753
63.642868500556524,58.31,79.97,73.44978165938865,68.8431625399863
63.937032914612814,58.11,80.18,73.97379912663754,69.05020801031259
1 gqa okvqa vqav2 aokvqa avg
2 56.78963269200191 56.29 73.86 70.91703056768559 64.46416581492187
3 59.66767371601208 54.41 75.21 72.48908296943232 65.4441891713611
4 61.33725552552075 56.65 76.37 73.62445414847161 66.9954274184981
5 62.013038638893306 59.54 77.12 73.27510917030567 67.98703695229975
6 62.927333439338526 57.51 78.17 73.62445414847161 68.05794689695253
7 63.67467005883288 57.53 78.9 73.01310043668123 68.27944262387852
8 63.5156622674511 58.04 79.41 74.06113537117905 68.75669940965753
9 63.642868500556524 58.31 79.97 73.44978165938865 68.8431625399863
10 63.937032914612814 58.11 80.18 73.97379912663754 69.05020801031259

9
process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,9 @@
gqa,okvqa,vqav2,aokvqa,avg
56.21720464302751,56.58,73.83,70.56768558951964,64.29872255813677
59.40531086023215,55.45,75.26,72.40174672489083,65.62926439628075
59.810780728255686,52.47,75.44,70.82969432314411,64.63761876284994
61.75862617268246,58.43,76.42,71.9650655021834,67.14342291871647
62.77627603752583,57.39,77.56,73.88646288209607,67.90318472990548
63.141993957703924,57.33,78.13,74.41048034934498,68.25311857676222
63.44410876132931,57.15,78.86,73.01310043668123,68.11680229950264
63.34870408650024,57.88,79.29,73.7117903930131,68.55762361987834
1 gqa okvqa vqav2 aokvqa avg
2 56.21720464302751 56.58 73.83 70.56768558951964 64.29872255813677
3 59.40531086023215 55.45 75.26 72.40174672489083 65.62926439628075
4 59.810780728255686 52.47 75.44 70.82969432314411 64.63761876284994
5 61.75862617268246 58.43 76.42 71.9650655021834 67.14342291871647
6 62.77627603752583 57.39 77.56 73.88646288209607 67.90318472990548
7 63.141993957703924 57.33 78.13 74.41048034934498 68.25311857676222
8 63.44410876132931 57.15 78.86 73.01310043668123 68.11680229950264
9 63.34870408650024 57.88 79.29 73.7117903930131 68.55762361987834

11
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
56.320559707425666,54.43,73.75,71.61572052401746,64.02907005786079
58.936237875655905,56.23,75.25,71.09170305676857,65.37698523310611
60.99538877404993,55.3,76.4,73.01310043668123,66.42712230268279
61.146446175862614,58.62,76.58,73.1004366812227,67.36172071427133
62.5934170774368,58.5,77.5,73.88646288209607,68.11996998988322
62.41850850691684,57.47,78.24,73.7117903930131,67.96007472498249
62.99888694546033,58.08,78.88,74.235807860262,68.54867370143059
63.77802512323104,58.72,79.34,74.06113537117905,68.97479012360252
63.74622356495468,58.5,79.51,74.235807860262,68.99800785630417
63.825727460645574,58.41,79.63,74.41048034934498,69.06905195249763
1 gqa okvqa vqav2 aokvqa avg
2 56.320559707425666 54.43 73.75 71.61572052401746 64.02907005786079
3 58.936237875655905 56.23 75.25 71.09170305676857 65.37698523310611
4 60.99538877404993 55.3 76.4 73.01310043668123 66.42712230268279
5 61.146446175862614 58.62 76.58 73.1004366812227 67.36172071427133
6 62.5934170774368 58.5 77.5 73.88646288209607 68.11996998988322
7 62.41850850691684 57.47 78.24 73.7117903930131 67.96007472498249
8 62.99888694546033 58.08 78.88 74.235807860262 68.54867370143059
9 63.77802512323104 58.72 79.34 74.06113537117905 68.97479012360252
10 63.74622356495468 58.5 79.51 74.235807860262 68.99800785630417
11 63.825727460645574 58.41 79.63 74.41048034934498 69.06905195249763

11
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
57.0122435999364,54.2,74.05,71.9650655021834,64.30682727552995
60.065193194466524,55.18,75.18,72.48908296943232,65.72856904097472
60.87613293051359,57.05,75.99,73.01310043668123,66.7323083417987
61.69502305612975,58.69,76.81,73.7991266375546,67.74853742342108
62.291302273811425,58.12,77.73,73.44978165938865,67.89777098330002
62.56956590872953,57.97,78.35,73.01310043668123,67.97566658635269
63.17379551598028,59.22,78.91,73.53711790393012,68.7102283549776
63.46000954046749,57.9,79.42,73.53711790393012,68.5792818610994
63.77802512323104,58.46,79.58,73.36244541484717,68.79511763451956
63.83367785021466,58.36,79.73,73.62445414847161,68.88703299967158
1 gqa okvqa vqav2 aokvqa avg
2 57.0122435999364 54.2 74.05 71.9650655021834 64.30682727552995
3 60.065193194466524 55.18 75.18 72.48908296943232 65.72856904097472
4 60.87613293051359 57.05 75.99 73.01310043668123 66.7323083417987
5 61.69502305612975 58.69 76.81 73.7991266375546 67.74853742342108
6 62.291302273811425 58.12 77.73 73.44978165938865 67.89777098330002
7 62.56956590872953 57.97 78.35 73.01310043668123 67.97566658635269
8 63.17379551598028 59.22 78.91 73.53711790393012 68.7102283549776
9 63.46000954046749 57.9 79.42 73.53711790393012 68.5792818610994
10 63.77802512323104 58.46 79.58 73.36244541484717 68.79511763451956
11 63.83367785021466 58.36 79.73 73.62445414847161 68.88703299967158

7
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,7 @@
gqa,okvqa,vqav2,aokvqa,avg
57.40976307839084,53.92,74.0,71.26637554585152,64.14903465606059
60.136746700588326,56.03,75.27,71.70305676855895,65.78495086728682
60.99538877404993,56.27,76.52,73.27510917030567,66.7651244860889
61.94943552234059,57.39,76.95,74.06113537117905,67.58764272337991
62.55366512959135,58.17,77.83,73.53711790393012,68.02269575838037
63.26920019080935,57.8,78.7,75.54585152838428,68.8287629297984
1 gqa okvqa vqav2 aokvqa avg
2 57.40976307839084 53.92 74.0 71.26637554585152 64.14903465606059
3 60.136746700588326 56.03 75.27 71.70305676855895 65.78495086728682
4 60.99538877404993 56.27 76.52 73.27510917030567 66.7651244860889
5 61.94943552234059 57.39 76.95 74.06113537117905 67.58764272337991
6 62.55366512959135 58.17 77.83 73.53711790393012 68.02269575838037
7 63.26920019080935 57.8 78.7 75.54585152838428 68.8287629297984

11
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
57.54491970106535,54.21,73.9,71.87772925764192,64.38316223967682
59.93798696136111,54.74,75.12,72.75109170305677,65.63726966610447
60.77277786611543,55.58,76.23,73.7991266375546,66.59547612591751
61.66322149785339,57.91,76.69,73.27510917030567,67.38458266703977
62.56956590872953,57.59,77.73,74.06113537117905,67.98767531997714
62.85577993321672,58.39,78.58,74.8471615720524,68.66823537631728
63.33280330736206,58.49,79.11,74.67248908296943,68.90132309758287
63.61106694228017,58.64,79.56,75.02183406113538,69.20822525085389
63.69852122754015,58.41,79.8,74.67248908296943,69.14525257762739
63.68262044840197,58.45,79.9,74.75982532751092,69.19811144397823
1 gqa okvqa vqav2 aokvqa avg
2 57.54491970106535 54.21 73.9 71.87772925764192 64.38316223967682
3 59.93798696136111 54.74 75.12 72.75109170305677 65.63726966610447
4 60.77277786611543 55.58 76.23 73.7991266375546 66.59547612591751
5 61.66322149785339 57.91 76.69 73.27510917030567 67.38458266703977
6 62.56956590872953 57.59 77.73 74.06113537117905 67.98767531997714
7 62.85577993321672 58.39 78.58 74.8471615720524 68.66823537631728
8 63.33280330736206 58.49 79.11 74.67248908296943 68.90132309758287
9 63.61106694228017 58.64 79.56 75.02183406113538 69.20822525085389
10 63.69852122754015 58.41 79.8 74.67248908296943 69.14525257762739
11 63.68262044840197 58.45 79.9 74.75982532751092 69.19811144397823

7
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,7 @@
gqa,okvqa,vqav2,aokvqa,avg
56.948640483383684,53.97,73.92,71.17903930131004,64.00441994617344
60.27985371283193,54.86,75.51,71.79039301310044,65.6100616814831
61.138495786293525,56.33,76.33,72.66375545851528,66.61556281120221
61.631419939577036,58.13,76.89,73.1877729257642,67.45979821633532
62.513913181745906,57.35,77.96,72.5764192139738,67.60008309892993
63.126093178565746,57.36,78.63,74.14847161572052,68.31614119857156
1 gqa okvqa vqav2 aokvqa avg
2 56.948640483383684 53.97 73.92 71.17903930131004 64.00441994617344
3 60.27985371283193 54.86 75.51 71.79039301310044 65.6100616814831
4 61.138495786293525 56.33 76.33 72.66375545851528 66.61556281120221
5 61.631419939577036 58.13 76.89 73.1877729257642 67.45979821633532
6 62.513913181745906 57.35 77.96 72.5764192139738 67.60008309892993
7 63.126093178565746 57.36 78.63 74.14847161572052 68.31614119857156

6
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,6 @@
gqa,okvqa,vqav2,aokvqa,avg
57.06789632692002,54.44,73.8,71.09170305676857,64.09989984592214
59.85848306567022,56.06,75.25,72.75109170305677,65.97989369218175
60.717125139131824,55.3,76.37,73.1877729257642,66.39372451622401
61.34520591508984,57.32,76.94,73.62445414847161,67.30741501589037
62.6649705835586,58.16,77.93,72.5764192139738,67.8328474493831
1 gqa okvqa vqav2 aokvqa avg
2 57.06789632692002 54.44 73.8 71.09170305676857 64.09989984592214
3 59.85848306567022 56.06 75.25 72.75109170305677 65.97989369218175
4 60.717125139131824 55.3 76.37 73.1877729257642 66.39372451622401
5 61.34520591508984 57.32 76.94 73.62445414847161 67.30741501589037
6 62.6649705835586 58.16 77.93 72.5764192139738 67.8328474493831

7
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,7 @@
gqa,okvqa,vqav2,aokvqa,avg
57.203052949594536,53.68,73.68,70.48034934497817,63.76085057364318
59.09524566703769,54.05,74.96,72.75109170305677,65.21408434252362
61.18619812370806,56.16,76.09,73.53711790393012,66.74332900690955
62.03688980760057,57.59,76.64,72.92576419213974,67.29816349993509
62.47416123390046,57.58,77.69,72.75109170305677,67.62381323423931
63.014787724598506,57.61,78.51,73.97379912663754,68.27714671280901
1 gqa okvqa vqav2 aokvqa avg
2 57.203052949594536 53.68 73.68 70.48034934497817 63.76085057364318
3 59.09524566703769 54.05 74.96 72.75109170305677 65.21408434252362
4 61.18619812370806 56.16 76.09 73.53711790393012 66.74332900690955
5 62.03688980760057 57.59 76.64 72.92576419213974 67.29816349993509
6 62.47416123390046 57.58 77.69 72.75109170305677 67.62381323423931
7 63.014787724598506 57.61 78.51 73.97379912663754 68.27714671280901

11
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
56.726029575449196,54.51,73.9,71.9650655021834,64.27527376940814
59.5802194307521,55.89,75.12,72.83842794759825,65.8571618445876
60.605819685164576,56.41,75.93,74.06113537117905,66.7517387640859
61.43266020034982,57.36,76.65,73.1004366812227,67.13577422039313
62.32310383208777,57.71,77.76,74.235807860262,68.00722792308744
62.80807759580219,58.29,78.22,73.7117903930131,68.25746699720382
63.07839084115122,58.6,78.88,74.235807860262,68.6985496753533
63.722372396247415,58.5,79.24,74.67248908296943,69.03371536980421
63.89728096676737,58.97,79.44,73.88646288209607,69.04843596221586
63.84957862935284,58.62,79.56,73.88646288209607,68.97901037786222
1 gqa okvqa vqav2 aokvqa avg
2 56.726029575449196 54.51 73.9 71.9650655021834 64.27527376940814
3 59.5802194307521 55.89 75.12 72.83842794759825 65.8571618445876
4 60.605819685164576 56.41 75.93 74.06113537117905 66.7517387640859
5 61.43266020034982 57.36 76.65 73.1004366812227 67.13577422039313
6 62.32310383208777 57.71 77.76 74.235807860262 68.00722792308744
7 62.80807759580219 58.29 78.22 73.7117903930131 68.25746699720382
8 63.07839084115122 58.6 78.88 74.235807860262 68.6985496753533
9 63.722372396247415 58.5 79.24 74.67248908296943 69.03371536980421
10 63.89728096676737 58.97 79.44 73.88646288209607 69.04843596221586
11 63.84957862935284 58.62 79.56 73.88646288209607 68.97901037786222

7
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,7 @@
gqa,okvqa,vqav2,aokvqa,avg
56.99634282079822,53.79,73.68,70.82969432314411,63.82400928598558
59.68357449515026,54.46,75.16,72.40174672489083,65.42633030501027
60.939736047066305,55.99,75.88,71.87772925764192,66.17186632617705
62.00508824932422,58.43,76.67,72.48908296943232,67.39854280468914
62.8716807123549,58.22,77.73,73.88646288209607,68.17703589861274
62.99093655589124,58.08,78.53,74.06113537117905,68.41551798176756
1 gqa okvqa vqav2 aokvqa avg
2 56.99634282079822 53.79 73.68 70.82969432314411 63.82400928598558
3 59.68357449515026 54.46 75.16 72.40174672489083 65.42633030501027
4 60.939736047066305 55.99 75.88 71.87772925764192 66.17186632617705
5 62.00508824932422 58.43 76.67 72.48908296943232 67.39854280468914
6 62.8716807123549 58.22 77.73 73.88646288209607 68.17703589861274
7 62.99093655589124 58.08 78.53 74.06113537117905 68.41551798176756

10
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,10 @@
gqa,okvqa,vqav2,aokvqa,avg
56.940690093814595,54.47,73.98,71.61572052401746,64.25160265445803
60.12879631101924,55.14,75.16,72.31441048034935,65.68580169784215
61.09874383844809,55.84,76.32,73.01310043668123,66.56796106878232
61.95738591190968,58.01,76.9,73.44978165938865,67.57929189282459
62.752424868818565,58.21,77.94,73.1004366812227,68.00071538751033
62.8001272062331,58.29,78.52,74.235807860262,68.46148376662377
63.17379551598028,58.75,79.04,73.88646288209607,68.71256459951908
63.4520591508984,57.67,79.69,73.88646288209607,68.67463050824861
63.5872157735729,58.24,79.92,73.62445414847161,68.84291748051113
1 gqa okvqa vqav2 aokvqa avg
2 56.940690093814595 54.47 73.98 71.61572052401746 64.25160265445803
3 60.12879631101924 55.14 75.16 72.31441048034935 65.68580169784215
4 61.09874383844809 55.84 76.32 73.01310043668123 66.56796106878232
5 61.95738591190968 58.01 76.9 73.44978165938865 67.57929189282459
6 62.752424868818565 58.21 77.94 73.1004366812227 68.00071538751033
7 62.8001272062331 58.29 78.52 74.235807860262 68.46148376662377
8 63.17379551598028 58.75 79.04 73.88646288209607 68.71256459951908
9 63.4520591508984 57.67 79.69 73.88646288209607 68.67463050824861
10 63.5872157735729 58.24 79.92 73.62445414847161 68.84291748051113

11
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
57.14740022261091,54.38,74.13,71.52838427947599,64.29644612552173
59.19860073143583,55.98,75.1,72.5764192139738,65.7137549863524
60.64557163301002,56.56,75.9,73.01310043668123,66.52966801742281
61.3770074733662,57.87,76.6,74.06113537117905,67.47703571113631
62.40260772777866,57.16,77.31,74.32314410480349,67.79893795814553
62.99888694546033,57.96,78.12,73.1877729257642,68.06666496780613
62.69677214183494,59.45,78.75,74.235807860262,68.78314500052424
63.126093178565746,58.74,79.22,74.41048034934498,68.87414338197769
63.30100174908571,58.32,79.5,73.7117903930131,68.7081980355247
63.29305135951662,58.05,79.62,73.7991266375546,68.6905444992678
1 gqa okvqa vqav2 aokvqa avg
2 57.14740022261091 54.38 74.13 71.52838427947599 64.29644612552173
3 59.19860073143583 55.98 75.1 72.5764192139738 65.7137549863524
4 60.64557163301002 56.56 75.9 73.01310043668123 66.52966801742281
5 61.3770074733662 57.87 76.6 74.06113537117905 67.47703571113631
6 62.40260772777866 57.16 77.31 74.32314410480349 67.79893795814553
7 62.99888694546033 57.96 78.12 73.1877729257642 68.06666496780613
8 62.69677214183494 59.45 78.75 74.235807860262 68.78314500052424
9 63.126093178565746 58.74 79.22 74.41048034934498 68.87414338197769
10 63.30100174908571 58.32 79.5 73.7117903930131 68.7081980355247
11 63.29305135951662 58.05 79.62 73.7991266375546 68.6905444992678

11
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
57.521068532358086,53.93,73.93,70.74235807860262,64.03085665274017
60.39910955636826,54.91,75.34,71.70305676855895,65.5880415812318
61.04309111146446,56.33,76.35,73.97379912663754,66.9242225595255
62.17204643027509,57.25,76.97,72.22707423580786,67.15478016652074
62.919383049769436,57.44,77.9,73.44978165938865,67.92729117728952
63.5156622674511,58.08,78.7,74.14847161572052,68.6110334707929
63.96088408332008,58.17,79.22,74.14847161572052,68.87483892476016
64.27889966608363,58.38,79.75,73.44978165938865,68.96467033136807
63.98473525202735,58.32,79.91,73.53711790393012,68.93796328898937
64.46175862617268,57.95,80.05,73.27510917030567,68.93421694911959
1 gqa okvqa vqav2 aokvqa avg
2 57.521068532358086 53.93 73.93 70.74235807860262 64.03085665274017
3 60.39910955636826 54.91 75.34 71.70305676855895 65.5880415812318
4 61.04309111146446 56.33 76.35 73.97379912663754 66.9242225595255
5 62.17204643027509 57.25 76.97 72.22707423580786 67.15478016652074
6 62.919383049769436 57.44 77.9 73.44978165938865 67.92729117728952
7 63.5156622674511 58.08 78.7 74.14847161572052 68.6110334707929
8 63.96088408332008 58.17 79.22 74.14847161572052 68.87483892476016
9 64.27889966608363 58.38 79.75 73.44978165938865 68.96467033136807
10 63.98473525202735 58.32 79.91 73.53711790393012 68.93796328898937
11 64.46175862617268 57.95 80.05 73.27510917030567 68.93421694911959

6
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,6 @@
gqa,okvqa,vqav2,aokvqa,avg
56.932739704245506,54.43,73.9,71.52838427947599,64.19778099593037
60.057242804897434,55.95,75.28,72.31441048034935,65.9004133213117
61.44061058991891,54.26,76.24,73.1877729257642,66.28209587892077
61.806328510096996,57.5,76.83,73.44978165938865,67.3965275423714
62.56956590872953,58.33,77.89,73.7991266375546,68.14717313657103
1 gqa okvqa vqav2 aokvqa avg
2 56.932739704245506 54.43 73.9 71.52838427947599 64.19778099593037
3 60.057242804897434 55.95 75.28 72.31441048034935 65.9004133213117
4 61.44061058991891 54.26 76.24 73.1877729257642 66.28209587892077
5 61.806328510096996 57.5 76.83 73.44978165938865 67.3965275423714
6 62.56956590872953 58.33 77.89 73.7991266375546 68.14717313657103

11
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
56.75783113372555,54.51,73.98,71.87772925764192,64.28139009784186
59.98568929877563,55.07,75.3,72.48908296943232,65.711193067052
60.88408332008268,56.75,76.03,73.97379912663754,66.90947061168006
61.75067578311337,57.82,76.87,73.97379912663754,67.60361872743772
62.442359675624104,57.72,77.97,73.53711790393012,67.91736939488857
62.8716807123549,57.88,78.51,73.97379912663754,68.30886995974811
63.308952138654796,58.55,79.09,74.235807860262,68.7961899997292
63.547463825727455,57.86,79.61,74.67248908296943,68.92248822717423
63.809826681507396,58.42,79.81,74.49781659388647,69.13441081884847
63.722372396247415,57.87,79.93,74.32314410480349,68.96137912526272
1 gqa okvqa vqav2 aokvqa avg
2 56.75783113372555 54.51 73.98 71.87772925764192 64.28139009784186
3 59.98568929877563 55.07 75.3 72.48908296943232 65.711193067052
4 60.88408332008268 56.75 76.03 73.97379912663754 66.90947061168006
5 61.75067578311337 57.82 76.87 73.97379912663754 67.60361872743772
6 62.442359675624104 57.72 77.97 73.53711790393012 67.91736939488857
7 62.8716807123549 57.88 78.51 73.97379912663754 68.30886995974811
8 63.308952138654796 58.55 79.09 74.235807860262 68.7961899997292
9 63.547463825727455 57.86 79.61 74.67248908296943 68.92248822717423
10 63.809826681507396 58.42 79.81 74.49781659388647 69.13441081884847
11 63.722372396247415 57.87 79.93 74.32314410480349 68.96137912526272

11
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,11 @@
gqa,okvqa,vqav2,aokvqa,avg
56.97249165209095,54.35,73.91,70.56768558951964,63.950044310402646
60.24805215455557,55.26,75.14,72.5764192139738,65.80611784213234
60.8681825409445,55.82,76.32,74.58515283842794,66.89833384484311
61.99713785975513,57.76,76.9,73.44978165938865,67.52672987978595
62.76832564795674,58.45,77.85,73.7991266375546,68.21686307137783
63.16584512641119,57.87,78.61,73.53711790393012,68.29574075758532
63.84162823978375,58.28,79.01,73.53711790393012,68.66718653592847
64.31865161392908,58.35,79.56,73.27510917030567,68.87594019605869
64.41405628875815,58.76,79.68,73.44978165938865,69.0759594870367
64.62076641755446,58.6,79.84,73.88646288209607,69.23680732491263
1 gqa okvqa vqav2 aokvqa avg
2 56.97249165209095 54.35 73.91 70.56768558951964 63.950044310402646
3 60.24805215455557 55.26 75.14 72.5764192139738 65.80611784213234
4 60.8681825409445 55.82 76.32 74.58515283842794 66.89833384484311
5 61.99713785975513 57.76 76.9 73.44978165938865 67.52672987978595
6 62.76832564795674 58.45 77.85 73.7991266375546 68.21686307137783
7 63.16584512641119 57.87 78.61 73.53711790393012 68.29574075758532
8 63.84162823978375 58.28 79.01 73.53711790393012 68.66718653592847
9 64.31865161392908 58.35 79.56 73.27510917030567 68.87594019605869
10 64.41405628875815 58.76 79.68 73.44978165938865 69.0759594870367
11 64.62076641755446 58.6 79.84 73.88646288209607 69.23680732491263

9
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,9 @@
gqa,okvqa,vqav2,aokvqa,avg
56.940690093814595,54.81,73.85,71.09170305676857,64.17309828764579
60.20034981714104,55.52,75.37,72.5764192139738,65.9166922577787
61.19414851327715,54.56,76.38,73.27510917030567,66.35231442089571
61.217999681984416,57.83,76.88,73.36244541484717,67.3226112742079
62.64111941485133,57.97,77.97,72.92576419213974,67.87672090174777
63.03863889330577,58.54,78.49,72.75109170305677,68.20493264909064
63.61106694228017,58.49,79.1,74.06113537117905,68.8155505783648
63.66671966926379,58.68,79.56,72.92576419213974,68.70812096535089
1 gqa okvqa vqav2 aokvqa avg
2 56.940690093814595 54.81 73.85 71.09170305676857 64.17309828764579
3 60.20034981714104 55.52 75.37 72.5764192139738 65.9166922577787
4 61.19414851327715 54.56 76.38 73.27510917030567 66.35231442089571
5 61.217999681984416 57.83 76.88 73.36244541484717 67.3226112742079
6 62.64111941485133 57.97 77.97 72.92576419213974 67.87672090174777
7 63.03863889330577 58.54 78.49 72.75109170305677 68.20493264909064
8 63.61106694228017 58.49 79.1 74.06113537117905 68.8155505783648
9 63.66671966926379 58.68 79.56 72.92576419213974 68.70812096535089

7
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv Normal file
View File

@ -0,0 +1,7 @@
gqa,okvqa,vqav2,aokvqa,avg
57.020193989505486,53.7,73.8,70.21834061135371,63.6846336502148
60.33550643981555,54.9,75.12,72.40174672489083,65.68931329117659
60.947686436635394,55.55,76.29,73.01310043668123,66.45019671832915
61.45651136905709,57.24,76.39,73.62445414847161,67.17774137938218
62.65702019398951,57.34,77.73,75.37117903930131,68.2745498083227
63.5156622674511,57.95,78.4,73.62445414847161,68.37252910398068
1 gqa okvqa vqav2 aokvqa avg
2 57.020193989505486 53.7 73.8 70.21834061135371 63.6846336502148
3 60.33550643981555 54.9 75.12 72.40174672489083 65.68931329117659
4 60.947686436635394 55.55 76.29 73.01310043668123 66.45019671832915
5 61.45651136905709 57.24 76.39 73.62445414847161 67.17774137938218
6 62.65702019398951 57.34 77.73 75.37117903930131 68.2745498083227
7 63.5156622674511 57.95 78.4 73.62445414847161 68.37252910398068

9
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv Normal file
View File

@ -0,0 +1,9 @@
gqa,okvqa,vqav2,aokvqa,avg
57.33025918269995,54.6,74.14,70.91703056768559,64.24682243759638
59.643822547304815,55.57,75.28,72.13973799126637,65.6583901346428
60.7966290348227,56.62,76.23,74.14847161572052,66.9487751626358
61.806328510096996,57.53,76.92,73.88646288209607,67.53569784804826
62.67292097312769,57.78,78.04,74.49781659388647,68.24768439175354
63.05453967244395,57.7,78.55,73.44978165938865,68.18858033295815
63.16584512641119,58.84,79.15,73.7991266375546,68.73874294099144
63.75417395452377,58.02,79.59,73.7117903930131,68.76899108688423
1 gqa okvqa vqav2 aokvqa avg
2 57.33025918269995 54.6 74.14 70.91703056768559 64.24682243759638
3 59.643822547304815 55.57 75.28 72.13973799126637 65.6583901346428
4 60.7966290348227 56.62 76.23 74.14847161572052 66.9487751626358
5 61.806328510096996 57.53 76.92 73.88646288209607 67.53569784804826
6 62.67292097312769 57.78 78.04 74.49781659388647 68.24768439175354
7 63.05453967244395 57.7 78.55 73.44978165938865 68.18858033295815
8 63.16584512641119 58.84 79.15 73.7991266375546 68.73874294099144
9 63.75417395452377 58.02 79.59 73.7117903930131 68.76899108688423

9
process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv Normal file
View File

@ -0,0 +1,9 @@
gqa,okvqa,vqav2,aokvqa,avg
53.46636985212275,48.02,71.28,73.27510917030567,61.5103697556071
55.525520750516776,50.12,72.37,74.32314410480349,63.08466621383006
56.91683892510733,49.66,73.78,75.54585152838428,63.9756726133729
57.616473207187155,52.98,74.3,76.85589519650655,65.43809210092343
58.538718397201464,52.81,74.97,75.9825327510917,65.57531278707329
59.54841787247575,52.71,75.53,76.68122270742359,66.11741014497483
59.60407059945937,53.43,75.81,76.68122270742359,66.38132332672075
59.68357449515026,53.71,75.99,77.117903930131,66.6253696063203
1 gqa okvqa vqav2 aokvqa avg
2 53.46636985212275 48.02 71.28 73.27510917030567 61.5103697556071
3 55.525520750516776 50.12 72.37 74.32314410480349 63.08466621383006
4 56.91683892510733 49.66 73.78 75.54585152838428 63.9756726133729
5 57.616473207187155 52.98 74.3 76.85589519650655 65.43809210092343
6 58.538718397201464 52.81 74.97 75.9825327510917 65.57531278707329
7 59.54841787247575 52.71 75.53 76.68122270742359 66.11741014497483
8 59.60407059945937 53.43 75.81 76.68122270742359 66.38132332672075
9 59.68357449515026 53.71 75.99 77.117903930131 66.6253696063203

18
process_log/results_text_cap.csv
View File

@ -1,11 +1,9 @@
Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr
0.7354185510459915,0.5532517544390539,0.40861793000486973,0.29983800193167554,0.24116643313333005,0.49331369213983123,1.0410874398111594
0.7337559429476805,0.5532662736631931,0.4093585176779233,0.2995878219607148,0.24114315075166085,0.49493696747386196,1.0409495699510138
0.7320402508385379,0.5518304887400776,0.4110219411525107,0.3038313373404956,0.24291044377257376,0.496058725623593,1.0575894770967331
0.735500933603824,0.557825793289975,0.41590254371174706,0.30692236406930773,0.2383527664105175,0.49362765900900546,1.044158754983385
0.7398393194706776,0.5620893308030489,0.41820732364444946,0.3078230351155729,0.24483177062749348,0.49847300976639547,1.0688814357108025
0.7371954616083516,0.5585083902477329,0.4158548739958792,0.30707857737975475,0.244730703390385,0.49852757085109406,1.0729374604921766
0.743162583518909,0.5634215669347339,0.42011923174763044,0.311173553245093,0.2441994031592775,0.49809622365158585,1.0774442793527572
0.7394105997457735,0.5596190520647151,0.4174127065616514,0.3086643930692146,0.24516092911412224,0.4984150187439041,1.0745130576356021
0.7401475669028302,0.5612229049005432,0.4184684096329846,0.3101423931360949,0.2459814560570504,0.49998501882495094,1.0809003006778692
0.7420413230468877,0.5634982579203833,0.4203003411022801,0.3115589386663839,0.245912807863165,0.4989393568537978,1.0805529900845852
0.7309287396242824,0.5494655932407475,0.4041141975269019,0.2921841096479282,0.23361135981392955,0.48650322014232156,0.9966076482929254
0.7382464874712654,0.5571950857230415,0.4121904524343956,0.30143019979529023,0.23826500378661586,0.4926618553461225,1.0232964787151213
0.7350922242488682,0.5559390933443953,0.4113206176871192,0.3009155411187479,0.2374533115810085,0.49274408438489914,1.0099444415721002
0.7369161501554021,0.556728672691215,0.41086927244074317,0.3004344996181908,0.23806683186363936,0.49184279207452697,1.0280928870205412
0.7407251133040751,0.5595661891353809,0.4138229944446712,0.3032027996796593,0.2415473861083878,0.4972778899306893,1.0479529588574925
0.7428951836605517,0.5623862396980985,0.4161405396490859,0.3047709156231164,0.2438504314752837,0.49858613995817086,1.0477488915322588
0.745208695011462,0.5654882473522628,0.4200936310645993,0.308981454910347,0.24371581659378194,0.49914038443933745,1.053786374859757
0.7444798335079819,0.5624985319454049,0.4157842492239835,0.3044642812661473,0.2432655219581268,0.49722674885535495,1.056752744677739

1 Bleu_1 Bleu_2 Bleu_3 Bleu_4 METEOR ROUGE_L CIDEr
2 0.7354185510459915 0.7309287396242824 0.5532517544390539 0.5494655932407475 0.40861793000486973 0.4041141975269019 0.29983800193167554 0.2921841096479282 0.24116643313333005 0.23361135981392955 0.49331369213983123 0.48650322014232156 1.0410874398111594 0.9966076482929254
3 0.7337559429476805 0.7382464874712654 0.5532662736631931 0.5571950857230415 0.4093585176779233 0.4121904524343956 0.2995878219607148 0.30143019979529023 0.24114315075166085 0.23826500378661586 0.49493696747386196 0.4926618553461225 1.0409495699510138 1.0232964787151213
4 0.7320402508385379 0.7350922242488682 0.5518304887400776 0.5559390933443953 0.4110219411525107 0.4113206176871192 0.3038313373404956 0.3009155411187479 0.24291044377257376 0.2374533115810085 0.496058725623593 0.49274408438489914 1.0575894770967331 1.0099444415721002
5 0.735500933603824 0.7369161501554021 0.557825793289975 0.556728672691215 0.41590254371174706 0.41086927244074317 0.30692236406930773 0.3004344996181908 0.2383527664105175 0.23806683186363936 0.49362765900900546 0.49184279207452697 1.044158754983385 1.0280928870205412
6 0.7398393194706776 0.7407251133040751 0.5620893308030489 0.5595661891353809 0.41820732364444946 0.4138229944446712 0.3078230351155729 0.3032027996796593 0.24483177062749348 0.2415473861083878 0.49847300976639547 0.4972778899306893 1.0688814357108025 1.0479529588574925
7 0.7371954616083516 0.7428951836605517 0.5585083902477329 0.5623862396980985 0.4158548739958792 0.4161405396490859 0.30707857737975475 0.3047709156231164 0.244730703390385 0.2438504314752837 0.49852757085109406 0.49858613995817086 1.0729374604921766 1.0477488915322588
8 0.743162583518909 0.745208695011462 0.5634215669347339 0.5654882473522628 0.42011923174763044 0.4200936310645993 0.311173553245093 0.308981454910347 0.2441994031592775 0.24371581659378194 0.49809622365158585 0.49914038443933745 1.0774442793527572 1.053786374859757
9 0.7394105997457735 0.7444798335079819 0.5596190520647151 0.5624985319454049 0.4174127065616514 0.4157842492239835 0.3086643930692146 0.3044642812661473 0.24516092911412224 0.2432655219581268 0.4984150187439041 0.49722674885535495 1.0745130576356021 1.056752744677739
0.7401475669028302 0.5612229049005432 0.4184684096329846 0.3101423931360949 0.2459814560570504 0.49998501882495094 1.0809003006778692
0.7420413230468877 0.5634982579203833 0.4203003411022801 0.3115589386663839 0.245912807863165 0.4989393568537978 1.0805529900845852

149
test.py Normal file
View File

@ -0,0 +1,149 @@
# from math import factorial
# def combine_number(n,k):
# combinations = factorial(n) / factorial(n - k)
# return combinations
# # 定义一个函数来计算给定字符串中平衡串的子序列数量
# def count_balanced_subsequences(s):
# MOD = 10**9 + 7
# # 初始化字母计数器
# count = [0] * 26
# for char in s:
# count[ord(char) - ord('a')] += 1
# result = 0
# while sum(count)>0:
# cnt = 0
# for i in range(26):
# if count[i] > 0:
# cnt += 1
# count[i] -= 1
# print(count)
# result += combine_number(cnt,2)
# print(result)
# return result
# # 示例输入
# n = 5
# s = "ababc"
# # 计算结果
# result = count_balanced_subsequences(s)
# print(result)
# def calculate_scores(k, x, y):
# # 根据题目描述，我们有以下等式：
# # a + b + c = k
# # c = a + x
# # c = b - y
# # 由上面两个等式可得：a + x = b - y
# # 因此我们可以解这个线性方程组得到a, b, c的值
# # a + (a + x) + (a + x + y) = k
# # 3a + 2x + y = k
# # 解得：
# a = (k - 2*x - y) // 3
# c = a + x
# b = c + y
# return a, b, c
# # 示例输入
# k, x, y = 441, 1, -20
# # 计算输出
# a, b, c = calculate_scores(k, x, y)
# print(a, b, c)
# def min_operations_to_equal(s, t):
# operations = []
# i = len(s) - 1
# while i >= 0:
# if s[i] != t[i]:
# operations.append((1, i + 1, t[i]))
# s = t[i] * (i + 1) + s[i + 1:] # 更新s字符串为操作后的状态
# i -= 1
# return operations
# # 示例输入
# s = "aabc"
# t = "abcc"
# # 计算所需的最小操作次数及具体操作
# operations1 = min_operations_to_equal(s, t)
# operations2 = min_operations_to_equal(t, s)
# if len(operations1) < len(operations2):
# operations = operations1
# else:
# operations = operations2
# # 输出结果
# print(len(operations))
# for op in operations:
# print(*op)
# #include <iostream>
# #include <linux/limits,h>
# #include <string>
# using namespace std;
# struct node {
# char a, b;
# }ans[100100];
# char col[9] = {'','a','b'
# int main(){
# string x; cin >>x;
# char w=x[0],s=x[1];
# int ww=w-'a'+ 1;
# int ss =s -'0';
# int tot = 0;
# for(int i=1;i<=8; ++i){
# if(1 != ss){
# ans[++tot]=(node){w,(char)(i +'@')};
# }
# }
# for(int i=1; i <= 8; ++i){
# if(1 != ww){
# ans[++tot]=(node){colli], s};
# }
# }
# for(int i=1;i<8;++1){
# for(int j=1;j<=8; ++j){
# if(i+j==ww + ss){
# if(i !=ss |lj != ww){
# ans[++tot]=(node){col[j],(char)}
# }
# }
# }
# }
# for(int i=1;i<=8;++i){
# for(int j=1;j<=8; ++j){
# if(i-j=ss-ww){
# if(i !=ss ||j!= ww){
# ans[++tot]=(node)fcol[j]，(char)
# }
# }
# }
# }
# cout << tot.<< endl;
# for(int i =1; i <= tot; ++i) cout << ans[i].a <<
# return 0;
# #include <linux/limits.h>
# #include <string>using namespace stdj
# const int N= 1001008
# int n, a[N];
# int main(){
# cin >> n;
# int tot1 = 0, tot0 = 0;
# for(int i=1;i<= n; ++1){
# cin >> a[i];
# if(a[i]%2=0) tot0++;
# else tot1++;
# cout << abs(tot1-(n/2))<<endl;
# return 0;

5
test_text_cap.py
View File

@ -1,5 +0,0 @@
from minigpt4.common.caption_tools.caption_utils import coco_caption_eval, textcaps_caption_eval
result_file = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_12epo_0317/20240317165/result/val_vqa_result_text_cap.json"
annotaion_file = "/mnt/pfs-guan-ssai/nlu/wanghanzi/data/TextCap/TextCaps_0.1_val.json"
eval = textcaps_caption_eval(annotaion_file, result_file)
print(eval.eval.item())