0401 cls cross/route/gate/route (universal)

minigpt4/datasets/datasets/aok_vqa_datasets.py

@@ -89,8 +89,8 @@ class AOKVQADataset(VQADataset, __DisplMixin):
         return {
             "image": data['image'],
             "image_id": data["image_id"],
-            # "q_input": q_input,
+            "q_input": q_input,
-            "q_input": llm_input,
+            # "q_input": llm_input,
             "llm_input": llm_input,
             "text_input": question,
             "text_output": answer,
@@ -164,8 +164,8 @@ class AOKVQAEvalDataset(VQAEvalDataset, __DisplMixin):
             "correct_choice_idx": correct_choice_idx_list,
             "direct_answers": direct_answers_list,
             "llm_input": llm_input_list,
-            "q_input": llm_input_list,
+            # "q_input": llm_input_list,
-            # "q_input": q_input_list,
+            "q_input": q_input_list,
             "gt_answers": gt_answers_list,
             "source": source_list,
         }
@@ -208,13 +208,13 @@ class AOKVQAEvalDataset(VQAEvalDataset, __DisplMixin):
             correct_choice_idx = None
             correct_answer = direct_answers
 
-        llm_input = ann.get("llm_input",random.choice(self.instruction_pool).format(question))
+        llm_input = ann.get("llm_input",random.choice(self.instruction_pool).format(question, ", ".join(choices)))
         # llm_input = random.choice(self.instruction_pool).format(question, ", ".join(choices))
 
         return {
             "image": image,
-            # "q_input": question,
+            "q_input": question,
-            "q_input": llm_input,
+            # "q_input": llm_input,
             "llm_input": llm_input,
             "text_input": question,
             "question_id": ann["question_id"],

minigpt4/datasets/datasets/coco_vqa_datasets.py

@@ -96,8 +96,8 @@ class COCOVQADataset(VQADataset, __DisplMixin):
             "image": data['image'],
             "image_id": data["image_id"],
             "question_id": data["question_id"],
-            # "q_input": q_input,
+            "q_input": q_input,
-            "q_input": llm_input,
+            # "q_input": llm_input,
             "llm_input": llm_input,
             "text_input": question,
             "text_output": answer,
@@ -181,7 +181,7 @@ class COCOVQAEvalDataset(VQAEvalDataset, __DisplMixin):
             "question_id": ann["question_id"],
             # "instance_id": ann["instance_id"],
             # "question": question,
-            "q_input": llm_input,
+            # "q_input": llm_input,
             "q_input": q_input,
             "llm_input": llm_input,
             "text_input": question,

minigpt4/datasets/datasets/gqa_datasets.py

@@ -75,8 +75,8 @@ class GQADataset(VQADataset, __DisplMixin):
             # "text_output": ann["fullAnswer"],
             "text_output": answers,
             # "instruction_input": instruction,
-            "q_input": llm_input,
+            # "q_input": llm_input,
-            # "q_input": q_input,
+            "q_input": q_input,
             "llm_input": llm_input,
             "gt_answers": answers,
             "source": "gqa",
@@ -150,8 +150,8 @@ class GQAEvalDataset(VQAEvalDataset, __DisplMixin):
             "gt_answers": answer,
             "fullAnswer": fullAnswer,
             "text_output": answer,
-            # "q_input": q_input,
+            "q_input": q_input,
-            "q_input": llm_input,
+            # "q_input": llm_input,
             "llm_input": llm_input,
             "question_id": ann["question_id"],
             # "instance_id": ann["instance_id"],

minigpt4/datasets/datasets/ok_vqa_datasets.py

@@ -98,8 +98,8 @@ class OKVQADataset(VQADataset, __DisplMixin):
             "image_id": data["image_id"],
             "question_id": data["question_id"],
             # "instruction_input": instruction,
-            "q_input": llm_input,
+            # "q_input": llm_input,
-            # "q_input": q_input,
+            "q_input": q_input,
             "llm_input": llm_input,
             "text_input": question,
             "text_output": answer,
@@ -181,8 +181,8 @@ class OKVQAEvalDataset(VQAEvalDataset, __DisplMixin):
             'image_path': image_path,
             "question_id": ann["question_id"],
             "question": question,
-            # "q_input": q_input,
+            "q_input": q_input,
-            "q_input": llm_input,
+            # "q_input": llm_input,
             "llm_input": llm_input,
             "text_input": question,
             "source": 'okvqa',

minigpt4/eval_scripts/eval_vqa.py

@@ -29,10 +29,21 @@ def list_of_str(arg):
 
 def eval_parser():
     parser = argparse.ArgumentParser(description="Demo")
+    parser.add_argument(
+        "--device",
+        default=0,
+        help="device to run the model",
+    )
     parser.add_argument(
         "--cfg-path", 
         default="/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/minigpt4/projects/qformer_moe_vicuna/eval/vqa_benchmark_evaluation.yaml",
         help="path to configuration file.")
+    parser.add_argument(
+        "--dataset",
+        default=['vizwiz','hm'],
+        type=list_of_str,
+        help="dataset to evaluate",
+    )
     parser.add_argument(
         "--options",
         nargs="+",
@@ -58,7 +69,6 @@ def init_model(cfg, device):
     model_cls = registry.get_model_class(model_config.arch)
     model = model_cls.from_config(model_config).to(device)
 
-#     import pudb; pudb.set_trace()
     key = list(cfg.datasets_cfg.keys())[0]
     vis_processor_cfg = cfg.datasets_cfg.get(key).vis_processor.train
     vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg)
@@ -68,13 +78,12 @@ def init_model(cfg, device):
     return model, vis_processor, text_processor
 
 parser = eval_parser()
-parser.add_argument("--dataset", type=list_of_str, default=['vizwiz','hm'], help="dataset to evaluate")
 args = parser.parse_args()
 cfg = Config(args)
 setup_seeds(cfg)
 print(cfg._convert_node_to_json(cfg.config))
 setup_logger()
-device = torch.device("cuda:5" if torch.cuda.is_available() else "cpu")
+device = torch.device("cuda:{}".format(args.device) if torch.cuda.is_available() else "cpu")
 
 model, vis_processor, _ = init_model(cfg, device)
 model.eval()

minigpt4/models/QformerRouteMoECLS.py

@@ -530,7 +530,7 @@ class BertLayer(nn.Module):
                     self.seq_len_dim,
                     attention_output[:, query_length:, :],
                 )
-                cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size]
+                cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size]
 
             # add moe query ffn
             moe_ffn_attention_input = query_attention_output[:, :query_length, :] # [bz, query_length+seq_len, 768]

minigpt4/models/QformerRouteMoECLSLN.py

@@ -524,7 +524,7 @@ class BertLayer(nn.Module):
                     self.seq_len_dim,
                     attention_output[:, query_length:, :],
                 )
-                cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size]
+                cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size]
 
             # add moe query ffn
             moe_ffn_attention_input = query_attention_output[:, :query_length, :] # [bz, query_length+seq_len, 768]

minigpt4/models/QformerRouteMoELNUni.py

@@ -524,7 +524,7 @@ class BertLayer(nn.Module):
                     self.seq_len_dim,
                     attention_output[:, query_length:, :],
                 )
-                cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size]
+                cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size]
 
             # add moe query ffn
             # query_attention_output size: [bz, query_length+seq_len, 768]

minigpt4/models/QformerRouteMoEUni.py

@@ -531,7 +531,7 @@ class BertLayer(nn.Module):
                     self.seq_len_dim,
                     attention_output[:, query_length:, :],
                 )
-                cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size]
+                cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size]
 
             # add moe query ffn
             # query_attention_output size: [bz, query_length+seq_len, 768]

minigpt4/models/moe/beam_search_universal.py

@@ -22,7 +22,7 @@ class UniRouteMoELayer(nn.Module):
         self.route_method = route_method
         if self.route_method == "pre-route-uni":
             self.gate = nn.Linear(hidden_size, num_experts, bias=False).float()
-        elif self.route_method in ["post-route-uni"]:
+        elif self.route_method in ["post-route-uni",'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
             gate = nn.Linear(hidden_size, 1, bias=False).float()
             self.gate = gate
 
@@ -38,7 +38,7 @@ class UniRouteMoELayer(nn.Module):
 
 
     def beam_search(self, current_scores_log, beam_scores, expert_route, batch_size):
-        if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni']:
+        if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni','uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
             # current_scores_log torch.Size([bz, num_experts-1])
             assert beam_scores==None and expert_route==None
             current_scores = torch.exp(current_scores_log)
@@ -143,7 +143,32 @@ class UniRouteMoELayer(nn.Module):
         # import pdb;pdb.set_trace()
         return candidate_output, beam_scores, expert_route, beam_idx, importance_loss
 
-    def forward_post_route_uni(self, x, beam_scores, expert_route, use_log=True):
+
+    def calculate_cls_gate_score(self, cls_hidden, output_x):
+
+        if self.route_method == 'uni-cls-route':
+            # cls_hidden = [bz, 768]
+            gate_score = self.gate(cls_hidden) # bz, 1
+        elif self.route_method == 'uni-cls-query-route': # add cls_hiddin on query_token mean pool hidden
+            mean_output = torch.mean(output_x, dim=1) # bz, 768
+            gate_score = self.gate(mean_output+cls_hidden) # bz, 1
+        elif self.route_method == 'uni-cls-cross-route':
+            # cls_hidden as Q, output_x as K, V calculate scaled dot-product attention between Q and K and V
+            # cls_hidden: bz, 768
+            # output_x: bz, 32, 768
+            Q = cls_hidden.unsqueeze(1) # bz, 1, 768
+            K = output_x # bz, 32, 768
+            V = output_x # bz, 32, 768
+            # scaled dot-product attention
+            QK = torch.matmul(Q, K.transpose(-1, -2)) / (K.size(-1) ** 0.5) # bz, 1, 32
+            QK = F.softmax(QK, dim=-1) # bz, 1, 32
+            gate_score = torch.matmul(QK, V) # bz, 1, 768
+            gate_score = gate_score.squeeze(1) # bz, 768
+            gate_score = self.gate(gate_score) # bz, 1
+        return gate_score
+
+
+    def forward_route_uni(self, x, beam_scores, expert_route, use_log=True, cls_hidden=None):
         
         if beam_scores == None:
             batch_size = x.shape[0]
@@ -155,8 +180,6 @@ class UniRouteMoELayer(nn.Module):
             x_masked, x_uniexpert = x[select_expert],x[select_universal]
         num_tokens = x.shape[1]
 
-        import pdb; pdb.set_trace()
-
         def forward_expert(input_x, expert_idx):
             output_x = self.experts[expert_idx].forward(input_x)
             return output_x
@@ -168,8 +191,14 @@ class UniRouteMoELayer(nn.Module):
         logits_gate_lst = list()
         for expert_idx in range(self.num_route_experts): # num_expert-1
             output_x = forward_expert(x_masked, expert_idx)
-            output_x_aver = torch.mean(output_x, dim=1)
+
-            gate_score = self.gate(output_x_aver)
+            if self.route_method == 'post-route-uni':
+                output_x_aver = torch.mean(output_x, dim=1)
+                gate_score = self.gate(output_x_aver)
+
+            elif self.route_method in ['uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route'] and cls_hidden is not None:
+                gate_score = self.calculate_cls_gate_score(cls_hidden, output_x)
+
             logits_gate_lst.append(gate_score)
             outputs.append(output_x.unsqueeze(0))
 
@@ -186,14 +215,12 @@ class UniRouteMoELayer(nn.Module):
         # beam_scores torch.Size([bz*(num_beam-1)]), expert_route torch.Size([bz*(num_beam-1), layer_n])
         current_select_expert = expert_route[:,-1] # torch.Size([bz*(num_beam-1)])
 
-        import pdb; pdb.set_trace()
         if self.layer_judge == 'first':
             replicated_tensor = candidate_output_raw.unsqueeze(2).expand(self.num_route_experts, batch_size, self.num_route_beam, num_tokens, self.hidden_size)
             candidate_output_raw = replicated_tensor.contiguous().view(self.num_route_experts, -1, num_tokens, self.hidden_size) # [bz*num_beams, 32,768]
             current_scores_t = current_scores.unsqueeze(1).expand(batch_size, self.num_route_beam, self.num_route_experts)
             current_scores = current_scores_t.contiguous().view(-1, self.num_route_experts) # [bz*(num_beams-1), num_experts-1]
         
-        import pdb; pdb.set_trace()
         candidate_output = candidate_output_raw.permute(1, 0, 2, 3)[beam_idx] # torch.Size([8, 2, 32, 768])
         expert_select_matrix = F.one_hot(current_select_expert, self.num_route_experts)
         if self.weight_type == 'ffn_prob':
@@ -203,8 +230,6 @@ class UniRouteMoELayer(nn.Module):
             output = candidate_output * expert_select_matrix.unsqueeze(-1).unsqueeze(-1)
         experts_output = torch.sum(output, dim=1) # [bz*num_beams-1, 32, 768]
 
-        import pdb; pdb.set_trace()
-
         ####################
         ### universal expert
         ####################
@@ -220,26 +245,26 @@ class UniRouteMoELayer(nn.Module):
             output.append(combine_tmp)
         final_output = torch.cat(output) # [bz*num_beam, 32 ,768]
 
-        import pdb; pdb.set_trace()
+        # import pdb; pdb.set_trace()
 
         return final_output, beam_scores, expert_route, beam_idx, importance_loss
     
-    def forward(self, x, attention_mask, beam_scores, expert_route, use_log=True):
+    def forward(self, x, attention_mask, beam_scores, expert_route, cls_hidden):
         """
             if first_layer: x [bz, 32, 768]
             else: x [bz*num_beams, 32, 768]
         """
         if self.route_method == 'pre-route-uni':
             candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_pre_route(x, beam_scores, expert_route, use_log=True)
-        elif self.route_method in ['post-route-uni']:
+        elif self.route_method in ['post-route-uni', 'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
-            candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_post_route_uni(x, beam_scores, expert_route, use_log=True)
+            candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_route_uni(x, beam_scores, expert_route, use_log=True, cls_hidden=cls_hidden)
 
-        import pdb;pdb.set_trace()
         return candidate_output, beam_scores, expert_route, beam_idx, importance_loss
 
 
 
 
+
 if __name__ == '__main__':
 
     import sys
@@ -305,7 +330,7 @@ if __name__ == '__main__':
                     num_experts=config.moebert_expert_num,
                     num_beams=config.moebert_num_beams,
                     layer_judge = layer_judge,
-                    route_method = "post-route-uni",
+                    route_method = "uni-cls-cross-rout[e",
                     weight_type="ffn_prob"
                 )
         layer_output = experts_post(x2, None, beam_scores2, expert_route2, False)

minigpt4/models/moe/uniroute_moe_layer.py

@@ -19,9 +19,9 @@ class UniRouteMoELayer(nn.Module):
         self.weight_type = weight_type
 
         self.route_method = route_method
-        if self.route_method == "pre-route-uni":
+        if self.route_method in ['pre-route-uni', 'uni-cls-gate-route']:
-            self.gate = nn.Linear(hidden_size, num_experts, bias=False).float()
+            self.gate = nn.Linear(hidden_size, self.num_route_experts, bias=False).float()
-        elif self.route_method in ["post-route-uni"]:
+        elif self.route_method in ["post-route-uni",'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
             gate = nn.Linear(hidden_size, 1, bias=False).float()
             self.gate = gate
 
@@ -35,9 +35,8 @@ class UniRouteMoELayer(nn.Module):
         # Compute coefficient of variation (i.e. std/mean) squared.
         return (std_importance_per_expert / mean_importance_per_expert)**2
 
-
     def beam_search(self, current_scores_log, beam_scores, expert_route, batch_size):
-        if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni']:
+        if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni','uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route','uni-cls-gate-route']:
             # current_scores_log torch.Size([bz, num_experts-1])
             assert beam_scores==None and expert_route==None
             current_scores = torch.exp(current_scores_log)
@@ -80,7 +79,6 @@ class UniRouteMoELayer(nn.Module):
 
         return beam_scores, expert_route, beam_idx
 
-
     def forward_gate(self, x):
         """
             TODO: Pre forward gate
@@ -166,8 +164,16 @@ class UniRouteMoELayer(nn.Module):
             gate_score = self.gate(gate_score) # bz, 1
         return gate_score
 
+    def adjust_cls_hidden(self, cls_hidden, output_x):
+        if cls_hidden.shape[0]/self.num_beams == output_x.shape[0]/self.num_route_beam:
+            cls_hidden_lst = list()
+            for i in range(cls_hidden.shape[0]):
+                if i % self.num_beams != 0:
+                    cls_hidden_lst.append(cls_hidden[i,:])
+            cls_hidden = torch.stack(cls_hidden_lst)
+        return cls_hidden
 
-    def forward_route_uni(self, x, beam_scores, expert_route, use_log=True, cls_hidden=None):
+    def forward_route_uni(self, x, beam_scores, expert_route, cls_hidden=None):
         
         if beam_scores == None:
             batch_size = x.shape[0]
@@ -186,6 +192,9 @@ class UniRouteMoELayer(nn.Module):
         ####################
         ### route expert
         ####################
+        if cls_hidden is not None:
+            cls_hidden = self.adjust_cls_hidden(cls_hidden, x_masked)
+
         outputs = list()
         logits_gate_lst = list()
         for expert_idx in range(self.num_route_experts): # num_expert-1
@@ -194,20 +203,24 @@ class UniRouteMoELayer(nn.Module):
             if self.route_method == 'post-route-uni':
                 output_x_aver = torch.mean(output_x, dim=1)
                 gate_score = self.gate(output_x_aver)
+                logits_gate_lst.append(gate_score)
 
             elif self.route_method in ['uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route'] and cls_hidden is not None:
                 gate_score = self.calculate_cls_gate_score(cls_hidden, output_x)
-
+                logits_gate_lst.append(gate_score)
-            logits_gate_lst.append(gate_score)
             outputs.append(output_x.unsqueeze(0))
 
         candidate_output_raw = torch.cat(outputs) # torch.Size([num_expert-1, bz*(num_beam-1), 32, 768])
-        logits_gate = torch.cat(logits_gate_lst,dim=1)# torch.Size([bz*(num_beam-1), num_expert-1])
+
-        current_scores = F.softmax(logits_gate, dim=-1) # torch.Size([bz*(num_beam-1), num_expert-1])
+        if self.route_method == 'uni-cls-gate-route':
-        if use_log:
+            # universal expert with cls_hidden state into nn.Linear(768,num_experts-1)
-            current_scores_log = torch.log(current_scores) # 取log之后可以直接相加 torch.Size([bz*(num_beam-1), num_expert-1])
+            logits_gate = self.gate(cls_hidden)
+            current_scores = F.softmax(logits_gate, dim=-1) # torch.Size([bz*(num_beam-1), num_expert-1])
         else:
-            current_scores_log = current_scores
+            logits_gate = torch.cat(logits_gate_lst,dim=1)# torch.Size([bz*(num_beam-1), num_expert-1])
+            current_scores = F.softmax(logits_gate, dim=-1) # torch.Size([bz*(num_beam-1), num_expert-1])
+
+        current_scores_log = torch.log(current_scores) # 取log之后可以直接相加 torch.Size([bz*(num_beam-1), num_expert-1])
         
         importance_loss = self._importance_auxiliary_loss(current_scores)
         beam_scores, expert_route, beam_idx = self.beam_search(current_scores_log, beam_scores, expert_route, current_scores_log.shape[0])
@@ -254,9 +267,9 @@ class UniRouteMoELayer(nn.Module):
             else: x [bz*num_beams, 32, 768]
         """
         if self.route_method == 'pre-route-uni':
-            candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_pre_route(x, beam_scores, expert_route, use_log=True)
+            candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_pre_route(x, beam_scores, expert_route)
-        elif self.route_method in ['post-route-uni', 'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']:
+        elif self.route_method in ['post-route-uni', 'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route','uni-cls-gate-route']:
-            candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_route_uni(x, beam_scores, expert_route, use_log=True, cls_hidden=cls_hidden)
+            candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_route_uni(x, beam_scores, expert_route, cls_hidden=cls_hidden)
 
         return candidate_output, beam_scores, expert_route, beam_idx, importance_loss
 

minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval.yaml

@@ -0,0 +1,143 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+model:
+  arch: blip2_vicuna_instruct
+  model_type: vicuna7b_pretrain
+  load_pretrained: True
+  load_finetuned: True
+  vit_model: eva_clip_g
+  pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
+  finetuned: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/checkpoint_7.pth"
+  q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
+
+  # vit encoder
+  image_size: 224
+  drop_path_rate: 0
+  use_grad_checkpoint: False
+  vit_precision: "fp16"
+
+  # Q-Former
+  num_query_token: 32
+  qformer_text_input: True
+
+  # vicuna
+  llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1"
+  prompt: ""
+  max_txt_len: 256
+  max_output_txt_len: 256
+
+  # freeze
+  freeze_vit: True
+  freeze_llm: True
+  freeze_qformer: False
+  freeze_t5_proj: False
+
+  # moe
+  general_version: 'uni_route_moe'
+  moebert_route_method: 'uni-cls-cross-route'
+  moebert_load_balance: 0.01
+  moebert_expert_num: 4
+  moebert_num_beams: 4
+  moe_weight_type: 'ffn_prob'
+  use_balance_loss: False
+  ln_position: "out"
+
+datasets:
+  # gqa:
+  #   type: balanced_sft_raw_eval
+  #   batch_size: 32
+  #   vis_processor:
+  #     eval:
+  #       name: "blip2_image_eval"
+  #       image_size: 224
+  #   text_processor:
+  #     eval:
+  #       name: "blip_caption"
+
+  # coco_vqa:    # 658104
+  #   type: vqa_v2_eval
+  #   batch_size: 32
+  #   vis_processor:
+  #     eval:
+  #       name: "blip2_image_eval"
+  #       image_size: 224
+  #   text_processor:
+  #     eval:
+  #       name: "blip_caption"
+
+  coco_caption: # 414113 train
+    type: coco_cap_eval
+    batch_size: 32
+    vis_processor:
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      eval:
+        name: "blip_caption"
+  
+  # ok_vqa: # train, valid (9009, 5046)
+  #   type: ok_vqa_eval
+  #   batch_size: 32
+  #   vis_processor:
+  #     eval:
+  #       name: "blip2_image_eval"
+  #       image_size: 224
+  #   text_processor:
+  #     eval:
+  #       name: "blip_caption"
+
+  # aok_vqa: # train: 17056, val: 1145
+  #   batch_size: 32
+  #   vis_processor:
+  #     eval:
+  #       name: "blip2_image_eval"
+  #       image_size: 224
+  #   text_processor:
+  #     eval:
+  #       name: "blip_caption"
+
+  textcaps_caption: # train: 109765, val: 15830
+    batch_size: 32
+    vis_processor:
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      eval:
+        name: "blip_caption"
+
+run:
+  task: instruction_tuning
+  # optimizer
+  lr_sched: "linear_warmup_cosine_lr"
+  init_lr: 5e-5
+  min_lr: 1e-6
+  warmup_lr: 1e-6
+  log_freq: 5
+  save_freq: 1500
+
+  weight_decay: 0.05
+  max_epoch: 10
+  num_workers: 4
+  warmup_steps: 600
+  iters_per_epoch: 3000
+
+  seed: 42
+  output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/eval/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/"
+  
+  amp: True
+  resume_ckpt_path: null
+
+  evaluate: True
+  test_splits: ["val"]
+  # test_splits: ["test"]
+
+  device: "cuda"
+  world_size: 1
+  dist_url: "env://"
+  distributed: True
+  

minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval1.yaml

@@ -0,0 +1,113 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+model:
+  arch: blip2_vicuna_instruct
+  model_type: vicuna7b_pretrain
+  load_pretrained: True
+  load_finetuned: True
+  vit_model: eva_clip_g
+  pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
+  finetuned: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/checkpoint_7.pth"
+  q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
+
+  # vit encoder
+  image_size: 224
+  drop_path_rate: 0
+  use_grad_checkpoint: False
+  vit_precision: "fp16"
+
+  # Q-Former
+  num_query_token: 32
+  qformer_text_input: True
+
+  # vicuna
+  llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1"
+  prompt: ""
+  max_txt_len: 256
+  max_output_txt_len: 256
+
+  # freeze
+  freeze_vit: True
+  freeze_llm: True
+  freeze_qformer: False
+  freeze_t5_proj: False
+
+  # moe
+  general_version: 'uni_route_moe'
+  moebert_route_method: 'uni-cls-cross-route'
+  moebert_load_balance: 0.01
+  moebert_expert_num: 4
+  moebert_num_beams: 4
+  moe_weight_type: 'ffn_prob'
+  use_balance_loss: False
+  ln_position: "out"
+
+datasets:
+
+  coco_caption: # 414113 train
+    type: coco_cap_eval
+    batch_size: 32
+    vis_processor:
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      eval:
+        name: "blip_caption"
+  
+  textcaps_caption: # train: 109765, val: 15830
+    batch_size: 32
+    vis_processor:
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      eval:
+        name: "blip_caption"
+
+evaluation_datasets:
+  vizwiz:
+    eval_file_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/VizWiz/val.json
+    img_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/VizWiz/val    
+    max_new_tokens: 20
+    batch_size: 32
+  hm:
+    eval_file_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/hm_data/dev_seen.jsonl
+    img_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/hm_data/
+    max_new_tokens: 20
+    batch_size: 32
+
+run:
+  task: instruction_tuning
+  # optimizer
+  lr_sched: "linear_warmup_cosine_lr"
+  init_lr: 5e-5
+  min_lr: 1e-6
+  warmup_lr: 1e-6
+  log_freq: 5
+  save_freq: 1500
+
+  weight_decay: 0.05
+  max_epoch: 10
+  num_workers: 4
+  warmup_steps: 600
+  iters_per_epoch: 3000
+
+  seed: 42
+  output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/eval/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/"
+  
+  amp: True
+  resume_ckpt_path: null
+
+  evaluate: True
+  # test_splits: ["val"]
+  test_splits: ["test"]
+
+  device: "cuda"
+  world_size: 1
+  dist_url: "env://"
+  distributed: True
+  

minigpt4/projects/qformer_moe_cls_vicuna/eval/mix_vqa_coco_vicuna_eval_coco_vqa_test.yaml

@@ -0,0 +1,89 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+model:
+  arch: blip2_vicuna_instruct
+  model_type: vicuna7b_pretrain
+  load_pretrained: True
+  load_finetuned: True
+  vit_model: eva_clip_g
+  pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
+  finetuned: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/checkpoint_7.pth"
+  q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
+
+  # vit encoder
+  image_size: 224
+  drop_path_rate: 0
+  use_grad_checkpoint: False
+  vit_precision: "fp16"
+
+  # Q-Former
+  num_query_token: 32
+  qformer_text_input: True
+
+  # vicuna
+  llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1"
+  prompt: ""
+  max_txt_len: 256
+  max_output_txt_len: 256
+
+  # freeze
+  freeze_vit: True
+  freeze_llm: True
+  freeze_qformer: False
+  freeze_t5_proj: False
+
+  # moe
+  general_version: 'uni_route_moe'
+  moebert_route_method: 'uni-cls-cross-route'
+  moebert_load_balance: 0.01
+  moebert_expert_num: 4
+  moebert_num_beams: 4
+  moe_weight_type: 'ffn_prob'
+  use_balance_loss: False
+  ln_position: "out"
+
+datasets:
+  coco_vqa:    # 658104
+    type: vqa_v2_eval
+    batch_size: 48
+    vis_processor:
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      eval:
+        name: "blip_caption"
+
+run:
+  task: instruction_tuning
+  # optimizer
+  lr_sched: "linear_warmup_cosine_lr"
+  init_lr: 5e-5
+  min_lr: 1e-6
+  warmup_lr: 1e-6
+  log_freq: 5
+  save_freq: 1500
+
+  weight_decay: 0.05
+  max_epoch: 10
+  num_workers: 4
+  warmup_steps: 600
+  iters_per_epoch: 3000
+
+  seed: 42
+  output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/eval/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/"
+  
+  amp: True
+  resume_ckpt_path: null
+
+  evaluate: True
+  test_splits: ["test"]
+
+  device: "cuda"
+  world_size: 1
+  dist_url: "env://"
+  distributed: True
+  

minigpt4/projects/qformer_moe_cls_vicuna/train/mix6_qformer_moe_route_cls_blip2_vicuna7b_0329.yaml

@@ -35,9 +35,10 @@ model:
   freeze_t5_proj: False
 
   # moe
+  # general_version: 'cls_route_moe'
   general_version: 'uni_route_moe'
-  moebert_route_method: "post-route-uni"
+  moebert_route_method: 'uni-cls-cross-route'
-  moebert_load_balance: 0.05
+  moebert_load_balance: 0.01
   moebert_expert_num: 3
   moebert_num_beams: 3
   moe_weight_type: 'ffn_prob'
@@ -154,13 +155,13 @@ run:
   save_freq: 1500
 
   weight_decay: 0.05
-  max_epoch: 8
+  max_epoch: 10
   num_workers: 4
   warmup_steps: 600
   iters_per_epoch: 5000
 
   seed: 42
-  output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0328/"
+  output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330/"
   
   amp: True
   resume_ckpt_path: null
@@ -170,7 +171,7 @@ run:
   valid_splits: ["val"]
 
   wandb_log: False
-  # job_name: mix6_uni_route_post_5e5_3ex3b_005_10ep_0319
+  # job_name: mix6_uni_cls_route_post_5e5_3ex3b_005_8ep_0330
 
   device: "cuda"
   world_size: 1

minigpt4/projects/qformer_moe_route_universal_vicuna/train/mix6_qformer_moe_route_uni_blip2_vicuna7b_0329.yaml

@@ -0,0 +1,180 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+model:
+  arch: blip2_vicuna_instruct
+  model_type: vicuna7b_pretrain
+  load_pretrained: True
+  load_finetuned: False
+  vit_model: eva_clip_g
+  pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
+  q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth"
+
+  # vit encoder
+  image_size: 224
+  drop_path_rate: 0
+  use_grad_checkpoint: False
+  vit_precision: "fp16"
+
+  # Q-Former
+  num_query_token: 32
+  qformer_text_input: True
+
+  # vicuna7b
+  llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1"
+  prompt: ""
+  max_txt_len: 256
+  max_output_txt_len: 256
+
+  # freeze
+  freeze_vit: True
+  freeze_llm: True
+  freeze_qformer: False
+  freeze_t5_proj: False
+
+  # moe
+  general_version: 'base'
+  # general_version: 'uni_route_moe'
+  # moebert_route_method: "post-route-uni"
+  # moebert_load_balance: 0.05
+  # moebert_expert_num: 3
+  # moebert_num_beams: 3
+  # moe_weight_type: 'ffn_prob'
+  # use_balance_loss: False
+  # ln_position: "out"
+
+datasets:
+  gqa: # train: 943000, 12578, 12578)
+    type: balanced_sft_raw
+    batch_size: 32
+    vis_processor:
+      train:
+        name: "blip2_image_train"
+        image_size: 224
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      train:
+        name: "blip_caption"
+      eval:
+        name: "blip_caption"
+    sample_ratio: 10
+  
+  ok_vqa: # train, valid (9009, 5046)
+    batch_size: 32
+    vis_processor:
+      train:
+        name: "blip2_image_train"
+        image_size: 224
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      train:
+        name: "blip_caption"
+      eval:
+        name: "blip_caption"
+    sample_ratio: 1
+  
+  coco_vqa:    # 658104
+    batch_size: 32
+    vis_processor:
+      train:
+        name: "blip2_image_train"
+        image_size: 224
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      train:
+        name: "blip_caption"
+      eval:
+        name: "blip_caption"
+    sample_ratio: 9
+
+  coco_caption: # 414113 train
+    batch_size: 32
+    vis_processor:
+      train:
+        name: "blip2_image_train"
+        image_size: 224
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      train:
+        name: "blip_caption"
+      eval:
+        name: "blip_caption"
+    sample_ratio: 7
+
+  aok_vqa: # train: 17056, val: 1145
+    batch_size: 32
+    vis_processor:
+      train:
+        name: "blip2_image_train"
+        image_size: 224
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      train:
+        name: "blip_caption"
+      eval:
+        name: "blip_caption"
+    sample_ratio: 2
+    # sample_ratio: 1
+  
+  textcaps_caption: # train: 109765, val: 15830
+    batch_size: 32
+    vis_processor:
+      train:
+        name: "blip2_image_train"
+        image_size: 224
+      eval:
+        name: "blip2_image_eval"
+        image_size: 224
+    text_processor:
+      train:
+        name: "blip_caption"
+      eval:
+        name: "blip_caption"
+    # sample_ratio: 3
+    sample_ratio: 4
+
+run:
+  task: instruction_tuning
+  lr_sched: "linear_warmup_cosine_lr"
+  init_lr: 5e-5
+  min_lr: 1e-6
+  warmup_lr: 1e-6
+  log_freq: 5
+  save_freq: 1500
+
+  weight_decay: 0.05
+  max_epoch: 8
+  num_workers: 4
+  warmup_steps: 600
+  iters_per_epoch: 5000
+
+  seed: 40
+  # output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0328/"
+  output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_base_lr5e5_textinqf_8epo_0329/"
+  
+  amp: True
+  resume_ckpt_path: null
+
+  evaluate: False 
+  train_splits: ["train"]
+  valid_splits: ["val"]
+
+  wandb_log: False
+  # job_name: mix6_uni_route_post_5e5_3ex3b_005_10ep_0319
+
+  device: "cuda"
+  world_size: 1
+  dist_url: "env://"
+  distributed: True

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.320559707425666,54.43,73.75,71.61572052401746,64.02907005786079
+58.936237875655905,56.23,75.25,71.09170305676857,65.37698523310611
+60.99538877404993,55.3,76.4,73.01310043668123,66.42712230268279
+61.146446175862614,58.62,76.58,73.1004366812227,67.36172071427133
+62.5934170774368,58.5,77.5,73.88646288209607,68.11996998988322
+62.41850850691684,57.47,78.24,73.7117903930131,67.96007472498249
+62.99888694546033,58.08,78.88,74.235807860262,68.54867370143059
+63.77802512323104,58.72,79.34,74.06113537117905,68.97479012360252
+63.74622356495468,58.5,79.51,74.235807860262,68.99800785630417
+63.825727460645574,58.41,79.63,74.41048034934498,69.06905195249763

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,8 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.40976307839084,53.92,74.0,71.26637554585152,64.14903465606059
+60.136746700588326,56.03,75.27,71.70305676855895,65.78495086728682
+60.99538877404993,56.27,76.52,73.27510917030567,66.7651244860889
+61.94943552234059,57.39,76.95,74.06113537117905,67.58764272337991
+62.55366512959135,58.17,77.83,73.53711790393012,68.02269575838037
+63.26920019080935,57.8,78.7,75.54585152838428,68.8287629297984
+63.46000954046749,58.15,79.09,74.58515283842794,68.82129059472385

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,7 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.948640483383684,53.97,73.92,71.17903930131004,64.00441994617344
+60.27985371283193,54.86,75.51,71.79039301310044,65.6100616814831
+61.138495786293525,56.33,76.33,72.66375545851528,66.61556281120221
+61.631419939577036,58.13,76.89,73.1877729257642,67.45979821633532
+62.513913181745906,57.35,77.96,72.5764192139738,67.60008309892993
+63.126093178565746,57.36,78.63,74.14847161572052,68.31614119857156

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,7 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.06789632692002,54.44,73.8,71.09170305676857,64.09989984592214
+59.85848306567022,56.06,75.25,72.75109170305677,65.97989369218175
+60.717125139131824,55.3,76.37,73.1877729257642,66.39372451622401
+61.34520591508984,57.32,76.94,73.62445414847161,67.30741501589037
+62.6649705835586,58.16,77.93,72.5764192139738,67.8328474493831
+62.943234218476704,59.1,78.55,74.32314410480349,68.72909458082005

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,8 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.203052949594536,53.68,73.68,70.48034934497817,63.76085057364318
+59.09524566703769,54.05,74.96,72.75109170305677,65.21408434252362
+61.18619812370806,56.16,76.09,73.53711790393012,66.74332900690955
+62.03688980760057,57.59,76.64,72.92576419213974,67.29816349993509
+62.47416123390046,57.58,77.69,72.75109170305677,67.62381323423931
+63.014787724598506,57.61,78.51,73.97379912663754,68.27714671280901
+63.43615837176022,57.94,79.02,73.62445414847161,68.50515313005796

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.726029575449196,54.51,73.9,71.9650655021834,64.27527376940814
+59.5802194307521,55.89,75.12,72.83842794759825,65.8571618445876
+60.605819685164576,56.41,75.93,74.06113537117905,66.7517387640859
+61.43266020034982,57.36,76.65,73.1004366812227,67.13577422039313
+62.32310383208777,57.71,77.76,74.235807860262,68.00722792308744
+62.80807759580219,58.29,78.22,73.7117903930131,68.25746699720382
+63.07839084115122,58.6,78.88,74.235807860262,68.6985496753533
+63.722372396247415,58.5,79.24,74.67248908296943,69.03371536980421
+63.89728096676737,58.97,79.44,73.88646288209607,69.04843596221586
+63.84957862935284,58.62,79.56,73.88646288209607,68.97901037786222

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,8 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.99634282079822,53.79,73.68,70.82969432314411,63.82400928598558
+59.68357449515026,54.46,75.16,72.40174672489083,65.42633030501027
+60.939736047066305,55.99,75.88,71.87772925764192,66.17186632617705
+62.00508824932422,58.43,76.67,72.48908296943232,67.39854280468914
+62.8716807123549,58.22,77.73,73.88646288209607,68.17703589861274
+62.99093655589124,58.08,78.53,74.06113537117905,68.41551798176756
+63.42820798219113,58.48,78.93,74.235807860262,68.76850396061329

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,7 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.932739704245506,54.43,73.9,71.52838427947599,64.19778099593037
+60.057242804897434,55.95,75.28,72.31441048034935,65.9004133213117
+61.44061058991891,54.26,76.24,73.1877729257642,66.28209587892077
+61.806328510096996,57.5,76.83,73.44978165938865,67.3965275423714
+62.56956590872953,58.33,77.89,73.7991266375546,68.14717313657103
+62.760375258387654,58.37,78.45,74.32314410480349,68.47587984079779

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,10 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.940690093814595,54.81,73.85,71.09170305676857,64.17309828764579
+60.20034981714104,55.52,75.37,72.5764192139738,65.9166922577787
+61.19414851327715,54.56,76.38,73.27510917030567,66.35231442089571
+61.217999681984416,57.83,76.88,73.36244541484717,67.3226112742079
+62.64111941485133,57.97,77.97,72.92576419213974,67.87672090174777
+63.03863889330577,58.54,78.49,72.75109170305677,68.20493264909064
+63.61106694228017,58.49,79.1,74.06113537117905,68.8155505783648
+63.66671966926379,58.68,79.56,72.92576419213974,68.70812096535089
+64.0244871998728,59.23,79.74,73.7117903930131,69.17656939822146

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,8 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.78963269200191,53.55,73.54,70.04366812227074,63.48082520356816
+59.95388774049929,54.89,75.12,71.52838427947599,65.37306800499383
+60.7966290348227,55.93,76.01,72.83842794759825,66.39376424560524
+61.84608045794244,56.83,76.7,73.44978165938865,67.20646552933277
+62.943234218476704,57.48,77.75,74.235807860262,68.10226051968468
+62.67292097312769,57.44,78.43,75.4585152838428,68.50035906424263
+63.36460486563842,57.82,78.92,75.10917030567686,68.80344379282882

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,8 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.020193989505486,53.7,73.8,70.21834061135371,63.6846336502148
+60.33550643981555,54.9,75.12,72.40174672489083,65.68931329117659
+60.947686436635394,55.55,76.29,73.01310043668123,66.45019671832915
+61.45651136905709,57.24,76.39,73.62445414847161,67.17774137938218
+62.65702019398951,57.34,77.73,75.37117903930131,68.2745498083227
+63.5156622674511,57.95,78.4,73.62445414847161,68.37252910398068
+63.6587692796947,57.77,78.75,74.49781659388647,68.6691464683953

process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.218953728732714,55.12,74.19,71.52838427947599,64.51433450205218
+60.0095404674829,55.87,75.27,72.66375545851528,65.95332398149955
+60.73302591827,56.65,76.19,73.44978165938865,66.75570189441466
+61.42470981078073,58.09,76.81,73.44978165938865,67.44362286754235
+62.37875655907139,57.71,78.0,73.53711790393012,67.90646861575038
+62.73652408968039,57.99,78.57,73.27510917030567,68.14290831499652
+63.26124980124026,58.96,79.16,74.75982532751092,69.03526878218779
+63.626967721418346,58.1,79.68,74.235807860262,68.9106938954201
+63.77007473366195,58.19,79.87,74.32314410480349,69.03830470961636
+63.626967721418346,57.76,79.98,74.49781659388647,68.9661960788262

process_log/caption_result/results_coco_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv

@@ -0,0 +1,9 @@
+Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr
+0.8182082924454812,0.6638925862784945,0.5165455592392635,0.3928161572696676,0.2920462773984222,0.5918020291082554,1.2987178029426083
+0.8240854949333832,0.6717746108928253,0.5257706306569868,0.40249821532303487,0.2936994611883999,0.5958284475381215,1.3272042634335437
+0.8246398014401575,0.6743122037328803,0.5292565031234792,0.40578728669235764,0.2964349874781921,0.597962455259089,1.3321248494065205
+0.8270564620397717,0.6793843847871991,0.5355102159092959,0.41366323151841006,0.2977148403261682,0.6024717705283715,1.3520977723112642
+0.8245944565058522,0.6751194271857663,0.5306411803879995,0.40888833957017,0.29642409734712727,0.598579033363294,1.3383802889190928
+0.8280804382261628,0.6796044258685356,0.537046439535483,0.4152107936908873,0.2990414503157605,0.602392043880006,1.3531755491021618
+0.8299061115859161,0.6814451792563676,0.5379678657467563,0.4156139159677314,0.2992684448837016,0.6027499590797065,1.3538095043242933
+0.8276954938807027,0.6800987491517746,0.5374861277345877,0.41554847690232694,0.2996663866671749,0.6026710548192867,1.3561296682313817

process_log/caption_result/results_text_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv

@@ -0,0 +1,9 @@
+Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr
+0.7309287396242824,0.5494655932407475,0.4041141975269019,0.2921841096479282,0.23361135981392955,0.48650322014232156,0.9966076482929254
+0.7382464874712654,0.5571950857230415,0.4121904524343956,0.30143019979529023,0.23826500378661586,0.4926618553461225,1.0232964787151213
+0.7350922242488682,0.5559390933443953,0.4113206176871192,0.3009155411187479,0.2374533115810085,0.49274408438489914,1.0099444415721002
+0.7369161501554021,0.556728672691215,0.41086927244074317,0.3004344996181908,0.23806683186363936,0.49184279207452697,1.0280928870205412
+0.7407251133040751,0.5595661891353809,0.4138229944446712,0.3032027996796593,0.2415473861083878,0.4972778899306893,1.0479529588574925
+0.7428951836605517,0.5623862396980985,0.4161405396490859,0.3047709156231164,0.2438504314752837,0.49858613995817086,1.0477488915322588
+0.745208695011462,0.5654882473522628,0.4200936310645993,0.308981454910347,0.24371581659378194,0.49914038443933745,1.053786374859757
+0.7444798335079819,0.5624985319454049,0.4157842492239835,0.3044642812661473,0.2432655219581268,0.49722674885535495,1.056752744677739

process_log/load_log.py

@@ -11,7 +11,53 @@ import pandas as pd
 # path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_4ex4b_2loss_005_top6layer_textinqf_10epo_0319/20240319110/"
 # path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr6e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0319/20240319105"
 # path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_tcap_raw_Qformer_base_lr5e5_10epo_0320_instruct/20240320230/"
-path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr3e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0326/20240326134"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr3e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0326/20240326134"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/flant5xxl/base/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328/20240328100/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0328/20240328164"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_base_lr5e5_textinqf_8epo_0329/20240329110"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0330/20240329232/"
+
+# mix5
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330211/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330210/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330211/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330183/"
+
+# mix6 3ex3b uni 005
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330211/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330182/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330173/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330173/"
+
+# mix6 3ex3b uni 001
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330/20240330221/"
+
+# mi6 non-universal
+# cls-cross-route-mix6-001-3ex3b-0330-wanghanzi-master-0
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330/20240330221/"
+# cls-cross-route-mix6-001-4ex4b-wanghanzi-master-0
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331214/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331214/"
+
+# mix6 4ex4b uni 001
+path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331/20240331132/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331144/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331143/"
+
+# cls cross
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331215/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331215/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/"
+
+# 8ex8b uni
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331/20240331150/"
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/"
+
+# post
+# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0331/20240331131/"
 modes = ['gqa','okvqa','vqav2','aokvqa']
 file_name = dict()
 data = dict()
@@ -25,7 +71,11 @@ for mode in modes:
     data[mode] = accs
 print(data)
 df = pd.DataFrame(data)
-df.to_csv("/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/results.csv",index=False)
+# calculate average score
+df['avg'] = df.mean(axis=1)
+name = path.split('/')[-3]
+df.to_csv(f"/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/0401_results_{name}.csv",index=False)
+print("save to /mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/0401_results_{}.csv".format(name))
 print(df)
 modes = ['coco_cap','text_cap']
 for mode in modes:
@@ -35,7 +85,7 @@ for mode in modes:
         for line in f: 
             tmp.append(json.loads(line)["val"])
     df1 = pd.DataFrame(tmp)
-    df1.to_csv("/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/results_{}.csv".format(mode),index=False)
+    df1.to_csv("/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/caption_result/results_{}_{}.csv".format(mode,name),index=False)
     print("\n",df1)
 
     

process_log/results.csv

@@ -1,11 +1,10 @@
-gqa,okvqa,vqav2,aokvqa
+gqa,okvqa,vqav2,aokvqa,avg
-56.20130386388933,54.9,73.99,70.65502183406113
+57.14740022261091,54.38,74.13,71.52838427947599,64.29644612552173
-59.98568929877563,56.83,75.05,71.87772925764192
+59.19860073143583,55.98,75.1,72.5764192139738,65.7137549863524
-60.81252981396088,56.97,76.33,72.31441048034935
+60.64557163301002,56.56,75.9,73.01310043668123,66.52966801742281
-61.217999681984416,58.72,76.9,72.13973799126637
+61.3770074733662,57.87,76.6,74.06113537117905,67.47703571113631
-62.63316902528224,57.77,77.9,73.53711790393012
+62.40260772777866,57.16,77.31,74.32314410480349,67.79893795814553
-62.63316902528224,58.7,78.62,72.31441048034935
+62.99888694546033,57.96,78.12,73.1877729257642,68.06666496780613
-63.0863412307203,58.44,79.17,73.01310043668123
+62.69677214183494,59.45,78.75,74.235807860262,68.78314500052424
-63.99268564159644,58.3,79.67,73.1004366812227
+63.126093178565746,58.74,79.22,74.41048034934498,68.87414338197769
-63.85752901892193,58.26,79.89,73.1877729257642
+63.30100174908571,58.32,79.5,73.7117903930131,68.7081980355247
-64.05628875814915,58.36,79.96,73.27510917030567

process_log/results_base.csv

@@ -0,0 +1,9 @@
+gqa,okvqa,vqav2,aokvqa,avg
+53.46636985212275,48.02,71.28,73.27510917030567,61.5103697556071
+55.525520750516776,50.12,72.37,74.32314410480349,63.08466621383006
+56.91683892510733,49.66,73.78,75.54585152838428,63.9756726133729
+57.616473207187155,52.98,74.3,76.85589519650655,65.43809210092343
+58.538718397201464,52.81,74.97,75.9825327510917,65.57531278707329
+59.54841787247575,52.71,75.53,76.68122270742359,66.11741014497483
+59.60407059945937,53.43,75.81,76.68122270742359,66.38132332672075
+59.68357449515026,53.71,75.99,77.117903930131,66.6253696063203

process_log/results_coco_cap.csv

@@ -1,11 +1,9 @@
 Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr
-0.823223287530701,0.6717046486084919,0.5289578677489075,0.40968911321073265,0.29959733857127835,0.6000811689013733,1.3453305422681376
+0.8182082924454812,0.6638925862784945,0.5165455592392635,0.3928161572696676,0.2920462773984222,0.5918020291082554,1.2987178029426083
-0.8223586805364383,0.6761940235369243,0.5372021051535613,0.42002107530341415,0.3007070731886293,0.6033562057362387,1.3625435411445188
+0.8240854949333832,0.6717746108928253,0.5257706306569868,0.40249821532303487,0.2936994611883999,0.5958284475381215,1.3272042634335437
-0.8268533010152048,0.6768505208938964,0.5354603117460063,0.41692252223016474,0.3025309435581166,0.6047665704453451,1.372728338529115
+0.8246398014401575,0.6743122037328803,0.5292565031234792,0.40578728669235764,0.2964349874781921,0.597962455259089,1.3321248494065205
-0.8330185365456568,0.6839215475169012,0.543703824076164,0.42514888177425736,0.3056707793309544,0.6059406486352524,1.3861562304957278
+0.8270564620397717,0.6793843847871991,0.5355102159092959,0.41366323151841006,0.2977148403261682,0.6024717705283715,1.3520977723112642
-0.82851059467086,0.6809176651975785,0.540587141996957,0.4231733852045105,0.30531255145802344,0.6074083433970782,1.3834468097216504
+0.8245944565058522,0.6751194271857663,0.5306411803879995,0.40888833957017,0.29642409734712727,0.598579033363294,1.3383802889190928
-0.826759683808046,0.6786303228237849,0.5393123491853253,0.4218864249035804,0.30435744091261596,0.6061024359922585,1.373555774390539
+0.8280804382261628,0.6796044258685356,0.537046439535483,0.4152107936908873,0.2990414503157605,0.602392043880006,1.3531755491021618
-0.8276490281491424,0.6807227995567375,0.5428624362839755,0.4269084803683001,0.30608939915890326,0.6078544738015538,1.3817698541866794
+0.8299061115859161,0.6814451792563676,0.5379678657467563,0.4156139159677314,0.2992684448837016,0.6027499590797065,1.3538095043242933
-0.8294752997477592,0.684006846582908,0.5462002712862497,0.43036414820903174,0.3072188868161371,0.6112755876824707,1.3946012917260822
+0.8276954938807027,0.6800987491517746,0.5374861277345877,0.41554847690232694,0.2996663866671749,0.6026710548192867,1.3561296682313817
-0.8295967394767231,0.6833360908793101,0.5455277195646591,0.4292003829675252,0.3075074477217475,0.6103970688371728,1.392473266614681
-0.8311746625208477,0.685154908394367,0.5473453025692251,0.4310962369321893,0.3077283192093123,0.6112527230048272,1.3960092087389069

process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.49546827794561,56.81,73.81,70.21834061135371,64.33345222232484
+58.97598982350135,54.6,75.05,70.91703056768559,64.88575509779673
+61.408809031642555,56.58,76.51,73.36244541484717,66.96531361162243
+62.020989028462395,58.58,76.98,72.13973799126637,67.43018175493219
+62.346955000795035,56.98,78.14,73.27510917030567,67.68551604277518
+63.43615837176022,57.51,78.88,73.1004366812227,68.23164876324573
+63.79392590236922,57.77,79.41,73.7117903930131,68.67142907384559
+63.722372396247415,58.26,79.93,73.88646288209607,68.94970881958587
+63.57131499443472,58.05,80.13,73.36244541484717,68.77844010232047
+63.817777071076485,58.43,80.19,73.7117903930131,69.0373918660224

process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,9 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.638575290189216,55.75,73.72,71.09170305676857,64.30006958673944
+59.111146446175866,56.38,75.02,73.1877729257642,65.92472984298502
+61.43266020034982,56.12,76.5,72.83842794759825,66.72277203698701
+62.283351884242336,58.41,77.06,72.40174672489083,67.5387746522833
+62.31515344251868,57.59,78.2,73.7117903930131,67.95423595888295
+63.42820798219113,57.71,78.77,73.97379912663754,68.47050177720718
+63.316902528223885,58.18,79.44,73.97379912663754,68.72767541371536
+63.74622356495468,58.59,79.9,74.235807860262,69.11800785630417

process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,10 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.78963269200191,56.29,73.86,70.91703056768559,64.46416581492187
+59.66767371601208,54.41,75.21,72.48908296943232,65.4441891713611
+61.33725552552075,56.65,76.37,73.62445414847161,66.9954274184981
+62.013038638893306,59.54,77.12,73.27510917030567,67.98703695229975
+62.927333439338526,57.51,78.17,73.62445414847161,68.05794689695253
+63.67467005883288,57.53,78.9,73.01310043668123,68.27944262387852
+63.5156622674511,58.04,79.41,74.06113537117905,68.75669940965753
+63.642868500556524,58.31,79.97,73.44978165938865,68.8431625399863
+63.937032914612814,58.11,80.18,73.97379912663754,69.05020801031259

process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,9 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.21720464302751,56.58,73.83,70.56768558951964,64.29872255813677
+59.40531086023215,55.45,75.26,72.40174672489083,65.62926439628075
+59.810780728255686,52.47,75.44,70.82969432314411,64.63761876284994
+61.75862617268246,58.43,76.42,71.9650655021834,67.14342291871647
+62.77627603752583,57.39,77.56,73.88646288209607,67.90318472990548
+63.141993957703924,57.33,78.13,74.41048034934498,68.25311857676222
+63.44410876132931,57.15,78.86,73.01310043668123,68.11680229950264
+63.34870408650024,57.88,79.29,73.7117903930131,68.55762361987834

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.320559707425666,54.43,73.75,71.61572052401746,64.02907005786079
+58.936237875655905,56.23,75.25,71.09170305676857,65.37698523310611
+60.99538877404993,55.3,76.4,73.01310043668123,66.42712230268279
+61.146446175862614,58.62,76.58,73.1004366812227,67.36172071427133
+62.5934170774368,58.5,77.5,73.88646288209607,68.11996998988322
+62.41850850691684,57.47,78.24,73.7117903930131,67.96007472498249
+62.99888694546033,58.08,78.88,74.235807860262,68.54867370143059
+63.77802512323104,58.72,79.34,74.06113537117905,68.97479012360252
+63.74622356495468,58.5,79.51,74.235807860262,68.99800785630417
+63.825727460645574,58.41,79.63,74.41048034934498,69.06905195249763

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.0122435999364,54.2,74.05,71.9650655021834,64.30682727552995
+60.065193194466524,55.18,75.18,72.48908296943232,65.72856904097472
+60.87613293051359,57.05,75.99,73.01310043668123,66.7323083417987
+61.69502305612975,58.69,76.81,73.7991266375546,67.74853742342108
+62.291302273811425,58.12,77.73,73.44978165938865,67.89777098330002
+62.56956590872953,57.97,78.35,73.01310043668123,67.97566658635269
+63.17379551598028,59.22,78.91,73.53711790393012,68.7102283549776
+63.46000954046749,57.9,79.42,73.53711790393012,68.5792818610994
+63.77802512323104,58.46,79.58,73.36244541484717,68.79511763451956
+63.83367785021466,58.36,79.73,73.62445414847161,68.88703299967158

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,7 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.40976307839084,53.92,74.0,71.26637554585152,64.14903465606059
+60.136746700588326,56.03,75.27,71.70305676855895,65.78495086728682
+60.99538877404993,56.27,76.52,73.27510917030567,66.7651244860889
+61.94943552234059,57.39,76.95,74.06113537117905,67.58764272337991
+62.55366512959135,58.17,77.83,73.53711790393012,68.02269575838037
+63.26920019080935,57.8,78.7,75.54585152838428,68.8287629297984

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.54491970106535,54.21,73.9,71.87772925764192,64.38316223967682
+59.93798696136111,54.74,75.12,72.75109170305677,65.63726966610447
+60.77277786611543,55.58,76.23,73.7991266375546,66.59547612591751
+61.66322149785339,57.91,76.69,73.27510917030567,67.38458266703977
+62.56956590872953,57.59,77.73,74.06113537117905,67.98767531997714
+62.85577993321672,58.39,78.58,74.8471615720524,68.66823537631728
+63.33280330736206,58.49,79.11,74.67248908296943,68.90132309758287
+63.61106694228017,58.64,79.56,75.02183406113538,69.20822525085389
+63.69852122754015,58.41,79.8,74.67248908296943,69.14525257762739
+63.68262044840197,58.45,79.9,74.75982532751092,69.19811144397823

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,7 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.948640483383684,53.97,73.92,71.17903930131004,64.00441994617344
+60.27985371283193,54.86,75.51,71.79039301310044,65.6100616814831
+61.138495786293525,56.33,76.33,72.66375545851528,66.61556281120221
+61.631419939577036,58.13,76.89,73.1877729257642,67.45979821633532
+62.513913181745906,57.35,77.96,72.5764192139738,67.60008309892993
+63.126093178565746,57.36,78.63,74.14847161572052,68.31614119857156

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,6 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.06789632692002,54.44,73.8,71.09170305676857,64.09989984592214
+59.85848306567022,56.06,75.25,72.75109170305677,65.97989369218175
+60.717125139131824,55.3,76.37,73.1877729257642,66.39372451622401
+61.34520591508984,57.32,76.94,73.62445414847161,67.30741501589037
+62.6649705835586,58.16,77.93,72.5764192139738,67.8328474493831

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,7 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.203052949594536,53.68,73.68,70.48034934497817,63.76085057364318
+59.09524566703769,54.05,74.96,72.75109170305677,65.21408434252362
+61.18619812370806,56.16,76.09,73.53711790393012,66.74332900690955
+62.03688980760057,57.59,76.64,72.92576419213974,67.29816349993509
+62.47416123390046,57.58,77.69,72.75109170305677,67.62381323423931
+63.014787724598506,57.61,78.51,73.97379912663754,68.27714671280901

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.726029575449196,54.51,73.9,71.9650655021834,64.27527376940814
+59.5802194307521,55.89,75.12,72.83842794759825,65.8571618445876
+60.605819685164576,56.41,75.93,74.06113537117905,66.7517387640859
+61.43266020034982,57.36,76.65,73.1004366812227,67.13577422039313
+62.32310383208777,57.71,77.76,74.235807860262,68.00722792308744
+62.80807759580219,58.29,78.22,73.7117903930131,68.25746699720382
+63.07839084115122,58.6,78.88,74.235807860262,68.6985496753533
+63.722372396247415,58.5,79.24,74.67248908296943,69.03371536980421
+63.89728096676737,58.97,79.44,73.88646288209607,69.04843596221586
+63.84957862935284,58.62,79.56,73.88646288209607,68.97901037786222

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,7 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.99634282079822,53.79,73.68,70.82969432314411,63.82400928598558
+59.68357449515026,54.46,75.16,72.40174672489083,65.42633030501027
+60.939736047066305,55.99,75.88,71.87772925764192,66.17186632617705
+62.00508824932422,58.43,76.67,72.48908296943232,67.39854280468914
+62.8716807123549,58.22,77.73,73.88646288209607,68.17703589861274
+62.99093655589124,58.08,78.53,74.06113537117905,68.41551798176756

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,10 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.940690093814595,54.47,73.98,71.61572052401746,64.25160265445803
+60.12879631101924,55.14,75.16,72.31441048034935,65.68580169784215
+61.09874383844809,55.84,76.32,73.01310043668123,66.56796106878232
+61.95738591190968,58.01,76.9,73.44978165938865,67.57929189282459
+62.752424868818565,58.21,77.94,73.1004366812227,68.00071538751033
+62.8001272062331,58.29,78.52,74.235807860262,68.46148376662377
+63.17379551598028,58.75,79.04,73.88646288209607,68.71256459951908
+63.4520591508984,57.67,79.69,73.88646288209607,68.67463050824861
+63.5872157735729,58.24,79.92,73.62445414847161,68.84291748051113

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.14740022261091,54.38,74.13,71.52838427947599,64.29644612552173
+59.19860073143583,55.98,75.1,72.5764192139738,65.7137549863524
+60.64557163301002,56.56,75.9,73.01310043668123,66.52966801742281
+61.3770074733662,57.87,76.6,74.06113537117905,67.47703571113631
+62.40260772777866,57.16,77.31,74.32314410480349,67.79893795814553
+62.99888694546033,57.96,78.12,73.1877729257642,68.06666496780613
+62.69677214183494,59.45,78.75,74.235807860262,68.78314500052424
+63.126093178565746,58.74,79.22,74.41048034934498,68.87414338197769
+63.30100174908571,58.32,79.5,73.7117903930131,68.7081980355247
+63.29305135951662,58.05,79.62,73.7991266375546,68.6905444992678

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.521068532358086,53.93,73.93,70.74235807860262,64.03085665274017
+60.39910955636826,54.91,75.34,71.70305676855895,65.5880415812318
+61.04309111146446,56.33,76.35,73.97379912663754,66.9242225595255
+62.17204643027509,57.25,76.97,72.22707423580786,67.15478016652074
+62.919383049769436,57.44,77.9,73.44978165938865,67.92729117728952
+63.5156622674511,58.08,78.7,74.14847161572052,68.6110334707929
+63.96088408332008,58.17,79.22,74.14847161572052,68.87483892476016
+64.27889966608363,58.38,79.75,73.44978165938865,68.96467033136807
+63.98473525202735,58.32,79.91,73.53711790393012,68.93796328898937
+64.46175862617268,57.95,80.05,73.27510917030567,68.93421694911959

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,6 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.932739704245506,54.43,73.9,71.52838427947599,64.19778099593037
+60.057242804897434,55.95,75.28,72.31441048034935,65.9004133213117
+61.44061058991891,54.26,76.24,73.1877729257642,66.28209587892077
+61.806328510096996,57.5,76.83,73.44978165938865,67.3965275423714
+62.56956590872953,58.33,77.89,73.7991266375546,68.14717313657103

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.75783113372555,54.51,73.98,71.87772925764192,64.28139009784186
+59.98568929877563,55.07,75.3,72.48908296943232,65.711193067052
+60.88408332008268,56.75,76.03,73.97379912663754,66.90947061168006
+61.75067578311337,57.82,76.87,73.97379912663754,67.60361872743772
+62.442359675624104,57.72,77.97,73.53711790393012,67.91736939488857
+62.8716807123549,57.88,78.51,73.97379912663754,68.30886995974811
+63.308952138654796,58.55,79.09,74.235807860262,68.7961899997292
+63.547463825727455,57.86,79.61,74.67248908296943,68.92248822717423
+63.809826681507396,58.42,79.81,74.49781659388647,69.13441081884847
+63.722372396247415,57.87,79.93,74.32314410480349,68.96137912526272

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,11 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.97249165209095,54.35,73.91,70.56768558951964,63.950044310402646
+60.24805215455557,55.26,75.14,72.5764192139738,65.80611784213234
+60.8681825409445,55.82,76.32,74.58515283842794,66.89833384484311
+61.99713785975513,57.76,76.9,73.44978165938865,67.52672987978595
+62.76832564795674,58.45,77.85,73.7991266375546,68.21686307137783
+63.16584512641119,57.87,78.61,73.53711790393012,68.29574075758532
+63.84162823978375,58.28,79.01,73.53711790393012,68.66718653592847
+64.31865161392908,58.35,79.56,73.27510917030567,68.87594019605869
+64.41405628875815,58.76,79.68,73.44978165938865,69.0759594870367
+64.62076641755446,58.6,79.84,73.88646288209607,69.23680732491263

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,9 @@
+gqa,okvqa,vqav2,aokvqa,avg
+56.940690093814595,54.81,73.85,71.09170305676857,64.17309828764579
+60.20034981714104,55.52,75.37,72.5764192139738,65.9166922577787
+61.19414851327715,54.56,76.38,73.27510917030567,66.35231442089571
+61.217999681984416,57.83,76.88,73.36244541484717,67.3226112742079
+62.64111941485133,57.97,77.97,72.92576419213974,67.87672090174777
+63.03863889330577,58.54,78.49,72.75109170305677,68.20493264909064
+63.61106694228017,58.49,79.1,74.06113537117905,68.8155505783648
+63.66671966926379,58.68,79.56,72.92576419213974,68.70812096535089

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv

@@ -0,0 +1,7 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.020193989505486,53.7,73.8,70.21834061135371,63.6846336502148
+60.33550643981555,54.9,75.12,72.40174672489083,65.68931329117659
+60.947686436635394,55.55,76.29,73.01310043668123,66.45019671832915
+61.45651136905709,57.24,76.39,73.62445414847161,67.17774137938218
+62.65702019398951,57.34,77.73,75.37117903930131,68.2745498083227
+63.5156622674511,57.95,78.4,73.62445414847161,68.37252910398068

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv

@@ -0,0 +1,9 @@
+gqa,okvqa,vqav2,aokvqa,avg
+57.33025918269995,54.6,74.14,70.91703056768559,64.24682243759638
+59.643822547304815,55.57,75.28,72.13973799126637,65.6583901346428
+60.7966290348227,56.62,76.23,74.14847161572052,66.9487751626358
+61.806328510096996,57.53,76.92,73.88646288209607,67.53569784804826
+62.67292097312769,57.78,78.04,74.49781659388647,68.24768439175354
+63.05453967244395,57.7,78.55,73.44978165938865,68.18858033295815
+63.16584512641119,58.84,79.15,73.7991266375546,68.73874294099144
+63.75417395452377,58.02,79.59,73.7117903930131,68.76899108688423

process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv

@@ -0,0 +1,9 @@
+gqa,okvqa,vqav2,aokvqa,avg
+53.46636985212275,48.02,71.28,73.27510917030567,61.5103697556071
+55.525520750516776,50.12,72.37,74.32314410480349,63.08466621383006
+56.91683892510733,49.66,73.78,75.54585152838428,63.9756726133729
+57.616473207187155,52.98,74.3,76.85589519650655,65.43809210092343
+58.538718397201464,52.81,74.97,75.9825327510917,65.57531278707329
+59.54841787247575,52.71,75.53,76.68122270742359,66.11741014497483
+59.60407059945937,53.43,75.81,76.68122270742359,66.38132332672075
+59.68357449515026,53.71,75.99,77.117903930131,66.6253696063203

process_log/results_text_cap.csv

@@ -1,11 +1,9 @@
 Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr
-0.7354185510459915,0.5532517544390539,0.40861793000486973,0.29983800193167554,0.24116643313333005,0.49331369213983123,1.0410874398111594
+0.7309287396242824,0.5494655932407475,0.4041141975269019,0.2921841096479282,0.23361135981392955,0.48650322014232156,0.9966076482929254
-0.7337559429476805,0.5532662736631931,0.4093585176779233,0.2995878219607148,0.24114315075166085,0.49493696747386196,1.0409495699510138
+0.7382464874712654,0.5571950857230415,0.4121904524343956,0.30143019979529023,0.23826500378661586,0.4926618553461225,1.0232964787151213
-0.7320402508385379,0.5518304887400776,0.4110219411525107,0.3038313373404956,0.24291044377257376,0.496058725623593,1.0575894770967331
+0.7350922242488682,0.5559390933443953,0.4113206176871192,0.3009155411187479,0.2374533115810085,0.49274408438489914,1.0099444415721002
-0.735500933603824,0.557825793289975,0.41590254371174706,0.30692236406930773,0.2383527664105175,0.49362765900900546,1.044158754983385
+0.7369161501554021,0.556728672691215,0.41086927244074317,0.3004344996181908,0.23806683186363936,0.49184279207452697,1.0280928870205412
-0.7398393194706776,0.5620893308030489,0.41820732364444946,0.3078230351155729,0.24483177062749348,0.49847300976639547,1.0688814357108025
+0.7407251133040751,0.5595661891353809,0.4138229944446712,0.3032027996796593,0.2415473861083878,0.4972778899306893,1.0479529588574925
-0.7371954616083516,0.5585083902477329,0.4158548739958792,0.30707857737975475,0.244730703390385,0.49852757085109406,1.0729374604921766
+0.7428951836605517,0.5623862396980985,0.4161405396490859,0.3047709156231164,0.2438504314752837,0.49858613995817086,1.0477488915322588
-0.743162583518909,0.5634215669347339,0.42011923174763044,0.311173553245093,0.2441994031592775,0.49809622365158585,1.0774442793527572
+0.745208695011462,0.5654882473522628,0.4200936310645993,0.308981454910347,0.24371581659378194,0.49914038443933745,1.053786374859757
-0.7394105997457735,0.5596190520647151,0.4174127065616514,0.3086643930692146,0.24516092911412224,0.4984150187439041,1.0745130576356021
+0.7444798335079819,0.5624985319454049,0.4157842492239835,0.3044642812661473,0.2432655219581268,0.49722674885535495,1.056752744677739
-0.7401475669028302,0.5612229049005432,0.4184684096329846,0.3101423931360949,0.2459814560570504,0.49998501882495094,1.0809003006778692
-0.7420413230468877,0.5634982579203833,0.4203003411022801,0.3115589386663839,0.245912807863165,0.4989393568537978,1.0805529900845852

test.py

@@ -0,0 +1,149 @@
+# from math import factorial
+# def combine_number(n,k):
+#     combinations = factorial(n) / factorial(n - k)
+#     return combinations
+
+# # 定义一个函数来计算给定字符串中平衡串的子序列数量
+# def count_balanced_subsequences(s):
+#     MOD = 10**9 + 7
+#     # 初始化字母计数器
+#     count = [0] * 26
+#     for char in s:
+#         count[ord(char) - ord('a')] += 1
+    
+#     result = 0
+#     while sum(count)>0:
+#         cnt = 0
+#         for i in range(26):
+#             if count[i] > 0:
+#                 cnt += 1
+#                 count[i] -= 1
+#         print(count)
+#         result += combine_number(cnt,2)
+#         print(result)
+#     return result
+
+# # 示例输入
+# n = 5
+# s = "ababc"
+# # 计算结果
+# result = count_balanced_subsequences(s)
+# print(result)
+
+# def calculate_scores(k, x, y):
+#     # 根据题目描述，我们有以下等式：
+#     # a + b + c = k
+#     # c = a + x
+#     # c = b - y
+#     # 由上面两个等式可得：a + x = b - y
+#     # 因此我们可以解这个线性方程组得到a, b, c的值
+
+#     # a + (a + x) + (a + x + y) = k
+#     # 3a + 2x + y = k
+#     # 解得：
+#     a = (k - 2*x - y) // 3
+#     c = a + x
+#     b = c + y
+
+#     return a, b, c
+
+# # 示例输入
+# k, x, y = 441, 1, -20
+# # 计算输出
+# a, b, c = calculate_scores(k, x, y)
+# print(a, b, c)
+
+
+
+# def min_operations_to_equal(s, t):
+#     operations = []
+#     i = len(s) - 1
+#     while i >= 0:
+#         if s[i] != t[i]:
+#             operations.append((1, i + 1, t[i]))
+#             s = t[i] * (i + 1) + s[i + 1:]  # 更新s字符串为操作后的状态
+#         i -= 1
+#     return operations
+
+# # 示例输入
+# s = "aabc"
+# t = "abcc"
+
+# # 计算所需的最小操作次数及具体操作
+# operations1 = min_operations_to_equal(s, t)
+# operations2 = min_operations_to_equal(t, s)
+
+# if len(operations1) < len(operations2):
+#     operations = operations1
+# else:
+#     operations = operations2
+# # 输出结果
+# print(len(operations))
+# for op in operations:
+#     print(*op)
+
+
+
+# #include <iostream>
+# #include <linux/limits,h>
+# #include <string>
+# using namespace std;
+# struct node {
+# char a, b;
+# }ans[100100];
+
+# char col[9] = {'','a','b'
+
+# int main(){
+#     string x; cin >>x;
+#     char w=x[0],s=x[1];
+#     int ww=w-'a'+ 1;
+#     int ss =s -'0';
+#     int tot = 0;
+#     for(int i=1;i<=8; ++i){
+#         if(1 != ss){
+#             ans[++tot]=(node){w,(char)(i +'@')};
+#         }
+#     }
+#     for(int i=1; i <= 8; ++i){
+#         if(1 != ww){
+#             ans[++tot]=(node){colli], s};
+#         }
+#     }
+
+#     for(int i=1;i<8;++1){
+#         for(int j=1;j<=8; ++j){
+#             if(i+j==ww + ss){
+#                 if(i !=ss |lj != ww){
+#                     ans[++tot]=(node){col[j],(char)}
+#                 }
+#             }
+#         }
+#     } 
+#     for(int i=1;i<=8;++i){
+#         for(int j=1;j<=8; ++j){
+#             if(i-j=ss-ww){
+#                 if(i !=ss ||j!= ww){
+#                     ans[++tot]=(node)fcol[j]，(char)
+#                 }
+#             }
+#         }
+#     }
+
+#     cout << tot.<< endl;
+#     for(int i =1; i <= tot; ++i) cout << ans[i].a <<
+#     return 0;
+
+# #include <linux/limits.h>
+# #include <string>using namespace stdj
+# const int N= 1001008
+# int n, a[N];
+# int main(){
+#     cin >> n;
+#     int tot1 = 0, tot0 = 0;
+#     for(int i=1;i<= n; ++1){
+#         cin >> a[i];
+#         if(a[i]%2=0) tot0++;
+#         else tot1++;
+#     cout << abs(tot1-(n/2))<<endl;
+#     return 0;

test_text_cap.py

@@ -1,5 +0,0 @@
-from minigpt4.common.caption_tools.caption_utils import coco_caption_eval, textcaps_caption_eval
-result_file = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_12epo_0317/20240317165/result/val_vqa_result_text_cap.json"
-annotaion_file = "/mnt/pfs-guan-ssai/nlu/wanghanzi/data/TextCap/TextCaps_0.1_val.json"
-eval = textcaps_caption_eval(annotaion_file, result_file)
-print(eval.eval.item())