diff --git a/minigpt4/datasets/datasets/aok_vqa_datasets.py b/minigpt4/datasets/datasets/aok_vqa_datasets.py index 08f06d9..ceb629f 100755 --- a/minigpt4/datasets/datasets/aok_vqa_datasets.py +++ b/minigpt4/datasets/datasets/aok_vqa_datasets.py @@ -89,8 +89,8 @@ class AOKVQADataset(VQADataset, __DisplMixin): return { "image": data['image'], "image_id": data["image_id"], - # "q_input": q_input, - "q_input": llm_input, + "q_input": q_input, + # "q_input": llm_input, "llm_input": llm_input, "text_input": question, "text_output": answer, @@ -164,8 +164,8 @@ class AOKVQAEvalDataset(VQAEvalDataset, __DisplMixin): "correct_choice_idx": correct_choice_idx_list, "direct_answers": direct_answers_list, "llm_input": llm_input_list, - "q_input": llm_input_list, - # "q_input": q_input_list, + # "q_input": llm_input_list, + "q_input": q_input_list, "gt_answers": gt_answers_list, "source": source_list, } @@ -208,13 +208,13 @@ class AOKVQAEvalDataset(VQAEvalDataset, __DisplMixin): correct_choice_idx = None correct_answer = direct_answers - llm_input = ann.get("llm_input",random.choice(self.instruction_pool).format(question)) + llm_input = ann.get("llm_input",random.choice(self.instruction_pool).format(question, ", ".join(choices))) # llm_input = random.choice(self.instruction_pool).format(question, ", ".join(choices)) return { "image": image, - # "q_input": question, - "q_input": llm_input, + "q_input": question, + # "q_input": llm_input, "llm_input": llm_input, "text_input": question, "question_id": ann["question_id"], diff --git a/minigpt4/datasets/datasets/coco_vqa_datasets.py b/minigpt4/datasets/datasets/coco_vqa_datasets.py index 53fd5d3..99461dc 100755 --- a/minigpt4/datasets/datasets/coco_vqa_datasets.py +++ b/minigpt4/datasets/datasets/coco_vqa_datasets.py @@ -96,8 +96,8 @@ class COCOVQADataset(VQADataset, __DisplMixin): "image": data['image'], "image_id": data["image_id"], "question_id": data["question_id"], - # "q_input": q_input, - "q_input": llm_input, + "q_input": q_input, + # "q_input": llm_input, "llm_input": llm_input, "text_input": question, "text_output": answer, @@ -181,7 +181,7 @@ class COCOVQAEvalDataset(VQAEvalDataset, __DisplMixin): "question_id": ann["question_id"], # "instance_id": ann["instance_id"], # "question": question, - "q_input": llm_input, + # "q_input": llm_input, "q_input": q_input, "llm_input": llm_input, "text_input": question, diff --git a/minigpt4/datasets/datasets/gqa_datasets.py b/minigpt4/datasets/datasets/gqa_datasets.py index 22e98c9..0853dc1 100755 --- a/minigpt4/datasets/datasets/gqa_datasets.py +++ b/minigpt4/datasets/datasets/gqa_datasets.py @@ -75,8 +75,8 @@ class GQADataset(VQADataset, __DisplMixin): # "text_output": ann["fullAnswer"], "text_output": answers, # "instruction_input": instruction, - "q_input": llm_input, - # "q_input": q_input, + # "q_input": llm_input, + "q_input": q_input, "llm_input": llm_input, "gt_answers": answers, "source": "gqa", @@ -150,8 +150,8 @@ class GQAEvalDataset(VQAEvalDataset, __DisplMixin): "gt_answers": answer, "fullAnswer": fullAnswer, "text_output": answer, - # "q_input": q_input, - "q_input": llm_input, + "q_input": q_input, + # "q_input": llm_input, "llm_input": llm_input, "question_id": ann["question_id"], # "instance_id": ann["instance_id"], diff --git a/minigpt4/datasets/datasets/ok_vqa_datasets.py b/minigpt4/datasets/datasets/ok_vqa_datasets.py index ae09a29..c8389e0 100755 --- a/minigpt4/datasets/datasets/ok_vqa_datasets.py +++ b/minigpt4/datasets/datasets/ok_vqa_datasets.py @@ -98,8 +98,8 @@ class OKVQADataset(VQADataset, __DisplMixin): "image_id": data["image_id"], "question_id": data["question_id"], # "instruction_input": instruction, - "q_input": llm_input, - # "q_input": q_input, + # "q_input": llm_input, + "q_input": q_input, "llm_input": llm_input, "text_input": question, "text_output": answer, @@ -181,8 +181,8 @@ class OKVQAEvalDataset(VQAEvalDataset, __DisplMixin): 'image_path': image_path, "question_id": ann["question_id"], "question": question, - # "q_input": q_input, - "q_input": llm_input, + "q_input": q_input, + # "q_input": llm_input, "llm_input": llm_input, "text_input": question, "source": 'okvqa', diff --git a/minigpt4/eval_scripts/eval_vqa.py b/minigpt4/eval_scripts/eval_vqa.py index 19d51b0..1c1d14b 100644 --- a/minigpt4/eval_scripts/eval_vqa.py +++ b/minigpt4/eval_scripts/eval_vqa.py @@ -29,10 +29,21 @@ def list_of_str(arg): def eval_parser(): parser = argparse.ArgumentParser(description="Demo") + parser.add_argument( + "--device", + default=0, + help="device to run the model", + ) parser.add_argument( "--cfg-path", default="/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/minigpt4/projects/qformer_moe_vicuna/eval/vqa_benchmark_evaluation.yaml", help="path to configuration file.") + parser.add_argument( + "--dataset", + default=['vizwiz','hm'], + type=list_of_str, + help="dataset to evaluate", + ) parser.add_argument( "--options", nargs="+", @@ -58,7 +69,6 @@ def init_model(cfg, device): model_cls = registry.get_model_class(model_config.arch) model = model_cls.from_config(model_config).to(device) -# import pudb; pudb.set_trace() key = list(cfg.datasets_cfg.keys())[0] vis_processor_cfg = cfg.datasets_cfg.get(key).vis_processor.train vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg) @@ -68,13 +78,12 @@ def init_model(cfg, device): return model, vis_processor, text_processor parser = eval_parser() -parser.add_argument("--dataset", type=list_of_str, default=['vizwiz','hm'], help="dataset to evaluate") args = parser.parse_args() cfg = Config(args) setup_seeds(cfg) print(cfg._convert_node_to_json(cfg.config)) setup_logger() -device = torch.device("cuda:5" if torch.cuda.is_available() else "cpu") +device = torch.device("cuda:{}".format(args.device) if torch.cuda.is_available() else "cpu") model, vis_processor, _ = init_model(cfg, device) model.eval() diff --git a/minigpt4/models/QformerRouteMoECLS.py b/minigpt4/models/QformerRouteMoECLS.py index 1bfff87..d4a5fd6 100644 --- a/minigpt4/models/QformerRouteMoECLS.py +++ b/minigpt4/models/QformerRouteMoECLS.py @@ -530,7 +530,7 @@ class BertLayer(nn.Module): self.seq_len_dim, attention_output[:, query_length:, :], ) - cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size] + cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size] # add moe query ffn moe_ffn_attention_input = query_attention_output[:, :query_length, :] # [bz, query_length+seq_len, 768] diff --git a/minigpt4/models/QformerRouteMoECLSLN.py b/minigpt4/models/QformerRouteMoECLSLN.py index 8292b96..c6246d1 100644 --- a/minigpt4/models/QformerRouteMoECLSLN.py +++ b/minigpt4/models/QformerRouteMoECLSLN.py @@ -524,7 +524,7 @@ class BertLayer(nn.Module): self.seq_len_dim, attention_output[:, query_length:, :], ) - cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size] + cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size] # add moe query ffn moe_ffn_attention_input = query_attention_output[:, :query_length, :] # [bz, query_length+seq_len, 768] diff --git a/minigpt4/models/QformerRouteMoELNUni.py b/minigpt4/models/QformerRouteMoELNUni.py index 2c1be9a..dbacea3 100644 --- a/minigpt4/models/QformerRouteMoELNUni.py +++ b/minigpt4/models/QformerRouteMoELNUni.py @@ -524,7 +524,7 @@ class BertLayer(nn.Module): self.seq_len_dim, attention_output[:, query_length:, :], ) - cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size] + cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size] # add moe query ffn # query_attention_output size: [bz, query_length+seq_len, 768] diff --git a/minigpt4/models/QformerRouteMoEUni.py b/minigpt4/models/QformerRouteMoEUni.py index a2ceae2..b67bde9 100644 --- a/minigpt4/models/QformerRouteMoEUni.py +++ b/minigpt4/models/QformerRouteMoEUni.py @@ -531,7 +531,7 @@ class BertLayer(nn.Module): self.seq_len_dim, attention_output[:, query_length:, :], ) - cls_hidden = layer_output_text[0][:, 0, :] # [bz, hidden_size] + cls_hidden = layer_output_text[:, 0, :] # [bz, hidden_size] # add moe query ffn # query_attention_output size: [bz, query_length+seq_len, 768] diff --git a/minigpt4/models/moe/beam_search_universal.py b/minigpt4/models/moe/beam_search_universal.py index b435709..b99755b 100644 --- a/minigpt4/models/moe/beam_search_universal.py +++ b/minigpt4/models/moe/beam_search_universal.py @@ -22,7 +22,7 @@ class UniRouteMoELayer(nn.Module): self.route_method = route_method if self.route_method == "pre-route-uni": self.gate = nn.Linear(hidden_size, num_experts, bias=False).float() - elif self.route_method in ["post-route-uni"]: + elif self.route_method in ["post-route-uni",'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']: gate = nn.Linear(hidden_size, 1, bias=False).float() self.gate = gate @@ -38,7 +38,7 @@ class UniRouteMoELayer(nn.Module): def beam_search(self, current_scores_log, beam_scores, expert_route, batch_size): - if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni']: + if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni','uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']: # current_scores_log torch.Size([bz, num_experts-1]) assert beam_scores==None and expert_route==None current_scores = torch.exp(current_scores_log) @@ -143,7 +143,32 @@ class UniRouteMoELayer(nn.Module): # import pdb;pdb.set_trace() return candidate_output, beam_scores, expert_route, beam_idx, importance_loss - def forward_post_route_uni(self, x, beam_scores, expert_route, use_log=True): + + def calculate_cls_gate_score(self, cls_hidden, output_x): + + if self.route_method == 'uni-cls-route': + # cls_hidden = [bz, 768] + gate_score = self.gate(cls_hidden) # bz, 1 + elif self.route_method == 'uni-cls-query-route': # add cls_hiddin on query_token mean pool hidden + mean_output = torch.mean(output_x, dim=1) # bz, 768 + gate_score = self.gate(mean_output+cls_hidden) # bz, 1 + elif self.route_method == 'uni-cls-cross-route': + # cls_hidden as Q, output_x as K, V calculate scaled dot-product attention between Q and K and V + # cls_hidden: bz, 768 + # output_x: bz, 32, 768 + Q = cls_hidden.unsqueeze(1) # bz, 1, 768 + K = output_x # bz, 32, 768 + V = output_x # bz, 32, 768 + # scaled dot-product attention + QK = torch.matmul(Q, K.transpose(-1, -2)) / (K.size(-1) ** 0.5) # bz, 1, 32 + QK = F.softmax(QK, dim=-1) # bz, 1, 32 + gate_score = torch.matmul(QK, V) # bz, 1, 768 + gate_score = gate_score.squeeze(1) # bz, 768 + gate_score = self.gate(gate_score) # bz, 1 + return gate_score + + + def forward_route_uni(self, x, beam_scores, expert_route, use_log=True, cls_hidden=None): if beam_scores == None: batch_size = x.shape[0] @@ -155,8 +180,6 @@ class UniRouteMoELayer(nn.Module): x_masked, x_uniexpert = x[select_expert],x[select_universal] num_tokens = x.shape[1] - import pdb; pdb.set_trace() - def forward_expert(input_x, expert_idx): output_x = self.experts[expert_idx].forward(input_x) return output_x @@ -168,8 +191,14 @@ class UniRouteMoELayer(nn.Module): logits_gate_lst = list() for expert_idx in range(self.num_route_experts): # num_expert-1 output_x = forward_expert(x_masked, expert_idx) - output_x_aver = torch.mean(output_x, dim=1) - gate_score = self.gate(output_x_aver) + + if self.route_method == 'post-route-uni': + output_x_aver = torch.mean(output_x, dim=1) + gate_score = self.gate(output_x_aver) + + elif self.route_method in ['uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route'] and cls_hidden is not None: + gate_score = self.calculate_cls_gate_score(cls_hidden, output_x) + logits_gate_lst.append(gate_score) outputs.append(output_x.unsqueeze(0)) @@ -186,14 +215,12 @@ class UniRouteMoELayer(nn.Module): # beam_scores torch.Size([bz*(num_beam-1)]), expert_route torch.Size([bz*(num_beam-1), layer_n]) current_select_expert = expert_route[:,-1] # torch.Size([bz*(num_beam-1)]) - import pdb; pdb.set_trace() if self.layer_judge == 'first': replicated_tensor = candidate_output_raw.unsqueeze(2).expand(self.num_route_experts, batch_size, self.num_route_beam, num_tokens, self.hidden_size) candidate_output_raw = replicated_tensor.contiguous().view(self.num_route_experts, -1, num_tokens, self.hidden_size) # [bz*num_beams, 32,768] current_scores_t = current_scores.unsqueeze(1).expand(batch_size, self.num_route_beam, self.num_route_experts) current_scores = current_scores_t.contiguous().view(-1, self.num_route_experts) # [bz*(num_beams-1), num_experts-1] - import pdb; pdb.set_trace() candidate_output = candidate_output_raw.permute(1, 0, 2, 3)[beam_idx] # torch.Size([8, 2, 32, 768]) expert_select_matrix = F.one_hot(current_select_expert, self.num_route_experts) if self.weight_type == 'ffn_prob': @@ -203,8 +230,6 @@ class UniRouteMoELayer(nn.Module): output = candidate_output * expert_select_matrix.unsqueeze(-1).unsqueeze(-1) experts_output = torch.sum(output, dim=1) # [bz*num_beams-1, 32, 768] - import pdb; pdb.set_trace() - #################### ### universal expert #################### @@ -220,26 +245,26 @@ class UniRouteMoELayer(nn.Module): output.append(combine_tmp) final_output = torch.cat(output) # [bz*num_beam, 32 ,768] - import pdb; pdb.set_trace() + # import pdb; pdb.set_trace() return final_output, beam_scores, expert_route, beam_idx, importance_loss - def forward(self, x, attention_mask, beam_scores, expert_route, use_log=True): + def forward(self, x, attention_mask, beam_scores, expert_route, cls_hidden): """ if first_layer: x [bz, 32, 768] else: x [bz*num_beams, 32, 768] """ if self.route_method == 'pre-route-uni': candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_pre_route(x, beam_scores, expert_route, use_log=True) - elif self.route_method in ['post-route-uni']: - candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_post_route_uni(x, beam_scores, expert_route, use_log=True) + elif self.route_method in ['post-route-uni', 'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']: + candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_route_uni(x, beam_scores, expert_route, use_log=True, cls_hidden=cls_hidden) - import pdb;pdb.set_trace() return candidate_output, beam_scores, expert_route, beam_idx, importance_loss + if __name__ == '__main__': import sys @@ -305,7 +330,7 @@ if __name__ == '__main__': num_experts=config.moebert_expert_num, num_beams=config.moebert_num_beams, layer_judge = layer_judge, - route_method = "post-route-uni", + route_method = "uni-cls-cross-rout[e", weight_type="ffn_prob" ) layer_output = experts_post(x2, None, beam_scores2, expert_route2, False) diff --git a/minigpt4/models/moe/uniroute_moe_layer.py b/minigpt4/models/moe/uniroute_moe_layer.py index 4a2ff02..003645e 100644 --- a/minigpt4/models/moe/uniroute_moe_layer.py +++ b/minigpt4/models/moe/uniroute_moe_layer.py @@ -19,9 +19,9 @@ class UniRouteMoELayer(nn.Module): self.weight_type = weight_type self.route_method = route_method - if self.route_method == "pre-route-uni": - self.gate = nn.Linear(hidden_size, num_experts, bias=False).float() - elif self.route_method in ["post-route-uni"]: + if self.route_method in ['pre-route-uni', 'uni-cls-gate-route']: + self.gate = nn.Linear(hidden_size, self.num_route_experts, bias=False).float() + elif self.route_method in ["post-route-uni",'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']: gate = nn.Linear(hidden_size, 1, bias=False).float() self.gate = gate @@ -35,9 +35,8 @@ class UniRouteMoELayer(nn.Module): # Compute coefficient of variation (i.e. std/mean) squared. return (std_importance_per_expert / mean_importance_per_expert)**2 - def beam_search(self, current_scores_log, beam_scores, expert_route, batch_size): - if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni']: + if self.layer_judge=='first' and self.route_method in ['pre-route-uni', 'post-route-uni','uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route','uni-cls-gate-route']: # current_scores_log torch.Size([bz, num_experts-1]) assert beam_scores==None and expert_route==None current_scores = torch.exp(current_scores_log) @@ -80,7 +79,6 @@ class UniRouteMoELayer(nn.Module): return beam_scores, expert_route, beam_idx - def forward_gate(self, x): """ TODO: Pre forward gate @@ -166,8 +164,16 @@ class UniRouteMoELayer(nn.Module): gate_score = self.gate(gate_score) # bz, 1 return gate_score + def adjust_cls_hidden(self, cls_hidden, output_x): + if cls_hidden.shape[0]/self.num_beams == output_x.shape[0]/self.num_route_beam: + cls_hidden_lst = list() + for i in range(cls_hidden.shape[0]): + if i % self.num_beams != 0: + cls_hidden_lst.append(cls_hidden[i,:]) + cls_hidden = torch.stack(cls_hidden_lst) + return cls_hidden - def forward_route_uni(self, x, beam_scores, expert_route, use_log=True, cls_hidden=None): + def forward_route_uni(self, x, beam_scores, expert_route, cls_hidden=None): if beam_scores == None: batch_size = x.shape[0] @@ -186,6 +192,9 @@ class UniRouteMoELayer(nn.Module): #################### ### route expert #################### + if cls_hidden is not None: + cls_hidden = self.adjust_cls_hidden(cls_hidden, x_masked) + outputs = list() logits_gate_lst = list() for expert_idx in range(self.num_route_experts): # num_expert-1 @@ -194,20 +203,24 @@ class UniRouteMoELayer(nn.Module): if self.route_method == 'post-route-uni': output_x_aver = torch.mean(output_x, dim=1) gate_score = self.gate(output_x_aver) + logits_gate_lst.append(gate_score) elif self.route_method in ['uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route'] and cls_hidden is not None: gate_score = self.calculate_cls_gate_score(cls_hidden, output_x) - - logits_gate_lst.append(gate_score) + logits_gate_lst.append(gate_score) outputs.append(output_x.unsqueeze(0)) candidate_output_raw = torch.cat(outputs) # torch.Size([num_expert-1, bz*(num_beam-1), 32, 768]) - logits_gate = torch.cat(logits_gate_lst,dim=1)# torch.Size([bz*(num_beam-1), num_expert-1]) - current_scores = F.softmax(logits_gate, dim=-1) # torch.Size([bz*(num_beam-1), num_expert-1]) - if use_log: - current_scores_log = torch.log(current_scores) # 取log之后可以直接相加 torch.Size([bz*(num_beam-1), num_expert-1]) + + if self.route_method == 'uni-cls-gate-route': + # universal expert with cls_hidden state into nn.Linear(768,num_experts-1) + logits_gate = self.gate(cls_hidden) + current_scores = F.softmax(logits_gate, dim=-1) # torch.Size([bz*(num_beam-1), num_expert-1]) else: - current_scores_log = current_scores + logits_gate = torch.cat(logits_gate_lst,dim=1)# torch.Size([bz*(num_beam-1), num_expert-1]) + current_scores = F.softmax(logits_gate, dim=-1) # torch.Size([bz*(num_beam-1), num_expert-1]) + + current_scores_log = torch.log(current_scores) # 取log之后可以直接相加 torch.Size([bz*(num_beam-1), num_expert-1]) importance_loss = self._importance_auxiliary_loss(current_scores) beam_scores, expert_route, beam_idx = self.beam_search(current_scores_log, beam_scores, expert_route, current_scores_log.shape[0]) @@ -254,9 +267,9 @@ class UniRouteMoELayer(nn.Module): else: x [bz*num_beams, 32, 768] """ if self.route_method == 'pre-route-uni': - candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_pre_route(x, beam_scores, expert_route, use_log=True) - elif self.route_method in ['post-route-uni', 'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route']: - candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_route_uni(x, beam_scores, expert_route, use_log=True, cls_hidden=cls_hidden) + candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_pre_route(x, beam_scores, expert_route) + elif self.route_method in ['post-route-uni', 'uni-cls-route', 'uni-cls-query-route', 'uni-cls-cross-route','uni-cls-gate-route']: + candidate_output, beam_scores, expert_route, beam_idx, importance_loss = self.forward_route_uni(x, beam_scores, expert_route, cls_hidden=cls_hidden) return candidate_output, beam_scores, expert_route, beam_idx, importance_loss diff --git a/minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval.yaml b/minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval.yaml new file mode 100644 index 0000000..b9aa429 --- /dev/null +++ b/minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval.yaml @@ -0,0 +1,143 @@ + # Copyright (c) 2022, salesforce.com, inc. + # All rights reserved. + # SPDX-License-Identifier: BSD-3-Clause + # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + +model: + arch: blip2_vicuna_instruct + model_type: vicuna7b_pretrain + load_pretrained: True + load_finetuned: True + vit_model: eva_clip_g + pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth" + finetuned: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/checkpoint_7.pth" + q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth" + + # vit encoder + image_size: 224 + drop_path_rate: 0 + use_grad_checkpoint: False + vit_precision: "fp16" + + # Q-Former + num_query_token: 32 + qformer_text_input: True + + # vicuna + llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1" + prompt: "" + max_txt_len: 256 + max_output_txt_len: 256 + + # freeze + freeze_vit: True + freeze_llm: True + freeze_qformer: False + freeze_t5_proj: False + + # moe + general_version: 'uni_route_moe' + moebert_route_method: 'uni-cls-cross-route' + moebert_load_balance: 0.01 + moebert_expert_num: 4 + moebert_num_beams: 4 + moe_weight_type: 'ffn_prob' + use_balance_loss: False + ln_position: "out" + +datasets: + # gqa: + # type: balanced_sft_raw_eval + # batch_size: 32 + # vis_processor: + # eval: + # name: "blip2_image_eval" + # image_size: 224 + # text_processor: + # eval: + # name: "blip_caption" + + # coco_vqa: # 658104 + # type: vqa_v2_eval + # batch_size: 32 + # vis_processor: + # eval: + # name: "blip2_image_eval" + # image_size: 224 + # text_processor: + # eval: + # name: "blip_caption" + + coco_caption: # 414113 train + type: coco_cap_eval + batch_size: 32 + vis_processor: + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + eval: + name: "blip_caption" + + # ok_vqa: # train, valid (9009, 5046) + # type: ok_vqa_eval + # batch_size: 32 + # vis_processor: + # eval: + # name: "blip2_image_eval" + # image_size: 224 + # text_processor: + # eval: + # name: "blip_caption" + + # aok_vqa: # train: 17056, val: 1145 + # batch_size: 32 + # vis_processor: + # eval: + # name: "blip2_image_eval" + # image_size: 224 + # text_processor: + # eval: + # name: "blip_caption" + + textcaps_caption: # train: 109765, val: 15830 + batch_size: 32 + vis_processor: + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + eval: + name: "blip_caption" + +run: + task: instruction_tuning + # optimizer + lr_sched: "linear_warmup_cosine_lr" + init_lr: 5e-5 + min_lr: 1e-6 + warmup_lr: 1e-6 + log_freq: 5 + save_freq: 1500 + + weight_decay: 0.05 + max_epoch: 10 + num_workers: 4 + warmup_steps: 600 + iters_per_epoch: 3000 + + seed: 42 + output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/eval/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/" + + amp: True + resume_ckpt_path: null + + evaluate: True + test_splits: ["val"] + # test_splits: ["test"] + + device: "cuda" + world_size: 1 + dist_url: "env://" + distributed: True + diff --git a/minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval1.yaml b/minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval1.yaml new file mode 100644 index 0000000..b29f383 --- /dev/null +++ b/minigpt4/projects/qformer_moe_cls_vicuna/eval/mix6_cls_route_uni_caption_eval1.yaml @@ -0,0 +1,113 @@ + # Copyright (c) 2022, salesforce.com, inc. + # All rights reserved. + # SPDX-License-Identifier: BSD-3-Clause + # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + +model: + arch: blip2_vicuna_instruct + model_type: vicuna7b_pretrain + load_pretrained: True + load_finetuned: True + vit_model: eva_clip_g + pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth" + finetuned: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/checkpoint_7.pth" + q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth" + + # vit encoder + image_size: 224 + drop_path_rate: 0 + use_grad_checkpoint: False + vit_precision: "fp16" + + # Q-Former + num_query_token: 32 + qformer_text_input: True + + # vicuna + llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1" + prompt: "" + max_txt_len: 256 + max_output_txt_len: 256 + + # freeze + freeze_vit: True + freeze_llm: True + freeze_qformer: False + freeze_t5_proj: False + + # moe + general_version: 'uni_route_moe' + moebert_route_method: 'uni-cls-cross-route' + moebert_load_balance: 0.01 + moebert_expert_num: 4 + moebert_num_beams: 4 + moe_weight_type: 'ffn_prob' + use_balance_loss: False + ln_position: "out" + +datasets: + + coco_caption: # 414113 train + type: coco_cap_eval + batch_size: 32 + vis_processor: + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + eval: + name: "blip_caption" + + textcaps_caption: # train: 109765, val: 15830 + batch_size: 32 + vis_processor: + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + eval: + name: "blip_caption" + +evaluation_datasets: + vizwiz: + eval_file_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/VizWiz/val.json + img_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/VizWiz/val + max_new_tokens: 20 + batch_size: 32 + hm: + eval_file_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/hm_data/dev_seen.jsonl + img_path: /mnt/pfs-guan-ssai/nlu/wanghanzi/data/hm_data/ + max_new_tokens: 20 + batch_size: 32 + +run: + task: instruction_tuning + # optimizer + lr_sched: "linear_warmup_cosine_lr" + init_lr: 5e-5 + min_lr: 1e-6 + warmup_lr: 1e-6 + log_freq: 5 + save_freq: 1500 + + weight_decay: 0.05 + max_epoch: 10 + num_workers: 4 + warmup_steps: 600 + iters_per_epoch: 3000 + + seed: 42 + output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/eval/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/" + + amp: True + resume_ckpt_path: null + + evaluate: True + # test_splits: ["val"] + test_splits: ["test"] + + device: "cuda" + world_size: 1 + dist_url: "env://" + distributed: True + diff --git a/minigpt4/projects/qformer_moe_cls_vicuna/eval/mix_vqa_coco_vicuna_eval_coco_vqa_test.yaml b/minigpt4/projects/qformer_moe_cls_vicuna/eval/mix_vqa_coco_vicuna_eval_coco_vqa_test.yaml new file mode 100644 index 0000000..56a94f5 --- /dev/null +++ b/minigpt4/projects/qformer_moe_cls_vicuna/eval/mix_vqa_coco_vicuna_eval_coco_vqa_test.yaml @@ -0,0 +1,89 @@ + # Copyright (c) 2022, salesforce.com, inc. + # All rights reserved. + # SPDX-License-Identifier: BSD-3-Clause + # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + +model: + arch: blip2_vicuna_instruct + model_type: vicuna7b_pretrain + load_pretrained: True + load_finetuned: True + vit_model: eva_clip_g + pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth" + finetuned: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/checkpoint_7.pth" + q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth" + + # vit encoder + image_size: 224 + drop_path_rate: 0 + use_grad_checkpoint: False + vit_precision: "fp16" + + # Q-Former + num_query_token: 32 + qformer_text_input: True + + # vicuna + llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1" + prompt: "" + max_txt_len: 256 + max_output_txt_len: 256 + + # freeze + freeze_vit: True + freeze_llm: True + freeze_qformer: False + freeze_t5_proj: False + + # moe + general_version: 'uni_route_moe' + moebert_route_method: 'uni-cls-cross-route' + moebert_load_balance: 0.01 + moebert_expert_num: 4 + moebert_num_beams: 4 + moe_weight_type: 'ffn_prob' + use_balance_loss: False + ln_position: "out" + +datasets: + coco_vqa: # 658104 + type: vqa_v2_eval + batch_size: 48 + vis_processor: + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + eval: + name: "blip_caption" + +run: + task: instruction_tuning + # optimizer + lr_sched: "linear_warmup_cosine_lr" + init_lr: 5e-5 + min_lr: 1e-6 + warmup_lr: 1e-6 + log_freq: 5 + save_freq: 1500 + + weight_decay: 0.05 + max_epoch: 10 + num_workers: 4 + warmup_steps: 600 + iters_per_epoch: 3000 + + seed: 42 + output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/eval/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/" + + amp: True + resume_ckpt_path: null + + evaluate: True + test_splits: ["test"] + + device: "cuda" + world_size: 1 + dist_url: "env://" + distributed: True + diff --git a/minigpt4/projects/qformer_moe_route_universal_vicuna/train/mix6_qformer_moe_route_uni_blip2_vicuna7b_0328.yaml b/minigpt4/projects/qformer_moe_cls_vicuna/train/mix6_qformer_moe_route_cls_blip2_vicuna7b_0329.yaml similarity index 91% rename from minigpt4/projects/qformer_moe_route_universal_vicuna/train/mix6_qformer_moe_route_uni_blip2_vicuna7b_0328.yaml rename to minigpt4/projects/qformer_moe_cls_vicuna/train/mix6_qformer_moe_route_cls_blip2_vicuna7b_0329.yaml index 2a3b186..299663e 100644 --- a/minigpt4/projects/qformer_moe_route_universal_vicuna/train/mix6_qformer_moe_route_uni_blip2_vicuna7b_0328.yaml +++ b/minigpt4/projects/qformer_moe_cls_vicuna/train/mix6_qformer_moe_route_cls_blip2_vicuna7b_0329.yaml @@ -35,9 +35,10 @@ model: freeze_t5_proj: False # moe + # general_version: 'cls_route_moe' general_version: 'uni_route_moe' - moebert_route_method: "post-route-uni" - moebert_load_balance: 0.05 + moebert_route_method: 'uni-cls-cross-route' + moebert_load_balance: 0.01 moebert_expert_num: 3 moebert_num_beams: 3 moe_weight_type: 'ffn_prob' @@ -154,13 +155,13 @@ run: save_freq: 1500 weight_decay: 0.05 - max_epoch: 8 + max_epoch: 10 num_workers: 4 warmup_steps: 600 iters_per_epoch: 5000 seed: 42 - output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0328/" + output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330/" amp: True resume_ckpt_path: null @@ -170,7 +171,7 @@ run: valid_splits: ["val"] wandb_log: False - # job_name: mix6_uni_route_post_5e5_3ex3b_005_10ep_0319 + # job_name: mix6_uni_cls_route_post_5e5_3ex3b_005_8ep_0330 device: "cuda" world_size: 1 diff --git a/minigpt4/projects/qformer_moe_route_universal_vicuna/train/mix6_qformer_moe_route_uni_blip2_vicuna7b_0329.yaml b/minigpt4/projects/qformer_moe_route_universal_vicuna/train/mix6_qformer_moe_route_uni_blip2_vicuna7b_0329.yaml new file mode 100644 index 0000000..a871d94 --- /dev/null +++ b/minigpt4/projects/qformer_moe_route_universal_vicuna/train/mix6_qformer_moe_route_uni_blip2_vicuna7b_0329.yaml @@ -0,0 +1,180 @@ + # Copyright (c) 2022, salesforce.com, inc. + # All rights reserved. + # SPDX-License-Identifier: BSD-3-Clause + # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause + +model: + arch: blip2_vicuna_instruct + model_type: vicuna7b_pretrain + load_pretrained: True + load_finetuned: False + vit_model: eva_clip_g + pretrained: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth" + q_former_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/blip2/blip2_vicuna7b/blip2_pretrained_vicuna7b.pth" + + # vit encoder + image_size: 224 + drop_path_rate: 0 + use_grad_checkpoint: False + vit_precision: "fp16" + + # Q-Former + num_query_token: 32 + qformer_text_input: True + + # vicuna7b + llm_model: "/mnt/pfs-guan-ssai/nlu/wanghanzi/models/vicuna-7b-v1.1" + prompt: "" + max_txt_len: 256 + max_output_txt_len: 256 + + # freeze + freeze_vit: True + freeze_llm: True + freeze_qformer: False + freeze_t5_proj: False + + # moe + general_version: 'base' + # general_version: 'uni_route_moe' + # moebert_route_method: "post-route-uni" + # moebert_load_balance: 0.05 + # moebert_expert_num: 3 + # moebert_num_beams: 3 + # moe_weight_type: 'ffn_prob' + # use_balance_loss: False + # ln_position: "out" + +datasets: + gqa: # train: 943000, 12578, 12578) + type: balanced_sft_raw + batch_size: 32 + vis_processor: + train: + name: "blip2_image_train" + image_size: 224 + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + eval: + name: "blip_caption" + sample_ratio: 10 + + ok_vqa: # train, valid (9009, 5046) + batch_size: 32 + vis_processor: + train: + name: "blip2_image_train" + image_size: 224 + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + eval: + name: "blip_caption" + sample_ratio: 1 + + coco_vqa: # 658104 + batch_size: 32 + vis_processor: + train: + name: "blip2_image_train" + image_size: 224 + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + eval: + name: "blip_caption" + sample_ratio: 9 + + coco_caption: # 414113 train + batch_size: 32 + vis_processor: + train: + name: "blip2_image_train" + image_size: 224 + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + eval: + name: "blip_caption" + sample_ratio: 7 + + aok_vqa: # train: 17056, val: 1145 + batch_size: 32 + vis_processor: + train: + name: "blip2_image_train" + image_size: 224 + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + eval: + name: "blip_caption" + sample_ratio: 2 + # sample_ratio: 1 + + textcaps_caption: # train: 109765, val: 15830 + batch_size: 32 + vis_processor: + train: + name: "blip2_image_train" + image_size: 224 + eval: + name: "blip2_image_eval" + image_size: 224 + text_processor: + train: + name: "blip_caption" + eval: + name: "blip_caption" + # sample_ratio: 3 + sample_ratio: 4 + +run: + task: instruction_tuning + lr_sched: "linear_warmup_cosine_lr" + init_lr: 5e-5 + min_lr: 1e-6 + warmup_lr: 1e-6 + log_freq: 5 + save_freq: 1500 + + weight_decay: 0.05 + max_epoch: 8 + num_workers: 4 + warmup_steps: 600 + iters_per_epoch: 5000 + + seed: 40 + # output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0328/" + output_dir: "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_base_lr5e5_textinqf_8epo_0329/" + + amp: True + resume_ckpt_path: null + + evaluate: False + train_splits: ["train"] + valid_splits: ["val"] + + wandb_log: False + # job_name: mix6_uni_route_post_5e5_3ex3b_005_10ep_0319 + + device: "cuda" + world_size: 1 + dist_url: "env://" + distributed: True \ No newline at end of file diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv new file mode 100644 index 0000000..bdfc6f2 --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.320559707425666,54.43,73.75,71.61572052401746,64.02907005786079 +58.936237875655905,56.23,75.25,71.09170305676857,65.37698523310611 +60.99538877404993,55.3,76.4,73.01310043668123,66.42712230268279 +61.146446175862614,58.62,76.58,73.1004366812227,67.36172071427133 +62.5934170774368,58.5,77.5,73.88646288209607,68.11996998988322 +62.41850850691684,57.47,78.24,73.7117903930131,67.96007472498249 +62.99888694546033,58.08,78.88,74.235807860262,68.54867370143059 +63.77802512323104,58.72,79.34,74.06113537117905,68.97479012360252 +63.74622356495468,58.5,79.51,74.235807860262,68.99800785630417 +63.825727460645574,58.41,79.63,74.41048034934498,69.06905195249763 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..d6768a1 --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,8 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.40976307839084,53.92,74.0,71.26637554585152,64.14903465606059 +60.136746700588326,56.03,75.27,71.70305676855895,65.78495086728682 +60.99538877404993,56.27,76.52,73.27510917030567,66.7651244860889 +61.94943552234059,57.39,76.95,74.06113537117905,67.58764272337991 +62.55366512959135,58.17,77.83,73.53711790393012,68.02269575838037 +63.26920019080935,57.8,78.7,75.54585152838428,68.8287629297984 +63.46000954046749,58.15,79.09,74.58515283842794,68.82129059472385 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..0ce33f1 --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,7 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.948640483383684,53.97,73.92,71.17903930131004,64.00441994617344 +60.27985371283193,54.86,75.51,71.79039301310044,65.6100616814831 +61.138495786293525,56.33,76.33,72.66375545851528,66.61556281120221 +61.631419939577036,58.13,76.89,73.1877729257642,67.45979821633532 +62.513913181745906,57.35,77.96,72.5764192139738,67.60008309892993 +63.126093178565746,57.36,78.63,74.14847161572052,68.31614119857156 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..2af372e --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,7 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.06789632692002,54.44,73.8,71.09170305676857,64.09989984592214 +59.85848306567022,56.06,75.25,72.75109170305677,65.97989369218175 +60.717125139131824,55.3,76.37,73.1877729257642,66.39372451622401 +61.34520591508984,57.32,76.94,73.62445414847161,67.30741501589037 +62.6649705835586,58.16,77.93,72.5764192139738,67.8328474493831 +62.943234218476704,59.1,78.55,74.32314410480349,68.72909458082005 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..c835f34 --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,8 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.203052949594536,53.68,73.68,70.48034934497817,63.76085057364318 +59.09524566703769,54.05,74.96,72.75109170305677,65.21408434252362 +61.18619812370806,56.16,76.09,73.53711790393012,66.74332900690955 +62.03688980760057,57.59,76.64,72.92576419213974,67.29816349993509 +62.47416123390046,57.58,77.69,72.75109170305677,67.62381323423931 +63.014787724598506,57.61,78.51,73.97379912663754,68.27714671280901 +63.43615837176022,57.94,79.02,73.62445414847161,68.50515313005796 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..2ed1b58 --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.726029575449196,54.51,73.9,71.9650655021834,64.27527376940814 +59.5802194307521,55.89,75.12,72.83842794759825,65.8571618445876 +60.605819685164576,56.41,75.93,74.06113537117905,66.7517387640859 +61.43266020034982,57.36,76.65,73.1004366812227,67.13577422039313 +62.32310383208777,57.71,77.76,74.235807860262,68.00722792308744 +62.80807759580219,58.29,78.22,73.7117903930131,68.25746699720382 +63.07839084115122,58.6,78.88,74.235807860262,68.6985496753533 +63.722372396247415,58.5,79.24,74.67248908296943,69.03371536980421 +63.89728096676737,58.97,79.44,73.88646288209607,69.04843596221586 +63.84957862935284,58.62,79.56,73.88646288209607,68.97901037786222 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..41f4e7c --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,8 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.99634282079822,53.79,73.68,70.82969432314411,63.82400928598558 +59.68357449515026,54.46,75.16,72.40174672489083,65.42633030501027 +60.939736047066305,55.99,75.88,71.87772925764192,66.17186632617705 +62.00508824932422,58.43,76.67,72.48908296943232,67.39854280468914 +62.8716807123549,58.22,77.73,73.88646288209607,68.17703589861274 +62.99093655589124,58.08,78.53,74.06113537117905,68.41551798176756 +63.42820798219113,58.48,78.93,74.235807860262,68.76850396061329 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..32bb78d --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,7 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.932739704245506,54.43,73.9,71.52838427947599,64.19778099593037 +60.057242804897434,55.95,75.28,72.31441048034935,65.9004133213117 +61.44061058991891,54.26,76.24,73.1877729257642,66.28209587892077 +61.806328510096996,57.5,76.83,73.44978165938865,67.3965275423714 +62.56956590872953,58.33,77.89,73.7991266375546,68.14717313657103 +62.760375258387654,58.37,78.45,74.32314410480349,68.47587984079779 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..c0f85f8 --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,10 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.940690093814595,54.81,73.85,71.09170305676857,64.17309828764579 +60.20034981714104,55.52,75.37,72.5764192139738,65.9166922577787 +61.19414851327715,54.56,76.38,73.27510917030567,66.35231442089571 +61.217999681984416,57.83,76.88,73.36244541484717,67.3226112742079 +62.64111941485133,57.97,77.97,72.92576419213974,67.87672090174777 +63.03863889330577,58.54,78.49,72.75109170305677,68.20493264909064 +63.61106694228017,58.49,79.1,74.06113537117905,68.8155505783648 +63.66671966926379,58.68,79.56,72.92576419213974,68.70812096535089 +64.0244871998728,59.23,79.74,73.7117903930131,69.17656939822146 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..b6ab696 --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,8 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.78963269200191,53.55,73.54,70.04366812227074,63.48082520356816 +59.95388774049929,54.89,75.12,71.52838427947599,65.37306800499383 +60.7966290348227,55.93,76.01,72.83842794759825,66.39376424560524 +61.84608045794244,56.83,76.7,73.44978165938865,67.20646552933277 +62.943234218476704,57.48,77.75,74.235807860262,68.10226051968468 +62.67292097312769,57.44,78.43,75.4585152838428,68.50035906424263 +63.36460486563842,57.82,78.92,75.10917030567686,68.80344379282882 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..fcad1ff --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,8 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.020193989505486,53.7,73.8,70.21834061135371,63.6846336502148 +60.33550643981555,54.9,75.12,72.40174672489083,65.68931329117659 +60.947686436635394,55.55,76.29,73.01310043668123,66.45019671832915 +61.45651136905709,57.24,76.39,73.62445414847161,67.17774137938218 +62.65702019398951,57.34,77.73,75.37117903930131,68.2745498083227 +63.5156622674511,57.95,78.4,73.62445414847161,68.37252910398068 +63.6587692796947,57.77,78.75,74.49781659388647,68.6691464683953 diff --git a/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..e0438d7 --- /dev/null +++ b/process_log/0401_results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.218953728732714,55.12,74.19,71.52838427947599,64.51433450205218 +60.0095404674829,55.87,75.27,72.66375545851528,65.95332398149955 +60.73302591827,56.65,76.19,73.44978165938865,66.75570189441466 +61.42470981078073,58.09,76.81,73.44978165938865,67.44362286754235 +62.37875655907139,57.71,78.0,73.53711790393012,67.90646861575038 +62.73652408968039,57.99,78.57,73.27510917030567,68.14290831499652 +63.26124980124026,58.96,79.16,74.75982532751092,69.03526878218779 +63.626967721418346,58.1,79.68,74.235807860262,68.9106938954201 +63.77007473366195,58.19,79.87,74.32314410480349,69.03830470961636 +63.626967721418346,57.76,79.98,74.49781659388647,68.9661960788262 diff --git a/process_log/caption_result/results_coco_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv b/process_log/caption_result/results_coco_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv new file mode 100644 index 0000000..1617ba4 --- /dev/null +++ b/process_log/caption_result/results_coco_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv @@ -0,0 +1,9 @@ +Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr +0.8182082924454812,0.6638925862784945,0.5165455592392635,0.3928161572696676,0.2920462773984222,0.5918020291082554,1.2987178029426083 +0.8240854949333832,0.6717746108928253,0.5257706306569868,0.40249821532303487,0.2936994611883999,0.5958284475381215,1.3272042634335437 +0.8246398014401575,0.6743122037328803,0.5292565031234792,0.40578728669235764,0.2964349874781921,0.597962455259089,1.3321248494065205 +0.8270564620397717,0.6793843847871991,0.5355102159092959,0.41366323151841006,0.2977148403261682,0.6024717705283715,1.3520977723112642 +0.8245944565058522,0.6751194271857663,0.5306411803879995,0.40888833957017,0.29642409734712727,0.598579033363294,1.3383802889190928 +0.8280804382261628,0.6796044258685356,0.537046439535483,0.4152107936908873,0.2990414503157605,0.602392043880006,1.3531755491021618 +0.8299061115859161,0.6814451792563676,0.5379678657467563,0.4156139159677314,0.2992684448837016,0.6027499590797065,1.3538095043242933 +0.8276954938807027,0.6800987491517746,0.5374861277345877,0.41554847690232694,0.2996663866671749,0.6026710548192867,1.3561296682313817 diff --git a/process_log/caption_result/results_text_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv b/process_log/caption_result/results_text_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv new file mode 100644 index 0000000..6987621 --- /dev/null +++ b/process_log/caption_result/results_text_cap_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv @@ -0,0 +1,9 @@ +Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr +0.7309287396242824,0.5494655932407475,0.4041141975269019,0.2921841096479282,0.23361135981392955,0.48650322014232156,0.9966076482929254 +0.7382464874712654,0.5571950857230415,0.4121904524343956,0.30143019979529023,0.23826500378661586,0.4926618553461225,1.0232964787151213 +0.7350922242488682,0.5559390933443953,0.4113206176871192,0.3009155411187479,0.2374533115810085,0.49274408438489914,1.0099444415721002 +0.7369161501554021,0.556728672691215,0.41086927244074317,0.3004344996181908,0.23806683186363936,0.49184279207452697,1.0280928870205412 +0.7407251133040751,0.5595661891353809,0.4138229944446712,0.3032027996796593,0.2415473861083878,0.4972778899306893,1.0479529588574925 +0.7428951836605517,0.5623862396980985,0.4161405396490859,0.3047709156231164,0.2438504314752837,0.49858613995817086,1.0477488915322588 +0.745208695011462,0.5654882473522628,0.4200936310645993,0.308981454910347,0.24371581659378194,0.49914038443933745,1.053786374859757 +0.7444798335079819,0.5624985319454049,0.4157842492239835,0.3044642812661473,0.2432655219581268,0.49722674885535495,1.056752744677739 diff --git a/process_log/load_log.py b/process_log/load_log.py index 4ad4599..33d62ac 100644 --- a/process_log/load_log.py +++ b/process_log/load_log.py @@ -11,7 +11,53 @@ import pandas as pd # path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_4ex4b_2loss_005_top6layer_textinqf_10epo_0319/20240319110/" # path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr6e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0319/20240319105" # path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_tcap_raw_Qformer_base_lr5e5_10epo_0320_instruct/20240320230/" -path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr3e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0326/20240326134" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr3e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0326/20240326134" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/flant5xxl/base/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328/20240328100/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0328/20240328164" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_base_lr5e5_textinqf_8epo_0329/20240329110" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_8epo_0330/20240329232/" + +# mix5 +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330211/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330210/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330211/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330183/" + +# mix6 3ex3b uni 005 +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330211/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330182/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330173/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330/20240330173/" + +# mix6 3ex3b uni 001 +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330/20240330221/" + +# mi6 non-universal +# cls-cross-route-mix6-001-3ex3b-0330-wanghanzi-master-0 +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330/20240330221/" +# cls-cross-route-mix6-001-4ex4b-wanghanzi-master-0 +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331214/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331214/" + +# mix6 4ex4b uni 001 +path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331/20240331132/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330/20240330222/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331144/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331143/" + +# cls cross +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331215/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331/20240331215/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/" + +# 8ex8b uni +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331/20240331150/" +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_mode_cls/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331/20240331213/" + +# post +# path = "/mnt/pfs-guan-ssai/nlu/wanghanzi/experiments/blip2/vicuna7b/qformer_moe_uni_route/mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Post_Route_Universal_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0331/20240331131/" modes = ['gqa','okvqa','vqav2','aokvqa'] file_name = dict() data = dict() @@ -25,7 +71,11 @@ for mode in modes: data[mode] = accs print(data) df = pd.DataFrame(data) -df.to_csv("/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/results.csv",index=False) +# calculate average score +df['avg'] = df.mean(axis=1) +name = path.split('/')[-3] +df.to_csv(f"/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/0401_results_{name}.csv",index=False) +print("save to /mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/0401_results_{}.csv".format(name)) print(df) modes = ['coco_cap','text_cap'] for mode in modes: @@ -35,7 +85,7 @@ for mode in modes: for line in f: tmp.append(json.loads(line)["val"]) df1 = pd.DataFrame(tmp) - df1.to_csv("/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/results_{}.csv".format(mode),index=False) + df1.to_csv("/mnt/pfs-guan-ssai/nlu/wanghanzi/multimodal/PromptMoE/process_log/caption_result/results_{}_{}.csv".format(mode,name),index=False) print("\n",df1) diff --git a/process_log/results.csv b/process_log/results.csv index 7b7729c..653f2d6 100644 --- a/process_log/results.csv +++ b/process_log/results.csv @@ -1,11 +1,10 @@ -gqa,okvqa,vqav2,aokvqa -56.20130386388933,54.9,73.99,70.65502183406113 -59.98568929877563,56.83,75.05,71.87772925764192 -60.81252981396088,56.97,76.33,72.31441048034935 -61.217999681984416,58.72,76.9,72.13973799126637 -62.63316902528224,57.77,77.9,73.53711790393012 -62.63316902528224,58.7,78.62,72.31441048034935 -63.0863412307203,58.44,79.17,73.01310043668123 -63.99268564159644,58.3,79.67,73.1004366812227 -63.85752901892193,58.26,79.89,73.1877729257642 -64.05628875814915,58.36,79.96,73.27510917030567 +gqa,okvqa,vqav2,aokvqa,avg +57.14740022261091,54.38,74.13,71.52838427947599,64.29644612552173 +59.19860073143583,55.98,75.1,72.5764192139738,65.7137549863524 +60.64557163301002,56.56,75.9,73.01310043668123,66.52966801742281 +61.3770074733662,57.87,76.6,74.06113537117905,67.47703571113631 +62.40260772777866,57.16,77.31,74.32314410480349,67.79893795814553 +62.99888694546033,57.96,78.12,73.1877729257642,68.06666496780613 +62.69677214183494,59.45,78.75,74.235807860262,68.78314500052424 +63.126093178565746,58.74,79.22,74.41048034934498,68.87414338197769 +63.30100174908571,58.32,79.5,73.7117903930131,68.7081980355247 diff --git a/process_log/results_base.csv b/process_log/results_base.csv new file mode 100644 index 0000000..5bcc69c --- /dev/null +++ b/process_log/results_base.csv @@ -0,0 +1,9 @@ +gqa,okvqa,vqav2,aokvqa,avg +53.46636985212275,48.02,71.28,73.27510917030567,61.5103697556071 +55.525520750516776,50.12,72.37,74.32314410480349,63.08466621383006 +56.91683892510733,49.66,73.78,75.54585152838428,63.9756726133729 +57.616473207187155,52.98,74.3,76.85589519650655,65.43809210092343 +58.538718397201464,52.81,74.97,75.9825327510917,65.57531278707329 +59.54841787247575,52.71,75.53,76.68122270742359,66.11741014497483 +59.60407059945937,53.43,75.81,76.68122270742359,66.38132332672075 +59.68357449515026,53.71,75.99,77.117903930131,66.6253696063203 diff --git a/process_log/results_coco_cap.csv b/process_log/results_coco_cap.csv index ce73625..1617ba4 100644 --- a/process_log/results_coco_cap.csv +++ b/process_log/results_coco_cap.csv @@ -1,11 +1,9 @@ Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr -0.823223287530701,0.6717046486084919,0.5289578677489075,0.40968911321073265,0.29959733857127835,0.6000811689013733,1.3453305422681376 -0.8223586805364383,0.6761940235369243,0.5372021051535613,0.42002107530341415,0.3007070731886293,0.6033562057362387,1.3625435411445188 -0.8268533010152048,0.6768505208938964,0.5354603117460063,0.41692252223016474,0.3025309435581166,0.6047665704453451,1.372728338529115 -0.8330185365456568,0.6839215475169012,0.543703824076164,0.42514888177425736,0.3056707793309544,0.6059406486352524,1.3861562304957278 -0.82851059467086,0.6809176651975785,0.540587141996957,0.4231733852045105,0.30531255145802344,0.6074083433970782,1.3834468097216504 -0.826759683808046,0.6786303228237849,0.5393123491853253,0.4218864249035804,0.30435744091261596,0.6061024359922585,1.373555774390539 -0.8276490281491424,0.6807227995567375,0.5428624362839755,0.4269084803683001,0.30608939915890326,0.6078544738015538,1.3817698541866794 -0.8294752997477592,0.684006846582908,0.5462002712862497,0.43036414820903174,0.3072188868161371,0.6112755876824707,1.3946012917260822 -0.8295967394767231,0.6833360908793101,0.5455277195646591,0.4292003829675252,0.3075074477217475,0.6103970688371728,1.392473266614681 -0.8311746625208477,0.685154908394367,0.5473453025692251,0.4310962369321893,0.3077283192093123,0.6112527230048272,1.3960092087389069 +0.8182082924454812,0.6638925862784945,0.5165455592392635,0.3928161572696676,0.2920462773984222,0.5918020291082554,1.2987178029426083 +0.8240854949333832,0.6717746108928253,0.5257706306569868,0.40249821532303487,0.2936994611883999,0.5958284475381215,1.3272042634335437 +0.8246398014401575,0.6743122037328803,0.5292565031234792,0.40578728669235764,0.2964349874781921,0.597962455259089,1.3321248494065205 +0.8270564620397717,0.6793843847871991,0.5355102159092959,0.41366323151841006,0.2977148403261682,0.6024717705283715,1.3520977723112642 +0.8245944565058522,0.6751194271857663,0.5306411803879995,0.40888833957017,0.29642409734712727,0.598579033363294,1.3383802889190928 +0.8280804382261628,0.6796044258685356,0.537046439535483,0.4152107936908873,0.2990414503157605,0.602392043880006,1.3531755491021618 +0.8299061115859161,0.6814451792563676,0.5379678657467563,0.4156139159677314,0.2992684448837016,0.6027499590797065,1.3538095043242933 +0.8276954938807027,0.6800987491517746,0.5374861277345877,0.41554847690232694,0.2996663866671749,0.6026710548192867,1.3561296682313817 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..f117adb --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.49546827794561,56.81,73.81,70.21834061135371,64.33345222232484 +58.97598982350135,54.6,75.05,70.91703056768559,64.88575509779673 +61.408809031642555,56.58,76.51,73.36244541484717,66.96531361162243 +62.020989028462395,58.58,76.98,72.13973799126637,67.43018175493219 +62.346955000795035,56.98,78.14,73.27510917030567,67.68551604277518 +63.43615837176022,57.51,78.88,73.1004366812227,68.23164876324573 +63.79392590236922,57.77,79.41,73.7117903930131,68.67142907384559 +63.722372396247415,58.26,79.93,73.88646288209607,68.94970881958587 +63.57131499443472,58.05,80.13,73.36244541484717,68.77844010232047 +63.817777071076485,58.43,80.19,73.7117903930131,69.0373918660224 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..f80a0c0 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,9 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.638575290189216,55.75,73.72,71.09170305676857,64.30006958673944 +59.111146446175866,56.38,75.02,73.1877729257642,65.92472984298502 +61.43266020034982,56.12,76.5,72.83842794759825,66.72277203698701 +62.283351884242336,58.41,77.06,72.40174672489083,67.5387746522833 +62.31515344251868,57.59,78.2,73.7117903930131,67.95423595888295 +63.42820798219113,57.71,78.77,73.97379912663754,68.47050177720718 +63.316902528223885,58.18,79.44,73.97379912663754,68.72767541371536 +63.74622356495468,58.59,79.9,74.235807860262,69.11800785630417 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..6a5d8ad --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,10 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.78963269200191,56.29,73.86,70.91703056768559,64.46416581492187 +59.66767371601208,54.41,75.21,72.48908296943232,65.4441891713611 +61.33725552552075,56.65,76.37,73.62445414847161,66.9954274184981 +62.013038638893306,59.54,77.12,73.27510917030567,67.98703695229975 +62.927333439338526,57.51,78.17,73.62445414847161,68.05794689695253 +63.67467005883288,57.53,78.9,73.01310043668123,68.27944262387852 +63.5156622674511,58.04,79.41,74.06113537117905,68.75669940965753 +63.642868500556524,58.31,79.97,73.44978165938865,68.8431625399863 +63.937032914612814,58.11,80.18,73.97379912663754,69.05020801031259 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..cc71fe5 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,9 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.21720464302751,56.58,73.83,70.56768558951964,64.29872255813677 +59.40531086023215,55.45,75.26,72.40174672489083,65.62926439628075 +59.810780728255686,52.47,75.44,70.82969432314411,64.63761876284994 +61.75862617268246,58.43,76.42,71.9650655021834,67.14342291871647 +62.77627603752583,57.39,77.56,73.88646288209607,67.90318472990548 +63.141993957703924,57.33,78.13,74.41048034934498,68.25311857676222 +63.44410876132931,57.15,78.86,73.01310043668123,68.11680229950264 +63.34870408650024,57.88,79.29,73.7117903930131,68.55762361987834 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv new file mode 100644 index 0000000..bdfc6f2 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_Base_textinqf_10epo_0331.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.320559707425666,54.43,73.75,71.61572052401746,64.02907005786079 +58.936237875655905,56.23,75.25,71.09170305676857,65.37698523310611 +60.99538877404993,55.3,76.4,73.01310043668123,66.42712230268279 +61.146446175862614,58.62,76.58,73.1004366812227,67.36172071427133 +62.5934170774368,58.5,77.5,73.88646288209607,68.11996998988322 +62.41850850691684,57.47,78.24,73.7117903930131,67.96007472498249 +62.99888694546033,58.08,78.88,74.235807860262,68.54867370143059 +63.77802512323104,58.72,79.34,74.06113537117905,68.97479012360252 +63.74622356495468,58.5,79.51,74.235807860262,68.99800785630417 +63.825727460645574,58.41,79.63,74.41048034934498,69.06905195249763 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..a5b22a9 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.0122435999364,54.2,74.05,71.9650655021834,64.30682727552995 +60.065193194466524,55.18,75.18,72.48908296943232,65.72856904097472 +60.87613293051359,57.05,75.99,73.01310043668123,66.7323083417987 +61.69502305612975,58.69,76.81,73.7991266375546,67.74853742342108 +62.291302273811425,58.12,77.73,73.44978165938865,67.89777098330002 +62.56956590872953,57.97,78.35,73.01310043668123,67.97566658635269 +63.17379551598028,59.22,78.91,73.53711790393012,68.7102283549776 +63.46000954046749,57.9,79.42,73.53711790393012,68.5792818610994 +63.77802512323104,58.46,79.58,73.36244541484717,68.79511763451956 +63.83367785021466,58.36,79.73,73.62445414847161,68.88703299967158 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..d51e5c8 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr4e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,7 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.40976307839084,53.92,74.0,71.26637554585152,64.14903465606059 +60.136746700588326,56.03,75.27,71.70305676855895,65.78495086728682 +60.99538877404993,56.27,76.52,73.27510917030567,66.7651244860889 +61.94943552234059,57.39,76.95,74.06113537117905,67.58764272337991 +62.55366512959135,58.17,77.83,73.53711790393012,68.02269575838037 +63.26920019080935,57.8,78.7,75.54585152838428,68.8287629297984 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..c994d1a --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.54491970106535,54.21,73.9,71.87772925764192,64.38316223967682 +59.93798696136111,54.74,75.12,72.75109170305677,65.63726966610447 +60.77277786611543,55.58,76.23,73.7991266375546,66.59547612591751 +61.66322149785339,57.91,76.69,73.27510917030567,67.38458266703977 +62.56956590872953,57.59,77.73,74.06113537117905,67.98767531997714 +62.85577993321672,58.39,78.58,74.8471615720524,68.66823537631728 +63.33280330736206,58.49,79.11,74.67248908296943,68.90132309758287 +63.61106694228017,58.64,79.56,75.02183406113538,69.20822525085389 +63.69852122754015,58.41,79.8,74.67248908296943,69.14525257762739 +63.68262044840197,58.45,79.9,74.75982532751092,69.19811144397823 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..0ce33f1 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_6ex6b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,7 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.948640483383684,53.97,73.92,71.17903930131004,64.00441994617344 +60.27985371283193,54.86,75.51,71.79039301310044,65.6100616814831 +61.138495786293525,56.33,76.33,72.66375545851528,66.61556281120221 +61.631419939577036,58.13,76.89,73.1877729257642,67.45979821633532 +62.513913181745906,57.35,77.96,72.5764192139738,67.60008309892993 +63.126093178565746,57.36,78.63,74.14847161572052,68.31614119857156 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..c426560 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,6 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.06789632692002,54.44,73.8,71.09170305676857,64.09989984592214 +59.85848306567022,56.06,75.25,72.75109170305677,65.97989369218175 +60.717125139131824,55.3,76.37,73.1877729257642,66.39372451622401 +61.34520591508984,57.32,76.94,73.62445414847161,67.30741501589037 +62.6649705835586,58.16,77.93,72.5764192139738,67.8328474493831 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..a059328 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_Universal_lnout_lr6e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,7 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.203052949594536,53.68,73.68,70.48034934497817,63.76085057364318 +59.09524566703769,54.05,74.96,72.75109170305677,65.21408434252362 +61.18619812370806,56.16,76.09,73.53711790393012,66.74332900690955 +62.03688980760057,57.59,76.64,72.92576419213974,67.29816349993509 +62.47416123390046,57.58,77.69,72.75109170305677,67.62381323423931 +63.014787724598506,57.61,78.51,73.97379912663754,68.27714671280901 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..2ed1b58 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.726029575449196,54.51,73.9,71.9650655021834,64.27527376940814 +59.5802194307521,55.89,75.12,72.83842794759825,65.8571618445876 +60.605819685164576,56.41,75.93,74.06113537117905,66.7517387640859 +61.43266020034982,57.36,76.65,73.1004366812227,67.13577422039313 +62.32310383208777,57.71,77.76,74.235807860262,68.00722792308744 +62.80807759580219,58.29,78.22,73.7117903930131,68.25746699720382 +63.07839084115122,58.6,78.88,74.235807860262,68.6985496753533 +63.722372396247415,58.5,79.24,74.67248908296943,69.03371536980421 +63.89728096676737,58.97,79.44,73.88646288209607,69.04843596221586 +63.84957862935284,58.62,79.56,73.88646288209607,68.97901037786222 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..3640050 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Cross_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,7 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.99634282079822,53.79,73.68,70.82969432314411,63.82400928598558 +59.68357449515026,54.46,75.16,72.40174672489083,65.42633030501027 +60.939736047066305,55.99,75.88,71.87772925764192,66.17186632617705 +62.00508824932422,58.43,76.67,72.48908296943232,67.39854280468914 +62.8716807123549,58.22,77.73,73.88646288209607,68.17703589861274 +62.99093655589124,58.08,78.53,74.06113537117905,68.41551798176756 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..67e32c4 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Gate_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,10 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.940690093814595,54.47,73.98,71.61572052401746,64.25160265445803 +60.12879631101924,55.14,75.16,72.31441048034935,65.68580169784215 +61.09874383844809,55.84,76.32,73.01310043668123,66.56796106878232 +61.95738591190968,58.01,76.9,73.44978165938865,67.57929189282459 +62.752424868818565,58.21,77.94,73.1004366812227,68.00071538751033 +62.8001272062331,58.29,78.52,74.235807860262,68.46148376662377 +63.17379551598028,58.75,79.04,73.88646288209607,68.71256459951908 +63.4520591508984,57.67,79.69,73.88646288209607,68.67463050824861 +63.5872157735729,58.24,79.92,73.62445414847161,68.84291748051113 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..84e7a37 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.14740022261091,54.38,74.13,71.52838427947599,64.29644612552173 +59.19860073143583,55.98,75.1,72.5764192139738,65.7137549863524 +60.64557163301002,56.56,75.9,73.01310043668123,66.52966801742281 +61.3770074733662,57.87,76.6,74.06113537117905,67.47703571113631 +62.40260772777866,57.16,77.31,74.32314410480349,67.79893795814553 +62.99888694546033,57.96,78.12,73.1877729257642,68.06666496780613 +62.69677214183494,59.45,78.75,74.235807860262,68.78314500052424 +63.126093178565746,58.74,79.22,74.41048034934498,68.87414338197769 +63.30100174908571,58.32,79.5,73.7117903930131,68.7081980355247 +63.29305135951662,58.05,79.62,73.7991266375546,68.6905444992678 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..f3a66f1 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.521068532358086,53.93,73.93,70.74235807860262,64.03085665274017 +60.39910955636826,54.91,75.34,71.70305676855895,65.5880415812318 +61.04309111146446,56.33,76.35,73.97379912663754,66.9242225595255 +62.17204643027509,57.25,76.97,72.22707423580786,67.15478016652074 +62.919383049769436,57.44,77.9,73.44978165938865,67.92729117728952 +63.5156622674511,58.08,78.7,74.14847161572052,68.6110334707929 +63.96088408332008,58.17,79.22,74.14847161572052,68.87483892476016 +64.27889966608363,58.38,79.75,73.44978165938865,68.96467033136807 +63.98473525202735,58.32,79.91,73.53711790393012,68.93796328898937 +64.46175862617268,57.95,80.05,73.27510917030567,68.93421694911959 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..192a6d5 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Query_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,6 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.932739704245506,54.43,73.9,71.52838427947599,64.19778099593037 +60.057242804897434,55.95,75.28,72.31441048034935,65.9004133213117 +61.44061058991891,54.26,76.24,73.1877729257642,66.28209587892077 +61.806328510096996,57.5,76.83,73.44978165938865,67.3965275423714 +62.56956590872953,58.33,77.89,73.7991266375546,68.14717313657103 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..d931fd6 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_3ex3b_2loss_005_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.75783113372555,54.51,73.98,71.87772925764192,64.28139009784186 +59.98568929877563,55.07,75.3,72.48908296943232,65.711193067052 +60.88408332008268,56.75,76.03,73.97379912663754,66.90947061168006 +61.75067578311337,57.82,76.87,73.97379912663754,67.60361872743772 +62.442359675624104,57.72,77.97,73.53711790393012,67.91736939488857 +62.8716807123549,57.88,78.51,73.97379912663754,68.30886995974811 +63.308952138654796,58.55,79.09,74.235807860262,68.7961899997292 +63.547463825727455,57.86,79.61,74.67248908296943,68.92248822717423 +63.809826681507396,58.42,79.81,74.49781659388647,69.13441081884847 +63.722372396247415,57.87,79.93,74.32314410480349,68.96137912526272 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..92cb134 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,11 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.97249165209095,54.35,73.91,70.56768558951964,63.950044310402646 +60.24805215455557,55.26,75.14,72.5764192139738,65.80611784213234 +60.8681825409445,55.82,76.32,74.58515283842794,66.89833384484311 +61.99713785975513,57.76,76.9,73.44978165938865,67.52672987978595 +62.76832564795674,58.45,77.85,73.7991266375546,68.21686307137783 +63.16584512641119,57.87,78.61,73.53711790393012,68.29574075758532 +63.84162823978375,58.28,79.01,73.53711790393012,68.66718653592847 +64.31865161392908,58.35,79.56,73.27510917030567,68.87594019605869 +64.41405628875815,58.76,79.68,73.44978165938865,69.0759594870367 +64.62076641755446,58.6,79.84,73.88646288209607,69.23680732491263 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..ff59d6f --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_Route_Universal_lnout_lr5e5_8ex8b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,9 @@ +gqa,okvqa,vqav2,aokvqa,avg +56.940690093814595,54.81,73.85,71.09170305676857,64.17309828764579 +60.20034981714104,55.52,75.37,72.5764192139738,65.9166922577787 +61.19414851327715,54.56,76.38,73.27510917030567,66.35231442089571 +61.217999681984416,57.83,76.88,73.36244541484717,67.3226112742079 +62.64111941485133,57.97,77.97,72.92576419213974,67.87672090174777 +63.03863889330577,58.54,78.49,72.75109170305677,68.20493264909064 +63.61106694228017,58.49,79.1,74.06113537117905,68.8155505783648 +63.66671966926379,58.68,79.56,72.92576419213974,68.70812096535089 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv new file mode 100644 index 0000000..3f02dd5 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_CLS_query_lnout_lr5e5_4ex4b_2loss_001_top6layer_textinqf_10epo_0331.csv @@ -0,0 +1,7 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.020193989505486,53.7,73.8,70.21834061135371,63.6846336502148 +60.33550643981555,54.9,75.12,72.40174672489083,65.68931329117659 +60.947686436635394,55.55,76.29,73.01310043668123,66.45019671832915 +61.45651136905709,57.24,76.39,73.62445414847161,67.17774137938218 +62.65702019398951,57.34,77.73,75.37117903930131,68.2745498083227 +63.5156622674511,57.95,78.4,73.62445414847161,68.37252910398068 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv new file mode 100644 index 0000000..d18fd07 --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_UNI_CLS_Gate_Route_lnout_lr5e5_3ex3b_2loss_001_top6layer_textinqf_10epo_0330.csv @@ -0,0 +1,9 @@ +gqa,okvqa,vqav2,aokvqa,avg +57.33025918269995,54.6,74.14,70.91703056768559,64.24682243759638 +59.643822547304815,55.57,75.28,72.13973799126637,65.6583901346428 +60.7966290348227,56.62,76.23,74.14847161572052,66.9487751626358 +61.806328510096996,57.53,76.92,73.88646288209607,67.53569784804826 +62.67292097312769,57.78,78.04,74.49781659388647,68.24768439175354 +63.05453967244395,57.7,78.55,73.44978165938865,68.18858033295815 +63.16584512641119,58.84,79.15,73.7991266375546,68.73874294099144 +63.75417395452377,58.02,79.59,73.7117903930131,68.76899108688423 diff --git a/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv new file mode 100644 index 0000000..5bcc69c --- /dev/null +++ b/process_log/results_mix_coco_gqa_ao_cocap_textcap_raw_QformerMoE_lr5e5_top6layer_textinqf_8epo_0328.csv @@ -0,0 +1,9 @@ +gqa,okvqa,vqav2,aokvqa,avg +53.46636985212275,48.02,71.28,73.27510917030567,61.5103697556071 +55.525520750516776,50.12,72.37,74.32314410480349,63.08466621383006 +56.91683892510733,49.66,73.78,75.54585152838428,63.9756726133729 +57.616473207187155,52.98,74.3,76.85589519650655,65.43809210092343 +58.538718397201464,52.81,74.97,75.9825327510917,65.57531278707329 +59.54841787247575,52.71,75.53,76.68122270742359,66.11741014497483 +59.60407059945937,53.43,75.81,76.68122270742359,66.38132332672075 +59.68357449515026,53.71,75.99,77.117903930131,66.6253696063203 diff --git a/process_log/results_text_cap.csv b/process_log/results_text_cap.csv index b825d8c..6987621 100644 --- a/process_log/results_text_cap.csv +++ b/process_log/results_text_cap.csv @@ -1,11 +1,9 @@ Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr -0.7354185510459915,0.5532517544390539,0.40861793000486973,0.29983800193167554,0.24116643313333005,0.49331369213983123,1.0410874398111594 -0.7337559429476805,0.5532662736631931,0.4093585176779233,0.2995878219607148,0.24114315075166085,0.49493696747386196,1.0409495699510138 -0.7320402508385379,0.5518304887400776,0.4110219411525107,0.3038313373404956,0.24291044377257376,0.496058725623593,1.0575894770967331 -0.735500933603824,0.557825793289975,0.41590254371174706,0.30692236406930773,0.2383527664105175,0.49362765900900546,1.044158754983385 -0.7398393194706776,0.5620893308030489,0.41820732364444946,0.3078230351155729,0.24483177062749348,0.49847300976639547,1.0688814357108025 -0.7371954616083516,0.5585083902477329,0.4158548739958792,0.30707857737975475,0.244730703390385,0.49852757085109406,1.0729374604921766 -0.743162583518909,0.5634215669347339,0.42011923174763044,0.311173553245093,0.2441994031592775,0.49809622365158585,1.0774442793527572 -0.7394105997457735,0.5596190520647151,0.4174127065616514,0.3086643930692146,0.24516092911412224,0.4984150187439041,1.0745130576356021 -0.7401475669028302,0.5612229049005432,0.4184684096329846,0.3101423931360949,0.2459814560570504,0.49998501882495094,1.0809003006778692 -0.7420413230468877,0.5634982579203833,0.4203003411022801,0.3115589386663839,0.245912807863165,0.4989393568537978,1.0805529900845852 +0.7309287396242824,0.5494655932407475,0.4041141975269019,0.2921841096479282,0.23361135981392955,0.48650322014232156,0.9966076482929254 +0.7382464874712654,0.5571950857230415,0.4121904524343956,0.30143019979529023,0.23826500378661586,0.4926618553461225,1.0232964787151213 +0.7350922242488682,0.5559390933443953,0.4113206176871192,0.3009155411187479,0.2374533115810085,0.49274408438489914,1.0099444415721002 +0.7369161501554021,0.556728672691215,0.41086927244074317,0.3004344996181908,0.23806683186363936,0.49184279207452697,1.0280928870205412 +0.7407251133040751,0.5595661891353809,0.4138229944446712,0.3032027996796593,0.2415473861083878,0.4972778899306893,1.0479529588574925 +0.7428951836605517,0.5623862396980985,0.4161405396490859,0.3047709156231164,0.2438504314752837,0.49858613995817086,1.0477488915322588 +0.745208695011462,0.5654882473522628,0.4200936310645993,0.308981454910347,0.24371581659378194,0.49914038443933745,1.053786374859757 +0.7444798335079819,0.5624985319454049,0.4157842492239835,0.3044642812661473,0.2432655219581268,0.49722674885535495,1.056752744677739 diff --git a/test.py b/test.py new file mode 100644 index 0000000..6917435 --- /dev/null +++ b/test.py @@ -0,0 +1,149 @@ +# from math import factorial +# def combine_number(n,k): +# combinations = factorial(n) / factorial(n - k) +# return combinations + +# # 定义一个函数来计算给定字符串中平衡串的子序列数量 +# def count_balanced_subsequences(s): +# MOD = 10**9 + 7 +# # 初始化字母计数器 +# count = [0] * 26 +# for char in s: +# count[ord(char) - ord('a')] += 1 + +# result = 0 +# while sum(count)>0: +# cnt = 0 +# for i in range(26): +# if count[i] > 0: +# cnt += 1 +# count[i] -= 1 +# print(count) +# result += combine_number(cnt,2) +# print(result) +# return result + +# # 示例输入 +# n = 5 +# s = "ababc" +# # 计算结果 +# result = count_balanced_subsequences(s) +# print(result) + +# def calculate_scores(k, x, y): +# # 根据题目描述，我们有以下等式： +# # a + b + c = k +# # c = a + x +# # c = b - y +# # 由上面两个等式可得：a + x = b - y +# # 因此我们可以解这个线性方程组得到a, b, c的值 + +# # a + (a + x) + (a + x + y) = k +# # 3a + 2x + y = k +# # 解得： +# a = (k - 2*x - y) // 3 +# c = a + x +# b = c + y + +# return a, b, c + +# # 示例输入 +# k, x, y = 441, 1, -20 +# # 计算输出 +# a, b, c = calculate_scores(k, x, y) +# print(a, b, c) + + + +# def min_operations_to_equal(s, t): +# operations = [] +# i = len(s) - 1 +# while i >= 0: +# if s[i] != t[i]: +# operations.append((1, i + 1, t[i])) +# s = t[i] * (i + 1) + s[i + 1:] # 更新s字符串为操作后的状态 +# i -= 1 +# return operations + +# # 示例输入 +# s = "aabc" +# t = "abcc" + +# # 计算所需的最小操作次数及具体操作 +# operations1 = min_operations_to_equal(s, t) +# operations2 = min_operations_to_equal(t, s) + +# if len(operations1) < len(operations2): +# operations = operations1 +# else: +# operations = operations2 +# # 输出结果 +# print(len(operations)) +# for op in operations: +# print(*op) + + + +# #include +# #include +# #include +# using namespace std; +# struct node { +# char a, b; +# }ans[100100]; + +# char col[9] = {'','a','b' + +# int main(){ +# string x; cin >>x; +# char w=x[0],s=x[1]; +# int ww=w-'a'+ 1; +# int ss =s -'0'; +# int tot = 0; +# for(int i=1;i<=8; ++i){ +# if(1 != ss){ +# ans[++tot]=(node){w,(char)(i +'@')}; +# } +# } +# for(int i=1; i <= 8; ++i){ +# if(1 != ww){ +# ans[++tot]=(node){colli], s}; +# } +# } + +# for(int i=1;i<8;++1){ +# for(int j=1;j<=8; ++j){ +# if(i+j==ww + ss){ +# if(i !=ss |lj != ww){ +# ans[++tot]=(node){col[j],(char)} +# } +# } +# } +# } +# for(int i=1;i<=8;++i){ +# for(int j=1;j<=8; ++j){ +# if(i-j=ss-ww){ +# if(i !=ss ||j!= ww){ +# ans[++tot]=(node)fcol[j]，(char) +# } +# } +# } +# } + +# cout << tot.<< endl; +# for(int i =1; i <= tot; ++i) cout << ans[i].a << +# return 0; + +# #include +# #include using namespace stdj +# const int N= 1001008 +# int n, a[N]; +# int main(){ +# cin >> n; +# int tot1 = 0, tot0 = 0; +# for(int i=1;i<= n; ++1){ +# cin >> a[i]; +# if(a[i]%2=0) tot0++; +# else tot1++; +# cout << abs(tot1-(n/2))<