From 53da7fa871722a7e424b6506b45b74e9fbf9b88d Mon Sep 17 00:00:00 2001 From: fuck Date: Fri, 12 May 2023 19:04:53 +0800 Subject: [PATCH] update text direct conversation --- .idea/.gitignore | 3 + .idea/MiniGPT-4.iml | 12 + .idea/inspectionProfiles/Project_Default.xml | 78 ++++++ .../inspectionProfiles/profiles_settings.xml | 6 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + README.md | 222 +++++++----------- README_cn.md | 100 ++++++++ README_official.md | 170 ++++++++++++++ demo.py | 38 ++- examples/e5b0d467fa14e2aa9b77a46b828a4e0.png | Bin 0 -> 64141 bytes minigpt4/conversation/conversation.py | 15 +- 12 files changed, 500 insertions(+), 158 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/MiniGPT-4.iml create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 README_cn.md create mode 100644 README_official.md create mode 100644 examples/e5b0d467fa14e2aa9b77a46b828a4e0.png diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..359bb53 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# 默认忽略的文件 +/shelf/ +/workspace.xml diff --git a/.idea/MiniGPT-4.iml b/.idea/MiniGPT-4.iml new file mode 100644 index 0000000..8b8c395 --- /dev/null +++ b/.idea/MiniGPT-4.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..4448092 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,78 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..8ab91a0 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index 7aa29f2..5525986 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,91 @@ -# MiniGPT-4: Enhancing Vision-language Understanding with Advanced Large Language Models -[Deyao Zhu](https://tsutikgiau.github.io/)* (On Job Market!), [Jun Chen](https://junchen14.github.io/)* (On Job Market!), [Xiaoqian Shen](https://xiaoqian-shen.github.io), [Xiang Li](https://xiangli.ac.cn), and [Mohamed Elhoseiny](https://www.mohamed-elhoseiny.com/). *Equal Contribution +# miniGPT4 +[Chineses version](README_cn.md)

🚀🚀

+This project has modified demo.py and conversation.py to support direct text conversation without the need to upload an image first. -**King Abdullah University of Science and Technology** +**TODO: Support multiple images uploading.** - [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1OK4kYsZphwt5DXchKkzMBjYF6jnkqh4R?usp=sharing) [![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://www.youtube.com/watch?v=__tftoxpBAw&feature=youtu.be) +exmple： +![show](./examples/e5b0d467fa14e2aa9b77a46b828a4e0.png) +The following is the process for configuring the project environment. If you have already completed this step, you can skip the environment configuration and directly run demo.py. -## News -We now provide a pretrained MiniGPT-4 aligned with Vicuna-7B! The demo GPU memory consumption now can be as low as 12GB. +[Offcial repo](https://github.com/Vision-CAIR/MiniGPT-4) +supports two pretrained weights [13B](https://drive.google.com/file/d/1a4zLvaiDBr-36pasffmgpvH5P7CKmpze/view) and [7B](https://drive.google.com/file/d/1a4zLvaiDBr-36pasffmgpvH5P7CKmpze/view) can be used fintuning +**GPU Memory needed** -## Online Demo +* 13B： 23G +* 7B: 11.5G -Click the image to chat with MiniGPT-4 around your images -[![demo](figs/online_demo.png)](https://minigpt-4.github.io) +### Getting start +**0.Environment setup** +make sure you install docker first -## Examples - | | | -:-------------------------:|:-------------------------: -![find wild](figs/examples/wop_2.png) | ![write story](figs/examples/ad_2.png) -![solve problem](figs/examples/fix_1.png) | ![write Poem](figs/examples/rhyme_1.png) +pull up [docker image](https://hub.docker.com/r/bewithmeallmylife/mini-gpt4-runtime-cuda-10.2) +```commandline +docker pull bewithmeallmylife/mini-gpt4-runtime-cuda-10.2:1.0.0 +``` +Build the container, expose the corresponding port to enable launching the frontend UI for local use. +```commandline +nvidia-docker run -v /data:/projects -v /data2:/data2 -p 1118:7778 --shm-size 8G --name minigpt4 -d bewithmeallmylife/mini-gpt4-runtime-cuda-10.2:1.0.0 tail -f /dev/null +``` +Get into docker +```commandline +docker exec -it minigpt4 bash +``` +conda environment **mini-gpt4** already have +```commandline +conda activate mini-gpt4 +``` +note: make sure torch is compatible with your cuda version, in this image, the torch version is 1.12.1+cu10.2 +if your cuda is above 11, then execute this step. -More examples can be found in the [project page](https://minigpt-4.github.io). - - - -## Introduction -- MiniGPT-4 aligns a frozen visual encoder from BLIP-2 with a frozen LLM, Vicuna, using just one projection layer. -- We train MiniGPT-4 with two stages. The first traditional pretraining stage is trained using roughly 5 million aligned image-text pairs in 10 hours using 4 A100s. After the first stage, Vicuna is able to understand the image. But the generation ability of Vicuna is heavilly impacted. -- To address this issue and improve usability, we propose a novel way to create high-quality image-text pairs by the model itself and ChatGPT together. Based on this, we then create a small (3500 pairs in total) yet high-quality dataset. -- The second finetuning stage is trained on this dataset in a conversation template to significantly improve its generation reliability and overall usability. To our surprise, this stage is computationally efficient and takes only around 7 minutes with a single A100. -- MiniGPT-4 yields many emerging vision-language capabilities similar to those demonstrated in GPT-4. - - -![overview](figs/overview.png) - - -## Getting Started -### Installation - -**1. Prepare the code and the environment** - -Git clone our repository, creating a python environment and ativate it via the following command - -```bash -git clone https://github.com/Vision-CAIR/MiniGPT-4.git -cd MiniGPT-4 -conda env create -f environment.yml -conda activate minigpt4 +The inference environment required for miniGPT4 is already included in the image, with PyTorch version 1.12.1+cu10.2, which does not support the sm86 architecture. If the GPU model is RTX A6000 with an architecture of 8.6, a version that supports this architecture such as torch1.12.1+cu11.3 needs to be reinstalled. +```commandline +conda install pytorch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 cudatoolkit=11.3 -c pytorch ``` +**1.Prepare the pretrained model weights.** -**2. Prepare the pretrained Vicuna weights** - -The current version of MiniGPT-4 is built on the v0 versoin of Vicuna-13B. -Please refer to our instruction [here](PrepareVicuna.md) -to prepare the Vicuna weights. -The final weights would be in a single folder in a structure similar to the following: +A total of three pretrained model weights need to be prepared: **vicuna** (7B/14G), **llama** (7B/12.5G), and **miniGPT4** (7B). +* vicuna +Pretrained weights in two sizes, 13B and 7B, are available for download. +```commandline +git lfs install +git clone https://huggingface.co/lmsys/vicuna-13b-delta-v0 # more powerful, need at least 24G gpu memory +# or +git clone https://huggingface.co/lmsys/vicuna-7b-delta-v0 # smaller, need 12G gpu memory ``` +These two weights are not working, we still need llama pretrained wieght. +* llama + +To download the llama weights, which are not officially available for download, a third-party download form is used. Choose between 7B or 13B. +```commandline +wget https://agi.gpt4.org/llama/LLaMA/tokenizer.model -O ./tokenizer.model +wget https://agi.gpt4.org/llama/LLaMA/tokenizer_checklist.chk -O ./tokenizer_checklist.chk +wget https://agi.gpt4.org/llama/LLaMA/7B/consolidated.00.pth -O ./7B/consolidated.00.pth +wget https://agi.gpt4.org/llama/LLaMA/7B/params.json -O ./7B/params.json +wget https://agi.gpt4.org/llama/LLaMA/7B/checklist.chk -O ./7B/checklist.chk +wget https://agi.gpt4.org/llama/LLaMA/13B/consolidated.00.pth -O ./13B/consolidated.00.pth +wget https://agi.gpt4.org/llama/LLaMA/13B/consolidated.01.pth -O ./13B/consolidated.01.pth +wget https://agi.gpt4.org/llama/LLaMA/13B/params.json -O ./13B/params.json +wget https://agi.gpt4.org/llama/LLaMA/13B/checklist.chk -O ./13B/checklist.chk +``` +After downloading llama, we need to transform it to Huggingface format +```commandline +git clone https://github.com/huggingface/transformers.git +python transformers/src/transformers/models/llama/convert_llama_weights_to_hf.py \ + --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path/to/llama-13bOR7b-hf/ +``` +After both the vicuna and llama weights are prepared, they need to be combined to obtain the vicuna weights that can be used. +```commandline +pip install git+https://github.com/lm-sys/FastChat.git@v0.1.10 +python -m fastchat.model.apply_delta --base /path/to/llama-13bOR7b-hf/ --target /path/to/save/working/vicuna/weight/ --delta /path/to/vicuna-13bOR7b-delta-v0/ +``` +Finally get a working weight, the weight folder is like below： +```commandline vicuna_weights ├── config.json ├── generation_config.json @@ -67,104 +93,16 @@ vicuna_weights ├── pytorch_model-00001-of-00003.bin ... ``` +Then, set the path to the vicuna weight in the model config file [here](minigpt4/configs/models/minigpt4.yaml) at Line 16. +* minigpt4 checkpoints -Then, set the path to the vicuna weight in the model config file -[here](minigpt4/configs/models/minigpt4.yaml#L16) at Line 16. +[13B checkpoint](https://drive.google.com/file/d/1a4zLvaiDBr-36pasffmgpvH5P7CKmpze/view?usp=share_link) +[7B checkpoint](https://drive.google.com/file/d/1RY9jV0dyqLX-o38LrumkKRh6Jtaop58R/view?usp=sharing) -**3. Prepare the pretrained MiniGPT-4 checkpoint** - -Download the pretrained checkpoints according to the Vicuna model you prepare. - -| Checkpoint Aligned with Vicuna 13B | Checkpoint Aligned with Vicuna 7B | -:------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------: - [Downlad](https://drive.google.com/file/d/1a4zLvaiDBr-36pasffmgpvH5P7CKmpze/view?usp=share_link) | [Download](https://drive.google.com/file/d/1RY9jV0dyqLX-o38LrumkKRh6Jtaop58R/view?usp=sharing) +Then, set the path to the pretrained checkpoint in the evaluation config file in [eval_configs/minigpt4_eval.yaml](eval_configs/minigpt4_eval.yaml) at Line 11. -Then, set the path to the pretrained checkpoint in the evaluation config file -in [eval_configs/minigpt4_eval.yaml](eval_configs/minigpt4_eval.yaml#L10) at Line 11. - - - -### Launching Demo Locally - -Try out our demo [demo.py](demo.py) on your local machine by running - -``` +**2.run demo.py** +```commandline python demo.py --cfg-path eval_configs/minigpt4_eval.yaml --gpu-id 0 ``` - -To save GPU memory, Vicuna loads as 8 bit by default, with a beam search width of 1. -This configuration requires about 23G GPU memory for Vicuna 13B and 11.5G GPU memory for Vicuna 7B. -For more powerful GPUs, you can run the model -in 16 bit by setting low_resource to False in the config file -[minigpt4_eval.yaml](eval_configs/minigpt4_eval.yaml) and use a larger beam search width. - -Thanks [@WangRongsheng](https://github.com/WangRongsheng), you can also run our code on [Colab](https://colab.research.google.com/drive/1OK4kYsZphwt5DXchKkzMBjYF6jnkqh4R?usp=sharing) - - -### Training -The training of MiniGPT-4 contains two alignment stages. - -**1. First pretraining stage** - -In the first pretrained stage, the model is trained using image-text pairs from Laion and CC datasets -to align the vision and language model. To download and prepare the datasets, please check -our [first stage dataset preparation instruction](dataset/README_1_STAGE.md). -After the first stage, the visual features are mapped and can be understood by the language -model. -To launch the first stage training, run the following command. In our experiments, we use 4 A100. -You can change the save path in the config file -[train_configs/minigpt4_stage1_pretrain.yaml](train_configs/minigpt4_stage1_pretrain.yaml) - -```bash -torchrun --nproc-per-node NUM_GPU train.py --cfg-path train_configs/minigpt4_stage1_pretrain.yaml -``` - -A MiniGPT-4 checkpoint with only stage one training can be downloaded -[here (13B)](https://drive.google.com/file/d/1u9FRRBB3VovP1HxCAlpD9Lw4t4P6-Yq8/view?usp=share_link) or [here (7B)](https://drive.google.com/file/d/1HihQtCEXUyBM1i9DQbaK934wW3TZi-h5/view?usp=share_link). -Compared to the model after stage two, this checkpoint generate incomplete and repeated sentences frequently. - - -**2. Second finetuning stage** - -In the second stage, we use a small high quality image-text pair dataset created by ourselves -and convert it to a conversation format to further align MiniGPT-4. -To download and prepare our second stage dataset, please check our -[second stage dataset preparation instruction](dataset/README_2_STAGE.md). -To launch the second stage alignment, -first specify the path to the checkpoint file trained in stage 1 in -[train_configs/minigpt4_stage1_pretrain.yaml](train_configs/minigpt4_stage2_finetune.yaml). -You can also specify the output path there. -Then, run the following command. In our experiments, we use 1 A100. - -```bash -torchrun --nproc-per-node NUM_GPU train.py --cfg-path train_configs/minigpt4_stage2_finetune.yaml -``` - -After the second stage alignment, MiniGPT-4 is able to talk about the image coherently and user-friendly. - - - - -## Acknowledgement - -+ [BLIP2](https://huggingface.co/docs/transformers/main/model_doc/blip-2) The model architecture of MiniGPT-4 follows BLIP-2. Don't forget to check this great open-source work if you don't know it before! -+ [Lavis](https://github.com/salesforce/LAVIS) This repository is built upon Lavis! -+ [Vicuna](https://github.com/lm-sys/FastChat) The fantastic language ability of Vicuna with only 13B parameters is just amazing. And it is open-source! - - -If you're using MiniGPT-4 in your research or applications, please cite using this BibTeX: -```bibtex -@article{zhu2023minigpt, - title={MiniGPT-4: Enhancing Vision-Language Understanding with Advanced Large Language Models}, - author={Zhu, Deyao and Chen, Jun and Shen, Xiaoqian and Li, Xiang and Elhoseiny, Mohamed}, - journal={arXiv preprint arXiv:2304.10592}, - year={2023} -} -``` - - -## License -This repository is under [BSD 3-Clause License](LICENSE.md). -Many codes are based on [Lavis](https://github.com/salesforce/LAVIS) with -BSD 3-Clause License [here](LICENSE_Lavis.md). diff --git a/README_cn.md b/README_cn.md new file mode 100644 index 0000000..65c9ada --- /dev/null +++ b/README_cn.md @@ -0,0 +1,100 @@ + +## miniGPT4

🚀🚀

+本项目修改了demo.py和conversation.py，能够支持直接文本对话，而无需先上传图片。 + +**TODO: 支持多图上传回答** + +演示： +![show](./examples/e5b0d467fa14e2aa9b77a46b828a4e0.png) + +以下是项目的环境配置过程，如果你已经配好了，跳过环境配置的阶段，直接运行demo.py即可 + +[官方](https://github.com/Vision-CAIR/MiniGPT-4) +提供参数量为[13B](https://drive.google.com/file/d/1a4zLvaiDBr-36pasffmgpvH5P7CKmpze/view)和[7B](https://drive.google.com/file/d/1a4zLvaiDBr-36pasffmgpvH5P7CKmpze/view)的checkpoint可供微调 + +**所需配置** + +官方使用A100显卡 +* 13B： 23G显存 +* 7B: 11.5G显存 + +### 步骤 +**0.环境准备** + +拉取已有docker[镜像](https://hub.docker.com/r/bewithmeallmylife/mini-gpt4-runtime-cuda-10.2) +```commandline +docker pull bewithmeallmylife/mini-gpt4-runtime-cuda-10.2:1.0.0 +``` +构建容器, 暴露对应端口，以便启动前端ui在本地使用 +```commandline +nvidia-docker run -v /data:/projects -v /data2:/data2 -p 1118:7778 --shm-size 8G --name minigpt4 -d bewithmeallmylife/mini-gpt4-runtime-cuda-10.2:1.0.0 tail -f /dev/null +``` +进入容器 +```commandline +docker exec -it minigpt4 bash +``` +启动conda虚拟环境**mini-gpt4** +```commandline +conda activate mini-gpt4 +``` +该镜像中miniGPT4所需的推理环境已有，pytorch版本为1.12.1+cu10.2，并不支持sm86的算力，如果显卡型号为RTX A6000，算力为8.6，需重新安装支持该算力的版本如torch1.12.1+cu11.3 +```commandline +conda install pytorch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 cudatoolkit=11.3 -c pytorch +``` + +**1.准备预训练的模型权重** + +一共需要准备3个预训练的模型权重 vicuna（7B/14G），llama（7B/12.5G），和miniGPT4（7B） +* vicuna +可下载13B和7B两种大小的预训练权重 +```commandline +git lfs install +git clone https://huggingface.co/lmsys/vicuna-13b-delta-v0 # more powerful, need at least 24G gpu memory +# or +git clone https://huggingface.co/lmsys/vicuna-7b-delta-v0 # smaller, need 12G gpu memory +``` +这两个权重还并非可用的，需搭配llama权重激活使用 +* 下载llama权重如下，官方并未开放下载，采用第三方下载形式, 选择7B或13B +```commandline +wget https://agi.gpt4.org/llama/LLaMA/tokenizer.model -O ./tokenizer.model +wget https://agi.gpt4.org/llama/LLaMA/tokenizer_checklist.chk -O ./tokenizer_checklist.chk +wget https://agi.gpt4.org/llama/LLaMA/7B/consolidated.00.pth -O ./7B/consolidated.00.pth +wget https://agi.gpt4.org/llama/LLaMA/7B/params.json -O ./7B/params.json +wget https://agi.gpt4.org/llama/LLaMA/7B/checklist.chk -O ./7B/checklist.chk +wget https://agi.gpt4.org/llama/LLaMA/13B/consolidated.00.pth -O ./13B/consolidated.00.pth +wget https://agi.gpt4.org/llama/LLaMA/13B/consolidated.01.pth -O ./13B/consolidated.01.pth +wget https://agi.gpt4.org/llama/LLaMA/13B/params.json -O ./13B/params.json +wget https://agi.gpt4.org/llama/LLaMA/13B/checklist.chk -O ./13B/checklist.chk +``` +下载完llama权重之后，还需要转换成huggingface的模型格式 +```commandline +git clone https://github.com/huggingface/transformers.git +python transformers/src/transformers/models/llama/convert_llama_weights_to_hf.py \ + --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path/to/llama-13bOR7b-hf/ +``` +当vicuna和llama两个权重都准备好了之后，还需要将它们组合在一起得到能够使用得vicuna权重 +```commandline +pip install git+https://github.com/lm-sys/FastChat.git@v0.1.10 +python -m fastchat.model.apply_delta --base /path/to/llama-13bOR7b-hf/ --target /path/to/save/working/vicuna/weight/ --delta /path/to/vicuna-13bOR7b-delta-v0/ +``` +最终获得一个可以使用的权重，它的文件格式如下： +```commandline +vicuna_weights +├── config.json +├── generation_config.json +├── pytorch_model.bin.index.json +├── pytorch_model-00001-of-00003.bin +... +``` +将该权重文件的路径添加到配置文件minigpt4/configs/models/minigpt4.yaml的第16行 +* minigpt4预训练权重下载 + +[13B的checkpoint](https://drive.google.com/file/d/1a4zLvaiDBr-36pasffmgpvH5P7CKmpze/view?usp=share_link) +[7B的checkpoint](https://drive.google.com/file/d/1RY9jV0dyqLX-o38LrumkKRh6Jtaop58R/view?usp=sharing) + +将下好的权重路径加到配置文件eval_configs/minigpt4_eval.yaml的第11行 + +**2.运行demo.py** +```commandline +python demo.py --cfg-path eval_configs/minigpt4_eval.yaml --gpu-id 0 +``` diff --git a/README_official.md b/README_official.md new file mode 100644 index 0000000..7aa29f2 --- /dev/null +++ b/README_official.md @@ -0,0 +1,170 @@ +# MiniGPT-4: Enhancing Vision-language Understanding with Advanced Large Language Models +[Deyao Zhu](https://tsutikgiau.github.io/)* (On Job Market!), [Jun Chen](https://junchen14.github.io/)* (On Job Market!), [Xiaoqian Shen](https://xiaoqian-shen.github.io), [Xiang Li](https://xiangli.ac.cn), and [Mohamed Elhoseiny](https://www.mohamed-elhoseiny.com/). *Equal Contribution + +**King Abdullah University of Science and Technology** + + [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1OK4kYsZphwt5DXchKkzMBjYF6jnkqh4R?usp=sharing) [![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://www.youtube.com/watch?v=__tftoxpBAw&feature=youtu.be) + + +## News +We now provide a pretrained MiniGPT-4 aligned with Vicuna-7B! The demo GPU memory consumption now can be as low as 12GB. + + +## Online Demo + +Click the image to chat with MiniGPT-4 around your images +[![demo](figs/online_demo.png)](https://minigpt-4.github.io) + + +## Examples + | | | +:-------------------------:|:-------------------------: +![find wild](figs/examples/wop_2.png) | ![write story](figs/examples/ad_2.png) +![solve problem](figs/examples/fix_1.png) | ![write Poem](figs/examples/rhyme_1.png) + +More examples can be found in the [project page](https://minigpt-4.github.io). + + + +## Introduction +- MiniGPT-4 aligns a frozen visual encoder from BLIP-2 with a frozen LLM, Vicuna, using just one projection layer. +- We train MiniGPT-4 with two stages. The first traditional pretraining stage is trained using roughly 5 million aligned image-text pairs in 10 hours using 4 A100s. After the first stage, Vicuna is able to understand the image. But the generation ability of Vicuna is heavilly impacted. +- To address this issue and improve usability, we propose a novel way to create high-quality image-text pairs by the model itself and ChatGPT together. Based on this, we then create a small (3500 pairs in total) yet high-quality dataset. +- The second finetuning stage is trained on this dataset in a conversation template to significantly improve its generation reliability and overall usability. To our surprise, this stage is computationally efficient and takes only around 7 minutes with a single A100. +- MiniGPT-4 yields many emerging vision-language capabilities similar to those demonstrated in GPT-4. + + +![overview](figs/overview.png) + + +## Getting Started +### Installation + +**1. Prepare the code and the environment** + +Git clone our repository, creating a python environment and ativate it via the following command + +```bash +git clone https://github.com/Vision-CAIR/MiniGPT-4.git +cd MiniGPT-4 +conda env create -f environment.yml +conda activate minigpt4 +``` + + +**2. Prepare the pretrained Vicuna weights** + +The current version of MiniGPT-4 is built on the v0 versoin of Vicuna-13B. +Please refer to our instruction [here](PrepareVicuna.md) +to prepare the Vicuna weights. +The final weights would be in a single folder in a structure similar to the following: + +``` +vicuna_weights +├── config.json +├── generation_config.json +├── pytorch_model.bin.index.json +├── pytorch_model-00001-of-00003.bin +... +``` + +Then, set the path to the vicuna weight in the model config file +[here](minigpt4/configs/models/minigpt4.yaml#L16) at Line 16. + +**3. Prepare the pretrained MiniGPT-4 checkpoint** + +Download the pretrained checkpoints according to the Vicuna model you prepare. + +| Checkpoint Aligned with Vicuna 13B | Checkpoint Aligned with Vicuna 7B | +:------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------: + [Downlad](https://drive.google.com/file/d/1a4zLvaiDBr-36pasffmgpvH5P7CKmpze/view?usp=share_link) | [Download](https://drive.google.com/file/d/1RY9jV0dyqLX-o38LrumkKRh6Jtaop58R/view?usp=sharing) + + +Then, set the path to the pretrained checkpoint in the evaluation config file +in [eval_configs/minigpt4_eval.yaml](eval_configs/minigpt4_eval.yaml#L10) at Line 11. + + + +### Launching Demo Locally + +Try out our demo [demo.py](demo.py) on your local machine by running + +``` +python demo.py --cfg-path eval_configs/minigpt4_eval.yaml --gpu-id 0 +``` + +To save GPU memory, Vicuna loads as 8 bit by default, with a beam search width of 1. +This configuration requires about 23G GPU memory for Vicuna 13B and 11.5G GPU memory for Vicuna 7B. +For more powerful GPUs, you can run the model +in 16 bit by setting low_resource to False in the config file +[minigpt4_eval.yaml](eval_configs/minigpt4_eval.yaml) and use a larger beam search width. + +Thanks [@WangRongsheng](https://github.com/WangRongsheng), you can also run our code on [Colab](https://colab.research.google.com/drive/1OK4kYsZphwt5DXchKkzMBjYF6jnkqh4R?usp=sharing) + + +### Training +The training of MiniGPT-4 contains two alignment stages. + +**1. First pretraining stage** + +In the first pretrained stage, the model is trained using image-text pairs from Laion and CC datasets +to align the vision and language model. To download and prepare the datasets, please check +our [first stage dataset preparation instruction](dataset/README_1_STAGE.md). +After the first stage, the visual features are mapped and can be understood by the language +model. +To launch the first stage training, run the following command. In our experiments, we use 4 A100. +You can change the save path in the config file +[train_configs/minigpt4_stage1_pretrain.yaml](train_configs/minigpt4_stage1_pretrain.yaml) + +```bash +torchrun --nproc-per-node NUM_GPU train.py --cfg-path train_configs/minigpt4_stage1_pretrain.yaml +``` + +A MiniGPT-4 checkpoint with only stage one training can be downloaded +[here (13B)](https://drive.google.com/file/d/1u9FRRBB3VovP1HxCAlpD9Lw4t4P6-Yq8/view?usp=share_link) or [here (7B)](https://drive.google.com/file/d/1HihQtCEXUyBM1i9DQbaK934wW3TZi-h5/view?usp=share_link). +Compared to the model after stage two, this checkpoint generate incomplete and repeated sentences frequently. + + +**2. Second finetuning stage** + +In the second stage, we use a small high quality image-text pair dataset created by ourselves +and convert it to a conversation format to further align MiniGPT-4. +To download and prepare our second stage dataset, please check our +[second stage dataset preparation instruction](dataset/README_2_STAGE.md). +To launch the second stage alignment, +first specify the path to the checkpoint file trained in stage 1 in +[train_configs/minigpt4_stage1_pretrain.yaml](train_configs/minigpt4_stage2_finetune.yaml). +You can also specify the output path there. +Then, run the following command. In our experiments, we use 1 A100. + +```bash +torchrun --nproc-per-node NUM_GPU train.py --cfg-path train_configs/minigpt4_stage2_finetune.yaml +``` + +After the second stage alignment, MiniGPT-4 is able to talk about the image coherently and user-friendly. + + + + +## Acknowledgement + ++ [BLIP2](https://huggingface.co/docs/transformers/main/model_doc/blip-2) The model architecture of MiniGPT-4 follows BLIP-2. Don't forget to check this great open-source work if you don't know it before! ++ [Lavis](https://github.com/salesforce/LAVIS) This repository is built upon Lavis! ++ [Vicuna](https://github.com/lm-sys/FastChat) The fantastic language ability of Vicuna with only 13B parameters is just amazing. And it is open-source! + + +If you're using MiniGPT-4 in your research or applications, please cite using this BibTeX: +```bibtex +@article{zhu2023minigpt, + title={MiniGPT-4: Enhancing Vision-Language Understanding with Advanced Large Language Models}, + author={Zhu, Deyao and Chen, Jun and Shen, Xiaoqian and Li, Xiang and Elhoseiny, Mohamed}, + journal={arXiv preprint arXiv:2304.10592}, + year={2023} +} +``` + + +## License +This repository is under [BSD 3-Clause License](LICENSE.md). +Many codes are based on [Lavis](https://github.com/salesforce/LAVIS) with +BSD 3-Clause License [here](LICENSE_Lavis.md). diff --git a/demo.py b/demo.py index b3659f1..2c5927d 100644 --- a/demo.py +++ b/demo.py @@ -28,8 +28,8 @@ def parse_args(): "--options", nargs="+", help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file (deprecate), " - "change to --cfg-options instead.", + "in xxx=yyy format will be merged into config file (deprecate), " + "change to --cfg-options instead.", ) args = parser.parse_args() return args @@ -64,6 +64,7 @@ vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config chat = Chat(model, vis_processor, device='cuda:{}'.format(args.gpu_id)) print('Initialization Finished') + # ======================================== # Gradio Setting # ======================================== @@ -73,7 +74,10 @@ def gradio_reset(chat_state, img_list): chat_state.messages = [] if img_list is not None: img_list = [] - return None, gr.update(value=None, interactive=True), gr.update(placeholder='Please upload your image first', interactive=False),gr.update(value="Upload & Start Chat", interactive=True), chat_state, img_list + return None, gr.update(value=None, interactive=True), gr.update(placeholder="chat with me", + interactive=True), gr.update( + value="Upload & Start Chat", interactive=True), chat_state, img_list + def upload_img(gr_img, text_input, chat_state): if gr_img is None: @@ -81,11 +85,16 @@ def upload_img(gr_img, text_input, chat_state): chat_state = CONV_VISION.copy() img_list = [] llm_message = chat.upload_img(gr_img, chat_state, img_list) - return gr.update(interactive=False), gr.update(interactive=True, placeholder='Type and press Enter'), gr.update(value="Start Chatting", interactive=False), chat_state, img_list + return gr.update(interactive=True), gr.update(interactive=True, placeholder='Type and press Enter'), gr.update( + value="Upload img", interactive=True), chat_state, img_list + def gradio_ask(user_message, chatbot, chat_state): if len(user_message) == 0: return gr.update(interactive=True, placeholder='Input should not be empty!'), chatbot, chat_state + # chat_state = CONV_VISION.copy() + if chat_state == None: + chat_state = CONV_VISION.copy() chat.ask(user_message, chat_state) chatbot = chatbot + [[user_message, None]] return '', chatbot, chat_state @@ -101,12 +110,13 @@ def gradio_answer(chatbot, chat_state, img_list, num_beams, temperature): chatbot[-1][1] = llm_message return chatbot, chat_state, img_list + title = """

Demo of MiniGPT-4

""" description = """

This is the demo of MiniGPT-4. Upload your images and start chatting!

""" article = """

""" -#TODO show examples below +# TODO show examples below with gr.Blocks() as demo: gr.Markdown(title) @@ -118,7 +128,7 @@ with gr.Blocks() as demo: image = gr.Image(type="pil") upload_button = gr.Button(value="Upload & Start Chat", interactive=True, variant="primary") clear = gr.Button("Restart") - + num_beams = gr.Slider( minimum=1, maximum=10, @@ -127,7 +137,7 @@ with gr.Blocks() as demo: interactive=True, label="beam search numbers)", ) - + temperature = gr.Slider( minimum=0.1, maximum=2.0, @@ -141,13 +151,15 @@ with gr.Blocks() as demo: chat_state = gr.State() img_list = gr.State() chatbot = gr.Chatbot(label='MiniGPT-4') - text_input = gr.Textbox(label='User', placeholder='Please upload your image first', interactive=False) - - upload_button.click(upload_img, [image, text_input, chat_state], [image, text_input, upload_button, chat_state, img_list]) - + text_input = gr.Textbox(label='User', placeholder='chat with me', interactive=True) + + upload_button.click(upload_img, [image, text_input, chat_state], + [image, text_input, upload_button, chat_state, img_list]) + # print(img_list) text_input.submit(gradio_ask, [text_input, chatbot, chat_state], [text_input, chatbot, chat_state]).then( gradio_answer, [chatbot, chat_state, img_list, num_beams, temperature], [chatbot, chat_state, img_list] ) - clear.click(gradio_reset, [chat_state, img_list], [chatbot, image, text_input, upload_button, chat_state, img_list], queue=False) + clear.click(gradio_reset, [chat_state, img_list], [chatbot, image, text_input, upload_button, chat_state, img_list], + queue=False) -demo.launch(share=True, enable_queue=True) +demo.launch(server_name="0.0.0.0", server_port=7778, share=True, enable_queue=True) diff --git a/examples/e5b0d467fa14e2aa9b77a46b828a4e0.png b/examples/e5b0d467fa14e2aa9b77a46b828a4e0.png new file mode 100644 index 0000000000000000000000000000000000000000..a27f7692318a599d6ff703a7d7ede51be0459e6b GIT binary patch literal 64141 zcmd?RcT`hb`!0&Qtu)I<1Su*iiZnq$x{7oJ1rk7-D7^*gJt_h!NGCJ_l}>2Vdjymg zAku{Z0U>k`}aLCG9v%T;0KJPQWGCDr?Pbv;cMDaSvs^w;4iGqjPp5nh_l`>WHJXQnwG zd{%X$d*6N`TI5aZwF@WI7%p9Sb)xyr+s_{k-IRINjrOc9SD-t!#Cz^h=*QRBqm=%( zKY3TN(YiwI+s6a%6S{B)2=>i_+`7%ax*Ec8XO?Lp!EOoETaas9W~b-p1M*v5Z}9~I zecSuIfH{3{MVj~Lg0u|{Z=9EK{JW9w_dUAOc;P(K(ucciRlQj!!3TbS{}F4gkDo2e zYMnXK`N@Li_t$if&sH4x>-W{2A#449T375EI&lH}>pJ3c-ogLeLasLEQYjWc@UQ=; zBi3f@V87)UyNn>%Kfbgv(Z^TmOpzr2^(0jk&K0|$MgH(XQ-8%y!G@h!%0LOMTX-#jbI|GHZEHkPi;r$Uwe5t`EW6?kufr|0B@AiJbGI{w zHC*aDDvRsNqbiLQ#s_;Y({F}Q&$&OGWxT;;T#N@^*i|^wrrW! z68TZ2zT(A*!-x2rF1vDwn;&eq(kbF~w5xrXU=Q*%Eg_e=TaJCsYKvgPQ~2%p_q6zoy= z*BjzmfvN@Ydu3G?Z}SL`TZy*IMa)NcMrSNjvNK92?XV)g`6fX*~XCtr(CVhu%AV22Re#~+N`e?@j_tLRBJ@DtyOvKDD@U|ebu+IdwJ-N zK4D3}O0AUoEA_Hi2KZ;W5x)MWSF4Zt^F)yK@^Z=5%Jxk=0pOGn1cHO}qlSkc4^=+r z*@ee=khQ*>$~5J9FLzfrUn?oh?AbM zwnPE))VT88ocH8{rw6}knXB4c$BLty78|BI#y@9HiHWnz9XSTD6Wxn;gyDJ|Zt6VV3({dHJviMo#1ArG*;m=r4Iapc%U53%e2hAx1iH?c2SB zbt(`$<9-vR(GrdM(*kz$7^5p5#t9r$OE1y`8k^=(iGCGs$Ox!dmK2(HNP22c+MCo= zabCzH|K}+?10CA5NLByZ2#@J)p;egKaJyAQ%#+0Lv+gDQ*`vJpZkQ~0osgS4{1-|~ zd$`AHhP(>C)3Bb)l<1Z6H>dbTmVtN+443cKqk5>0bB>EWA7fFDqDCunY!h9Sq-}z3 zS^pP*P}yC=D7<$3OJ^*oc*K`$>{h>Wu=(Kpe1{>crh%ng+LMu~;!-#aA89crhdHZ@ zg-PUzEPG3mxC|Z>V{hfLUlfBJ=_OgibqCY49_(6Jmur_fzN#L64QdW$AV0 zEdX)CywtmTKQ%SoSyrKqAC8yQdg-cP0!m>zloJZGpL46Qn-j^Q&rCkGJax=Ix8ko+ zH_8}7b-roh^n9oHyD$PV37ir9fL}dnmMm6`H@;{Fa*fZc(Bn7L7jyC_4LzxWjhsvp z-^J@guU(+f@v@u+`_)UGrn~&=-OqJjxsi)@xmNAjH$^jS=LTFWPx(-_H)Ib1mPg|g z?vd<2TN8fNNN-y)ErpfC+CN~nN&=bMHv83UXSigI&nGWm%<)No0h3?KHTS!f?d<`l zZD-7u#OZDj)f+@(%9S6-bB?gd2-)C$*9aUII_KZK zOP6;PdG_$A`$V6GkuS_XcD~4GV-wN_R#KXWxsgIYNjEOPEmDIhvsPtbzJ6c*YBSiA z&~tN^Kz!yHk;NmBUp~m7`kp{Mu%034cB8uE4&Sia^h$8jRKb+M_UX?Ps{9e!OTUog zWz}sWj;pA=9@NF+4?=obk6(>eOC1Yz7m9uqiAwJ_>kpPrpRQJcSNvSsK8;oY=t-(G z^sLrWQ0vOgNekVcqAdY7mtc^rMb9tJ$gj_X`OlqP;(ioWWcx-Ja^leK60Szr%YTok zue6BIn=D~x7k_hZKz{Bd56Wlyn4)g^Lmu}V)%xlMKQs)^cnN~@zH0}0o-3>8TPLdc zafQ4#W#>{PiOJ@@`^yioB%jbVbrgcNC-y>K`-h7A3mMk5gH5@iHqGV9Z@&2|4h~IF z?y5`SEq}2+QEC=Ak*0PMTvlbJ!*{ONEvee!@*Z%y_kkmJDev6~mms@vx}yhcbUnpH z>Y+_a5uYI;yT*hm6W+vMGb9xX@t^m4XmW$g@J61>Nme@t%lk7KU(EVvj`Bw?HQcH# z;&-F}`VsV~O0;F|H7G6DP5_kRMLJ1{C$j< z@0_?BTyb)I`ToZ;(H@~=R)e~{jgmXm*L7(hYAX8uCb^dcqL>gyvGc5TD~N^f>;}T} zHMxe3GwQL07NA?YZEWB&xN0!pP-CtNGHd!c{8EMZ_N-qHv*UsI>MNfV%s{D$tpcZ* z``Ys({hvp^5K`8UT`y_l)zaT7?@FZaPWReT8$Uk(4%(C8_aS4;g4FFyYn90E&sX6< zrB7eBC8=)@DBqmK&W+xMrY%Qy%skBK43oj^P`C}-bjpJqn=3OVx`6%BAS7vaJy9D~ zD8g|9E+>D1ec6u>Jrk}%2)Pc`kJSP>l!El`dHUSJf5JGBql_w|cSPOz>3y=R3P=mn z0r+b6Fp2Uu;u5>LU#Y@bhL82n2pIH_md zX{#w;JNa0>8h{HqmSWB*F^~Jz(kx)MB}-&t-^@H5S^`dCf*dv|QJZ&HmgC+Njskmn z%Srpz?e>Y${NnLy{)$dJtIShXbtxeJ5$l4Y_G?c}m>d&?4y8>_39m%vM-J$VTT(?2 z)}9UqwTxU4t^I0M9{QdEXQZDI1UTnjc@__lntpA`oY~c@){gslcYY1x=87UE8(Qp% zFUR{etImV(IGu$_z8OEL`P!l`wWX?zm{7CWc5xwY&ZwmuxhI%%?{Q}Jsf<#;mBttp z*dH*lY0Bj{Vt~gK&@v1&#>xfG9rbXGI;G?}dEgdnr^Ftyw9dEW@8gg+4zmHRuPZk1I#Clcny(uVpq+_XGMqmq# zdpWweE1K@8?`Z|VqjYq4hW=yblC7F#xY z0IQJxz6_dE+DEd|sPuQ=vA6~2(1@@BYUjr< zErNTV(UocBvjV{V>PO=|)uyL~!K3#b0lN~Zm9cyZeaqLQZaYUpgHoK7Pbpz5Gf$Nm zn|QFEPYKy<1w-Y)8YKjwz$CD0UDrD7bf=3tySVAR+=l#(GC%)IdF&F>n?o%m#-!{0 zW92sun~#jLuUA=7I_gf^CFBj2g62cPFhibC6F*b0l9fTo2ZuiQ zmK+OceJTh$x{x;yySTc1X#1gXnvdr6dbiv9GGA5OFhck7ykDYqhsaRIx|caz7fs^H z@D3@+B3KX~Y*J&*68WJO7V@}eu9~%kQuAOA87Gwi37XLwgx!kbj_{Sj+Pjwk(GPynE^@D$UCB; z&<43AU)D7=(pbn!@R`X_dDRo?>yCz(^WND=JV^0h_sc2F-;^_55MH?@m6y9&fic41 zz8ComvF6=3i*-wIgki~|6~vt(t%fHBn|EsXK=-^D!)+ET`pWR77p&DC_scDRgvFiS z@f27^AH*bT()Z+LQSLb}d8DpBK4mAUdKs`mp3{B(B33 z@?ITcKM%&|3{5y7GD!XvI~~4d7VDQ_NyeB7ly;IaW$%sm3{Pi${543k-G5bz;dZ%D z0Stdv*l8@NIqLNyanvv;vFXr>JQg7vuMRiqnPuI`1{w&zdv9 zqp&$P3B)1uX)RObrd)Rk&r{5cqa|Kgg6KrOg-kho(L zmAvCOT-mFH+-eMR_3R%or!GPOq^tUx_7dLLdvy89SbsW`T;KJq*=1-LnVq0+US|-O zGdJs@mktU?Xw4BUrlfDJ`oVh>ZpjzR;2dM6My*O5a3=I+TRwsatJ>|_0e;ilj5wcL zKjYR5gwR%m#M<=K2ln0#1ZP*1m;&Um-D&`wjT=_Kk>}I2v>OB`hTik#8#rXU<+V%;fsXr5Qe(g?&XgwN-N>IfWKuHTU2$bPy&-*OY_#6jNV8f$bJ46z*RTr0#lG>fM8=ZAEX zLi~B-@dnQZ494&ll*N<|Y8^*X>M2X^5Qw}RE~q8A163jTQ+b0tz#QUSsIVYHd0L-0 z^fv>mroR;kp=l+p7;{_n?d>J5{mFH$~oA%mO~xd-~YR$nV%Y zo;M+bzg8Q*H(u0Ux7_|^y#315sbMuOUpuzD_`*f61Jc=Kl&%^>(i29WSF6keVAYn7 zNl?SwjqfV*9B;!|`B5*6`c=NkvJc1mLj+ZiTDGC-jx^nGIzhWDr%L3Q zd2L8TCVJT-2c|V4H6n-MzRK{thxHARKgy`Z={14PS%R5ajRnt?R64snD?;f@R2xWe zpY(Qm{%txGw9d}Pt%!rzlxwsgPfW1#4STlHSPYFc2xNt(Ej;f8w8VJe!HK}B4l8>W zb}*53x>E*m8;zb$l+E+506Q`&gbJTiJACRb#YyAjX~lpvGj);bG87<|XxN+_x0e*|2CgZl*C_`?**M zH#6dlI4G-}oY!H+;_zwmC^-R>F@%8dkiE_8u)|{%f3mq(`W5|FYI1~?A9ibTZP{bO zBiEqfnJCFn4s&2az?KKC{c=ILUSrdt{wBtZttDc7J^b+Wn5%p1t6A^8rR^6w$KW5# z5fNGTtkfvh%h64t#)}p1kMM?I)v~3tFZjF;)ELL+F&heU{^VU?MQM&46z=qX)OVWW z+l)Z%x!n9CJuly2q#+P<*OGwq3>$7OeZwvXOlFA988uDzOikXuLz1v+(Icz6G z(_UfM4J+?g2dAbUIUTt?ajPjLyA5?+LD+8zXZ*2$&WklpjG^5qEOZWQOiO6mN`2ttm2p@+zU&M(&u}mC3!E<_QYtq7z-oOAhbjE00 z%cElHS=)t6H#LME7<+npnFa%h8jau^qmB`5A!XPP27*G{+6^MQ|4anh7-%AiFr9L-?xI6xuUa%dHW z_4K~Wiw4ws%uQisWL?M7v&_Ex^4xWd`dgCbk#4ezfLF#w%fkcn%^Qgt3}-Z&xEKV1R~;nvdEZ{RY^u^X-OlGFRm+m{X+i1G4Kx{RYC0re zKEU7n8U}L3&Lm_D~6;m$&Z7M6RN^HWNK{&a*kNyG7mveoa!B$kzN{m zUU?x6gi{RVSK3me>DsGw%CC=wov{r9jjzBbzEENG0XDJ$=rP5|qxbcN-c8s2I42i< zZG962HiO5+!Fwx`)r>eGXR16?>%~C2&jgEt*NHiKg=#3P`5{rZ{L_tuH)BjfzV7h} z&9eR#Myb)f`ezZ)+np=r0SU;+n|1dDH?9UBPt}3dR;G&Oy)#WmUBxK6pkCN$vy5m9 zmp0d&T*lZsDHRDyGo2So3$n1`ynSVh|5=5Yoe-$OfuqR{GTQRF;y@|m8Asl3n_!f3 z&xN)=bjKjNJHg@U^K(ad2cq4m#W>t2x$wgkRfhSgao3w3X1?$$egJ*)V_1aZHJuvV zVaAWXa{$Pe+>6nPPUQA;EgBw(q2LPs=)+>_>6ZRwBYhHe7%ZkG4bVtQhbhI;--VGL_P|DRH zU=9Jp$z4Y4q2df%LgNfsaVt%4LJbGm-qHsgdO#VkG0d3Gl0ar(?=QYudo5`-g1RA^ zJa3vJB5$$J&ceh_K|H_zPI3k%d5Ydd`lHl>J(t~i>Ehz_*Z zUS2+b^Wbpp!i^RyD~d@NQ?KBNc;ix*67U*oag&|V(1GmvtL}t;aJj{a)Vi#u{K6Cg zQA@3m(O+NUaYYj*=e$(rrx#{89fi7|*!s?wM{J@w_g?UR1>T12%)`D!gk*g&s~+X3 zdcx)D5v^-6avJ%@_?*2^+OokFbO3!qJs@)5LThqEp9d#+F+o_YJz0b3kE@5i1SM&+ zqGY)f2nHXxcx}fQ9}lsxxev};XlJdTtGZ1kn)Mft)x;;euXJ8{Q2n()WO{vf)y-a@ zaTxhP-Ti*b#OL)QwdHqt9_Z>f#|KjDpwrIg8if&f2%C~&_h|KPPV3jd)*&?j0_GZX zoutvcVgp6(=fGEx7>0u%A2wS7%J%4H286|mW2F54)r^Dr8%wj=Xr&sPHCOv#OGD#F zR}MAJc~3Z*ds`;=_svUb2`TKp!fCDl+%Uji7oQ?}O|4=o1PWHF_Y%>b$RSy$<|Wj9 zGo)^m7DH#!uLJ`^b}CRc$3cC@?QP!RTza$WQ76?J*kO#kJ9VM7?M=Sgj-L)BS*AJ<2h)=h zP}1x@Bs91je>$(?16t)`)BQ4L$0jz!sFCin*9o$;^NrFch@zz={~OH;#r0|xvY7N# z!RRISpXrVc=NOWnaSOT92=y32?x)NXArD7LjP=TSxbybsyZt2x^!x+;tXQbBsn=;z zrToz+HoKdl%0~t6GTTS-87rYWMV{qArMq zu>-^g2i}UGDFS`fTLM;_s-3ZOD@C2LV*dJNE^j?*YG+q&Cmzm6xYnoy+Amq?&JR5= zP2WR*_TRu7X4t*h)8N30r=lA~>vFV5xwwRY`sn8J_)7&Y2$9)7_HZN}{X+>a)S4)| zMVVUeEL?eIyy1Zu@a-=7(iH%&vYh^IcRMd0_g96M3wF~_AOp=Pl2)yBwu1l~b&nB{ zf7kPT#6E|qJQmmofE{=RDcs|2I=%l-FRpU*F4d+u6It|HWA#i7B99=}7|VqoeaU36@N^Bc!a=;rvjdwtd-s3u?K{BTkuXV%EfljzvbwGEFB+kWBh%4Zz;zmK`=U;f}++S=@URzuns=Go) z5B$3VLBzImc#vFq@o5sllqPhRSHD_O3<5=0?`#;_{WxC8b8c;oX`ho;xAP^o#?_?S8qeZIJ9~T|6$@q4-CVlDEH*?`mr8 z3l$`NopU5BxWA6GPfs)MLF(H@zrj8XR60I_<}sB94S7%7&bk{u4xy3#J(g&`<|)|T z79L!jFAZ8;VE5aW<3}3#f*H8wWI{~IX8pAv>&M+BpUxZlJDQ++VVal7dajXz#-a5A z*hddy!fN>lHg!n9;vwCI!FaP^rU(<`l-RmRoBUjf0LLm6_u%=>H?{Q_e8YLxY(c`K zGn;MH{wDLUK(Y<3(@W`htUKCT^wXCqP7Xn|eFYe%nkSp)n9PRht4JC|NSV}=d!HSm zw+Q#;PlC=c6UIM@AZ}cd}A%~mp+-~^A68VwTj@l5IL!VOM)H5_FVS(v{rX(XsmW~myNFnu4x4pyYeMZLd) zxIC(7O@qdELK&repzOB9NdaoTV)%w)_p6A6cIHt{7FY^qZ^Lmxiu%5gz`=-4CBImAoR`aZu;BUCqFg*h>4SiTazJ2 zG>*eoW@i%baC#Y^^~4`8#=mJzVLft$lGe+9Nys`nzptoyeh`Z%X8P)9`Fw*exfRBd z`ZGJufbk~|es-M;1{KyPEONcp#9<}|J?gD9lbGPkP#RBUQ^>0c%76ho*k*Kr7lL%= z)FR`Ic#|-lAN=))|KN@w2DYg0Cx)^P^zy*`!>v6_wZQkQ8JB(P`A2zk5IjlnsaFTg zXj`Qu(}!1*Ha_X)ASi3w7kVKNO(ki1-RHNKqOnqpy_Bk=^~ih?`L+7oncT@UjYWJz z6wkTxRf(wTG>oa7NMUbNGL1HNRfr}zYgNrnjc!fxGebPX<^3{1g{8^-`65wFbMTGyYe(WhWR1v4~J8ZYCBlLp!a>giu&ysuG=DrO4!Luc3k9UE0 zVq4Fii|xHG0d1@PcR;udNG4Bx4iSu^Qpm^`2M%fTWsJEIZ}#AcLf_gQ-+HTW;I=nz zI=6R@96b4(zXt!gsqz29OH*vT;|x9qSy*|mpmweO-P-pAgqSVTbnJ_Ez3?_BZ{h6k zhAV@M+AkW{5MpqCI3Jd|pHM^ImQMW{4X=(XG+X1A-;aG2*v(bW>7ndmxtDBIrebUm zZe|{ z#u>>|DAXZ6LaX$9-ybe!|BgzTV_ut_a`L!5e@cFgXcb8y7?3l}*)Zi{|DilXfbyJ& zEaEK6oY1_NicS6d%iykzoqnJ&j-jjc(M>4m1EwT1z0B(TjmF%;Y2~2n5ka8f3OU;) zRy@%LK1G9UVQmXH0&?YBYt^BbXijq)s6GkWB}hM!;&z*pmqSjn9tJd?zzVilNVW4t zOZ|_y04D8kg5=^%ymK;)1n|!KCdp_m!7z7(dBn!D?8BfWQlo?NRI~&A147_alT2Ez zFp?QQmNWPjNt1pWgwIOrEr{!%aAn=fFK#3iiseb2PIz)M?8?mgdPTmPbwtIKs6ok- zm)n@`Wr-0X8s#btzl}NCdkf0FG?u_M-9LLU9zh^vk`!AaF%s!5U%ur1+y6lN9dT>` zOI{+Ie}AZs`1&e87`rlXcp5LvwgNSYZ7SAFS?W=&-r%~>6hwQx8PXa#$Q20zxVKBC=G4K`IaD+S>jU z7^A_A0VKGFYjDabP?HjZ#ZuqZ;&a0YM@7d{EjkY_$th$ylA~LFeCk%~o{ymy-SpPS z=$pT2m@fQ@Ky$j7sACa9g>PkQ=35umvgcq-aQ-HrmwjeK+bhBdIoH5Nbv%k2%<1ot zf`M!adzmB6B0a!|@uf8Cq$v`L=oij7qDZ3*5wH=ym)9G;KhtUniepBE`wva8>R`5` zTVEa_;?B#T&4DUVCoG{`hGWzYdblg#Z=#3w40>SB&gKs=vx(cskA4?wm6NSc&@rfDoox!FozQ+ARj)^MC~3_dOE^l zqR*l+_=6BhxuK2!A-_F)_?hA`Gn5x%OP^yJIQolH2I@-^%39E=duSWJ-Gt;OS_u2* z*5oYQ4mL5pNOSv%V57~)MW@hbMtVYZD)RsB?aPNgmcCqz7=vm-Mp>8YwWHIuUxVf@ z*f%VYYP0Rl`uV(CHb&PQzkW>F#5j80rJ}<_b&2X#L_DhFhqAdzI=Rmn8Y>}@u$C|X z)|d22Lu`~bp)6gCrmHs~s}mvWW3dWr$_cCtz&2ZNNVr73M3_NX--bW_Q{K(qo+o_k zl;bnNq9pN(uqSt#kSXou9RUuB1$0( z2^FbzUi{@s`!csoBMxWGt#;ndS=LKjZIMezNnzuamF>kHeM?wZLpc5kH5pH*?BTupYwakaQ@Jil2Oa@$S03ghkF7 zDs)f)M(YXxxU(gE=kr4C?*3}V&c*+QeL=C$s*?wG1QEYZRIRn&NZzP<9)R_jWHe;s zGo$6}F-FvQr`j3s?u*`O{^J_(Qu#jW_8-pXJHI#a zyq>1?-gy|UM}8Sq#vrCZ6TslCjIDMm+xdJf1}dC-ccZ{*EbQi?gA|@Z!%RoRf8V~} z^bsk97@1qkze(}aJ8VHe;IQzYJgO@WB;ZgE$sR9ASbgiP9;~OlYp-fxN0tTfW~BAa zDIOX26M=;k5|CZDOew?Gw5gR&J?JEvwe5V`dtKry-?JS$TZADg1dR;gI$oVltM#AQ z(z1_493Jh3lCcl%Nr3kMqgIPJW7v1KA)#Pwf0c2Do!+x$xbuCvNQdy@AO{n4;bKxz zN5>F%29;Veh4DE@KwN^}bn%#4t?wyXDr&9IqAaImbrwzIMrSC`x#6Ns%Pp-&A!W-QzpGy+oGmoW4~>;?Z#e9iPIY_G2fGqPlXrgDlvg&5 zZ`8vInOuWW$>{UUSA(=l@COg@|axDsb{`i7tXN(@!{QWCuJf(_`G7nqD4 z9NY5$T1K2OqizJr{3>f6Mzm{qxcSOyl5Ed_V!5Dzs&kw-cE2w$aCCO$iImXENoA;sIZr(X(XKYd}>|zZRLMON0C`m zvHrwit)yHj^_i~Rtc@D|8li~6aKI*R1(-V9Xf|6!-kZ=8?99ZiY2;bpY)<0f4YA1K z%Pw^`hrKtWFIrZz3@?-s8*V3<)YMkgx(7z`iRF>fYS(dAD*ax$0o&fMF;Ln96p&h5 zE6AxgY}5eabQvpnDU$GvQWEzkgzP6GgxF*Px9My#g~3~R(C||}b^S6DXD=jNM*c^w zzh}r?8mjgL$z~Q$;={R7i*n<*FJ!cvdI7t>!>`s5PBiA4df^332-vj*FASoO0wLlp zGO5-l!5O5MmT=fG=Fj+uxTsL9f02C4)BEy}U(tqfy|Ok`d1LMo*<8`~17O^t4TMkx zb$D-<7rEL2c!O&-6PR(X1FtPAyzXla$@Ki_S-(L{3j)z!8hu42ms4-Xdy{bC$loa?=gF1jdi7Vu zf%8I4bGCmn=;CH_?ZZ|JlQ+6YdQq}X)&FLz4ucujs1U(w6%qj_bm$0!%JU5BT2aAT zdy(Vap|YU`L{(+T$a6>Oh3fyDWs42}|JY{KcC!{*Z^aaxH$~7?n#eC%vaZ-Ou^pnL zY2DkPAW?nYk2815(aL=}ZrL9OQ6J5m$-2YB&vJ zi#F)Gh4a!R+|42GG2>8~Jfz6CVqI}evMj|A-BmRq*vO|mclb+h*PMVUg~Ya&QJ8$G z#UG1SW~4mr4haB`v;F#}BbC_*hsvzpTKH1)_r_6t&df#>RE}sbNwv#*93Fj*0l~Wi z?D5(4&R%Kcbt^(lNj1|-*B6JGF{d+Tu>n-#6t4X^X6}ONwyCn73d@?1U>R!b^8vcA zhcQxVH6OeOhy%|&bw0RjY7xDY!jzX*5q(>-yFFH}=tY~3C)=UB8YY0_J}uk5gXnfW z#PcBhXJmI{P2~ospx^nht7NF(mi&Bw=@-$(=cY171O~kY zeWGY#WO}_Du-1*=uh8Aexb}S=z9C;*@4hxP@az(MIOfc5OBtx5&u+U#)|N!8+u}Uq zrFI%(nE(vf&Nj1I1~l~CGK(JT8AiL(g4=y#@PafwmyE4R7Aj7OZsG(;p_`tsJVZeG z{LZlW6v^vH48!NX->yO5Sy(;!plPqro$SA7aMmU}PY0Kw1A4lPaTH|ph8n@z|~d4e?R{RPkW&zGm9vBO-O{?}(TqOR-k zw%L^XGmY;J{zCf9n{pTD$^d{Zs+)cS4cJ`Bz#hj)8P;!UITUZAW`+~<*VH=0^C<0a zL(~Da_f$HQH7%lOzJsZ5s(g+XUS?`N@-3-u!k+SZV+-MfUKbG?_8VSb8!pIS`(~c6 z6?NE9E@JTP12sZL0_a8!p}$?Lk(A+r&Cw&MDEI{|7SQgvBdT>Xy! zRNPpZGI-3zBp2OhAzkaJH5w!7YqlJbHD({7cb1~kg`b~kV=0+wk8#<+%NP*GloHM} z_$+U=U}H&~Yv1BqDO+OJ7pIouaCh;lmT&a4{|8mET%jg=;w4E zZOKiVPJfQ3>lx2|hNr>u|^qAn(;1tEt^H+pVdY1DG*BEsY1J=a50W4tn&CyXO0A5N+YS zRpz~%Y-!0`O|S$`Mvq4M{|j)qdgi6yyr!QjBcigq35Ea;iq;%jT@ ziT(`dd@sM@L+3NhkkKNoNhaDdc|8{v8ua9Z5dw|g2h94Ph`z7rymi)L|D5sPK&IJo zM0awt` zH-?j`aB{vM1~EqT4Nw%G3d$R}D(eg#`1p;Bh6=rP|y zsTfQmd9kwcFS_i%0lK%61p@540KAloQ%SL<-N>i@u zjck>b7=gSKsfr-l-5(EHh!;lJ9;X_JAt7Iu)L(-Fa1oMYxFeIi@+%OUl|RIjc5?st zhV@^Kd^;O)?|Ym>e}|nuvn9q)+3P4{K&Hr>+ zj zA9Ks&#}?62)Nlj7!n&tn=t4oUmg&qu5Nj8DzO1PJ$NA|ODi9DBKj?&4mD)8mWUnRA!Bj^#E|+Z@He2ZOwC+62qo9K*?3cZa#^ zi^0KTkHz!^Yh9xlPB@;)u2|}}(ym>zCLyx4q=T)1^F2TUL%Z!z^CK1g4Ey}tPPYJl=sK6eK>ro zI8gg+n!{k_XS6eFB)IMg-`plN$$?WSXRAof5YW9r6&3B(0@72lZiW^f=fK9K%35vi zv0ql6kzGzo;ad+smF^FQ>2i96;@jXV@-u@|YD^IQGg<2}eW+6^} zV+%U`0*4GF@#kg+ftqawWv?%|`(0opzXCt3DQ`ccgHP4nKi`8@)W^yJQ_xw>XEi^c zx@-?>esPB!pJ{%qMQrX{RH^tVWtASONsK%#f`A1s8AvDXNjWF1eQ)o?FiN4?Od}2^=K2D6yZD z@9kOZ&9X3%>sTf}sIEox+F6WNzgNna!Qjg`dGYMEB0gxu#&Jxnpp({k;6*+CEgGAcyq%vXXlZ zH6?4ZhzECQ;;UcA)))2vwchI9pMsSf0)TKS!G2f{m|Dnk7g=}(G(ZPLNmcM(!hdmj zui`0dM+1B~&AFNo_*@&^wI(!De3b6t#jeE>l(&Y^5i`)-iqEcKeWI}M1g2*-D?N`n zX=OTfG=Dj4jcjf3$JD-Zbtki3o3)z(=`^4y*JfV?wF2(|cFB};+WIqV%&=SSLrqC| zFd?uZa7T_v?q@P~@KD%)#26iMmYBBmfde=u4k(`^4~2@hkse)=?Xw~-X2C?cO{N99QvH>7dR+{+88@saYPVatYM!<=%k83}}=Dv?_?H&}lvwkVh893CUPzfv$ z-Ld%HcMjTaxgAlnL#xMje#zovam=q4!TpnGogsTR__B0==hp}JPpVY{AlR|Y?$2Pv zVmrDhc`X}H_xE7@sNGAoCk25pf9Pco+;Q4Ix|4)Idi}F|#UDlr;P`lx&;jRd`PG=P zf#scNUZlm{n0fbScN{YUL73Z1Mv`?3kh!1FiavOBAFFz@LP*E`=A}09ayI@V;k{2>^?%+5GznLIEm_&Rlv2Oa^{PN6yMx z$GFz4XurC?GnzZmb5$8B&*rN#Q(Vdy`dYj-AM=q?{vq<*F9QNz5Df^T0xlZD&(cE@ zfjNfzc9%vLkomMz1Kr1%jSo5~*|PrFl#t=BM@ed!7I#giovj`_StAUP%_FY+!Dl2x zg;pRH8wA({)stm*OaJ(=!J&&%#DC~>FBtw?$`36mIdWGYd(!dHF#rbFzlR$q2RRI5 zdva#Jj|D)a30yEKv%O}o%vNQmM9){{EPW{Z;yV1h&*jO-fXNB`Z+Uk&_Hvszg_$vh zI-kIGBfgzdcdBM3b5fG#C3BZX%5J(=|1MOz1IOYILshAOzZobE_DSyQy|l**lH>1w z9^ac(Vn5GhXV{u$^p+&=D{{`}a-JQh4FM>2n%F8N92!LP|M9zrw6^fm-P3jM8Ruy}NQ}3d&uE@6DCk z&rCTT-H8Xr3&5l#V}CpqG3(^_5ss`TFsV~Jh3LLs`JXbbzsuN512>)Yb|cij*YWt0 z6h3k+Kq*6Ww+Nk6WJ=AvWZu$9`AI3p-v)oY6AT7B;h$v&jDA*Z<;2{umFIlR?vIzy z3HG??$p)7tqtD*UGW)LsDww3raeDXW?vDTA%?%DP%EDo$k~q8d+seS4;eX(Sm7;n8 zI0vBE(--c)+cBfenIdmNJL#5A<(7(lD^G~Oprg};0V%HE=4a;Uhx+HkRXYLI^Q9%? z-{s&lrR6T8My|fUM}*eg!5#=uxWFh&lcLAO@8a;B)bYJgKRsJz|IG1l_T+F^+XQf) zId~^!zi8L~{kTpGR{*v)L@C{Wad(~M;qeV4Fv`YZrn+jj^1qMhHgzGKT;La!^?y>t zRCvqB*|G3b4mWw%2)YMl&@~Z%sRCOeY{3_bt80mA`=| z!Mk@5>rTd{Jlo{|1-)UD^812U`xLlLb4SttGaRn9tRgO(^~VHHeX&Z7l+@v;Xg9ZE zCz}nbcWwFO_G?f+umbQ0xFxcpUV{jZF@!LV7*<@_2K@*T9pJ1VOiZZT1L61mzC1<0 z1&FxjH&|V4bBb(#j9^-)I}wqRJWG__c)E7h476|`ppt(hd*|U@h`!L~`R^lxsy5{+ zYPE;d$iteH}WYR(wnx>vo)mO}cpr&2O5>|y2}638cf>c1IQlhcP`^B{aB zSZ>#=v9JC&$jI`T6{tI668UF!|4t2IlQaoU=|S>y%r4382l@X7DfvDa^$c`iT;_jB zXWxB4-7f*DyWlr-j%V_Q({yo5 zfFN?_qM_a96&3&+06A6h;(v=HAiohQN-Bib-~^*Sp@QSpZeYMzdSNAylQHwzq|Ce`@3Ub75{4p#6`%Nr%dHS_`(yG|nFS(E14kpjD&SlIG>TOZwT__S5Cn>HqlHAt5 z;$i+_?Bo@x9OEx4%~#2z#887mR4ZrOGF@WcZ$X0wtR*-@IDZjvX4Foy(cEP-x@)}u zF+*dwLu6k);pO7c{84b3Awex5N5bBe*QKDC5z~>GAH#`Tip}Wdr*OC-olCmh3t|kT zDAWB3ZU(%}mun7L7w#1#4qw40aLG{_dYX>nD>xx-=8f&-wP2&ZwL?xt&0$R^iJu;MG1 zL2f%tj>8Ph&!nSuobTCG_{2m}{}-Rt8^c@$fyK_2*YsL9-s*^)BfhJ#S7Q0>S&dk< zn;3YpVDs_4&0u7F(oZakXHCUv;zZlz1tSw^6uQ~x+PD`yMIb&qM@m0L$6C~>dAo0q zz=4kF-CmT@K8vh3*%(OLBe9e@ct2coujwVI8qJAL&TS6;ytk>yc)K-HkRQHCnf_|P zE#B6EEX(nIvI7ePVV_=GJIZs9YAPR${@mxzO(8It99RWr#vr{N*7Hj#qBNQ48S`bW zZuV_R1Rw>}JaRLH?x|;asFBTPq24_bfvf($0y1GcP#pKk#84zN4R*w`p`UyHyARvHRXZN51EChiL}DCViWa7K0~V(e@6!pa{dP60BDG zE=Y{UxtxZ!onT3kZBM>4Kd?{)EfW~MB*HoTNeB%t2+{TcSULa~WFR*;^k+*3)cy8i z0Q=gLpPzTw>Yuk2JBXLdbuFR6a4d4u-#j(>{c=z`PqS%oZ=wpz{SL%u#c4PSr zoLp&#U5)}bPxSA`K84;Demxxmm+vG9>$zZfjla6^4S%;86a{4=xW+&NVaur(<5{|S z@RCy2ev6YZ8JRHfgcP(abE3Y@GYHhCLaLJff#cWalc7ACLy4KZCMFo0nv&Atd2(yj zvhw$)m%nYihCz*f)YIK&r=Ilw=b@WAj`Cn5dJ0UF1YK1-_m}C*KajyG6AG>dS>~-` zd(JjSwl4IhI;Jw-uY53Dkk$9oW2*(oK(`A2SM@8Mm$p$~+6y=9e@?oIc!yVVc&BTjgvNCsC%%i`xau=&(m6W>p>{l>$c~$LiS=o)sN){oY zKYr6k29d*^$2gEI>8fPx*XY@>Ym~_ywGC%qO&1;>Pj01q%Uh)ZTg}b+tdcWR3dD^7 ziOv+R-;C+O?du?S+jIFN*r(Bej*9_dXZ&FZ`5$sU6>7wC+HWF1M#G!G`IJ*spSpOR zcZ%~MUR~(?ej$s&83@Bn1j3}s#pTYuMw?tt{?YeB`G43L6@yl@fJ{uh#!X=$*My`kU`ZNZHbmG=u41FiGwwFvOS*vP<+;R~b_5`Rr ztX)zPYT(1L^k3ro+&;FjQeWOE$!8R)8~A=cu{6#YToG1rF1WFb;aGs2GF!3YXLnK|1=#gXHM4VG5>wPsA-MEUK7;13pELPZ~{E+ z{rMcQ&D8(+|7D5(dlKvKL;E4x?N*o zyss2UQ1p(q?YpM=q*9Y2n|y8pfw}Zee^6hOj#?ixz+A&YYkL&&10H%f`8<)srJ}B< za3nQCoKx5fDu-|ov(#@2Q;2dn)In#~d)fG8-{m}fxcr!g9`viOhWt5o)J?<+DRHhWbfO8|&Z5|#kfR0(~Y1Z98{G_hq zPuHnO5G^QqD9E=r<_PW$ExMr-yN9`N>6i{=w5V?ewDQdIn2Xz>d_ww} z=0QX&<6D%EhfgXY!$`?h*{2S+1<0j67ikcP$JCp+93Njn-rDDNU%VE20lm4J7g<1ZYa2Q6%Vkk@8RbKqlr&TVn5U=&)b-|%#?1{-s*8XfmAAkx^N zmoCoWpM4QWa~QM45>3Et1@_}tdkTQ&YK7W0E%`+MO{KM0jm4zHP}*d^rXaOtv>Rf6 z^gsgo2KM2M`c?sZkWwZ8Nl$r^QDmEELZw^jm}iYOhQs*CCnf5hB)XILUGRacl`-9A zsX%TJ_yE~CZ{_~vf4>aS`UllvIq2lzFz^mb%WRUFYQSF>3J$=l_dOR*odgPE6)M!% zaru$=HxkB?IKg4vgIT#nR2B7kfN=6I9zcqkmcwSbpueq)^n`ULB5v_oJruRL#5SgS zdnYAn88}2&hy%q1+;B-gq)o;BySIYr0U&|SEh4pLu?D`xgZQNu!1}9QSrpGrmD3-E zieiG^(mj?3`flnfWyi=(Jhi60BLNvM>St(P9)6euLdgFe&QNoju_Ew396uoOZ9CI^ zv}a8gYMkX=815X1r1ab)KmY_KsD3{}evhD9O*J3{wgW?Lk&Q2#eO&R6_#WoVrRTPa z*|I5snE?fD9h}01f2FNFekJn%wt?L2#wml9jsDx#x`@3MnudNhvfjh@BWZcAUq3iJ zd)GAQd(^V6^6EJ4PSFnC&kKHK-nxL(<3DNrYzv#XcecB|2M-arJb1@|)C8^iD(?j} z1@Doj?$=?V>o$9{@Gy{*ehql;=N-1RLJIBR?Hm8(6G;?cH>C*O;ozeI_a z&C@!oVtj?&nR1kp(-jjuD3uK`2_fWR{59vCvCUzn1O?IYMg#eu=VM$05M0Vxlk?a5 zzK#}OlZ9VvV10;*B2?eyD(W&FU<%Dw6WO z%1H^U|(}r3dJIR%2ec~z6OYU3k zxnFy!})%%A4;IzI(qk&X}#HRb;S13MO2^cwQzi5 z8yGUq=rf;~u40Ku(gSZA2J@D$qLgxs^7&mu3m>QbMiixK6$;=Vsnwrh>GaIY}SJJX}9l6gcH0qx5dj8IgRC!t|q8aZ#EP+ z#t1dz%?mDHr?k zsQN{aKJp69!uuXV3UGx)w~JNRsbdC8l?Bf+ z=qnJ#^}hx@{P7OX0?ISple|%Kse*%$U7q6w>uc2Jy4Qh5mq3d~R$zE#oI)|nftpsP z)&iBSiL+rdtxXGiZ1v<9()IO%pDIq7DbZ|F<2nov4RuK1&pYfwkqwG zwzRhsnv35Qq3)BzE6tBC+6-Q(p$JLsg-&hxYBz3`8#-qgn7Lc|ZLP8E5LfCHXSZum z=9=T;n0jb?UG~6Ka<>U%D#^^QsfRMGF&Wd?=$FQvRvS)-*_5|HK=IS4yFsovxu8LQV3~2w|vC27R%3f$! zRoM#DPw4QKT3r>T3*7mcX#p#ow-6C-NX__r)}bx)3MZ-}cC93fyLRq0TEANe4y?H0 zm9(&TsrSUp@?_%d*jDv=U+qZhviYvAZ_2*QO@CYucptg#-%=Y*!hYaLq%^@NSq2we zGC7iLy0+M|QxP&9aQw}(-R+S;s-V1B=2!?=-_Gsyn=_k~pHm(3a>a?q?A~*(>miYc zmzsRVXX1z)+-ry?zx~H`g5h^W9O}pYOn_c9#zEAvo z1j!B5L-TsoRplZ=E4*M>`Uu@^hC|~Hq1)tRfs6pNMz~&%| z-j-qm9Glf@vh82N^AfJ4eC$W%d;-p*%}?+nJ7*q`YK{DyumUNFjTB#N0+QwWE`Xo* zlYVVvNHX^2TaLBX;k#_Peza)fqYzYG!P<1Nh3(_JEsknUf>m=OHI?1gC7shTZ?lax zw7`M5XCjuIW-}J-8$46$v!=1Aahu_ELQCg#6JXPCoY$o1LaNm~O+}o95R0Vi2 zN?Q$+-!BMWCD>&#n%E~yn8N$hL@zV#qG8@6F&qNO+U_>R)%wkCqiqaywEPDDYa*em z0JNOMw9TOc1+Blt%M58XK6v&$X(f64hs=Wn={uvWvV_OkAf_n@s{ZP9 z-k`C{%|ELm+Pyw__zD@$CMerC3^6uELAeQ`HjAGG+zb?m7CsDNx9+;q_rHLxbRv4m1$rXpAjOmo`v%SygPm8Ox=`n2g_GT)t zpHrMmPe_%ayXzu%-uH{D>Uj61$&?*=0zOcG$a)3oor~x$AG5z|I?1KBWVPTb!r8*M zHiU4|W@lu3AK2fyS(xXV%2%=6=d*?MY4ipxn}M zy2Dl8#ZgiCMP}PH-eM>kTD~oW1ZTQ%#|o(Ulv6;_iD#U#L(h1- zX`ucM{pIo79TEKwS6UzQPbsT8zlLj|{&J(>w-FK4%EiI1 zdhea14v#qNnLXbyCrqqGcy-($K5}tuS}t_x=P+C=%2jgFwfJLQ{6{DeA7=e$>QeB< z1#|1|7aBVrci`n}HfY|2$P!*qNLA#L^ct+;*q*-Sqw}EzV&eMsj5;LqOf>0i+)1fe z@283?5*33fx%uTbXE`*dKq9{2U-XwO^eb_*~-gL%esGOuK&WmD;?- zT>9P2tnH0f!|it%+fSSu52bZPy!Jb`oNuFUuoSW5CiGdJv1hm%mj*~5J87*|;c_yo zD)Yj#-lcw73!BME7%}NLS7eZdz@|ongnKcOXD!X44iV5=O5uNY*23A3d6u%=eNsru zbOU$JwI*{iMVXA5fMYyG6m~Z=C4Q!SQrJo}yk0LC`34fS-sjynSva9IzqWFT@HvTS z(H2)7PO_LUm2Td(SPh)Sf5;7oLO^o%6yGKXEN<&4otem?c?GgaH;Oe3b|;w1ND1w0 zpG4o=7!8gBsb()LyARRs)0xmR+ceH@onjljEXbF5s}Z>EqxMlWd}QPTcUs~ViorCu zs+Bmmw-d93)Sik|z?$#0p;Ucf3V(MDN7XWEcaRuYC^Cl{cE)rHUxz1N)N`2-kr-3G zIafF-;}#W{FzB1G`x6?(n`Lz_bM8E46*JlHLa_*dN;1a#}bx|{WP7e%DloPUJcz?n5JLVbWWIj zBN{sU(sF_no452W!F|$$+is1>n&q+`%O&fRF>=SM{-neQUyfs;j@tS28n<`xNv3n0pOtySUriP?RaT z>2w*h)2*=YQ{{(q;0fBl=02$@6`tVL;^Z`Y5%QFiHHlbmyb132i zA1K&ZY0obf(8dKbnhbNwI>_eoshG`>PWkhNZAAq{rtF?K&lhO_T(%e6B{ z>XOXuzc%N;u^(d`v2DG_bmN89^h^gKEjof2aDm$?$G5-6{fnCXRKA8m9g%*de1Ws3 z#gkjH~TobyM7VQMWv#I1f5TY{nD6B9FG9>=k@*Bg0z^lJk9bK);4WB(R zA##$NdAr`049!4r4ouZ!Fm*k}M_@tM#;qN$`0o#24MaB#t2~@Z#?Sbfs`=p1@5TdLrD0OLKgVQ0fc)om=Hy&>-)sjeZySk&3 zXE8ZG;ZgY=qlho>gUkh_8`-k?n}3=om0 z324iiQgK^L9OA3R8XetZHlZyaYClpX5nozmv%Eg(-Gz*XJwPtU zaCik>@G~Zhj*Bg}^=@;{7b{9p?nc`HaImN(YKGOA&v>>rJ4S$NzK$ly1+H#bew~mN z918xrVr=!3`PD_7#(+P^3OVd$jf)Yu=3%_Ivc@)t;xq^Ocl6_$;RK)P;l|35WxUHK zOehd1zN>()`+*9JsbX%a3j0PJKsC2vJ3=Z&W->r12L&+uSaZJk2R^i9n=Ac?U98ho z)tst8M_=P}a5#3h%i4{1 zA2%c}PnTC>iuZrWmAgo~fIy#BxXz3jmriZtkhO^O3{#Q&a8W6VD*TBv>y*da4R=>u ziaD7JBK!%8!iP2~c^mmY2eY?owm6nNA~EFcZCnGpc_kj#p4@2B_6Kg~9b4L(S5Ai{ z+jgYLQ%*unI^>GDM@^U#77Ta7)lwm7ZWWjQ&Jsm^W4Md6>pcStJT2dnUjjAOy#B|r z=6sEu^_QtNy$-fl0LVR1I`kkTGg}~V&iHv3X(9NDmzg^(l5LWmQHc<&8^)Y3DJ}Y< zIDz^vMFTzZhh%>ihr6LZse4U;^hKx;W8c?nd@qk@ZOa#MVNW0 zD56ul(s?VOR>s_4adB$$tYq2w+%v0`$GZ)0oD>n=qDh?i(uX6R4b$T;5`>*@8LA_c z_;$7bXETCVvewKzT?6LbCjnmSSR4>j!A)eY%>}6f0oz{m2x}0ZBgD~m#MM@GG9_Ws z?;djj+2{J7u1axMK>u!g^vu=u($1UJeD;+ShszjE?LYUP^h>84=p~a%lHHGA&NPM7>Q+*22kgB-`}`Gt>dz8~RldAjcP2$O)}$0-T4Nqk#dY0z%Q z^`9vX@|o0@U}%{a6GoomuU*<&^Is`EBPsK{-R;@2+guclW0mkx%%e#)Al(~fcF&uJ%!$n)6Vf&;@ zP>1to(`pr^NdjCRl35$%JDIXDi`pDNzpPq{kGle@iunLLR&bK#ZL_U*wWt%Z<_po4 zr64@HE$;Qhq@Ou_dr~(hAXkb|ZMAN^2I>W>driuFYv1d)doY_dBY8dI))sv$Y!>cFrUhGNbh9}z$k^s~PN2{IVwb@`JmYn@tQeB=~?7E8LoW{^7 z`6kHrM?{yR&D;Idji@fRY)KdG`G|f1L~_OpKq>gR(9bh?0JUSo<|(5R)*HYRY8=j`44Zc)NPE4-k!IA5k_)o)ozlt%k#GEz~K*<|;zx43JiErciB^y&K8!u66=z&SeY3D#qmy_7R2pQZZ^mOQbKdj2RPQrtP z@*C#1+4R+Stq7u0oQ5|~OQi$6-=v%OG z1LH_Fx#@wEi0Z?;h*Yf*lhH?us4kYhYEsiyV=X2q#PAZl@3}*F?FHwuF;=&}NBc;w z4FJ_pkJVBLBG?t^4kcCn#4_g93VV#jS{Y_PDP%NmT6vPoDR8XcU;S!TK+=z6xdm&K zP7FC6#op;e0dkSb^In6mR$o@V>zhLQjEXodr=2_LI!D@Lv8u+F=(Y7Ph`~5IceBjH z?uBD;zy@0N~l*jWc*gSyP2Jm1XsEVBa=u@5$Za@AR=q1=U%~4ng88G0NiJ zx=+3oPSl%!kim;FCxtbV*}4+UUaYaw1+TjLnZCtY9i|wglxV0t#x^0It#mh6%Z?~FI0$w9shTg^ze=E&H@#jXxFrn@qgaEC_zzshVcE5PSwx^}t=mTL3GS+_|ZQl;YZz@cctA z9{`{NXedCpJnGLs_bxjLq+D52umspd5DtQm^RI$^L=6?=-a#b-Fdm}q`{q6Jbji9* z66rfs6vC1eraZN=_T|CIs1?Nn8r&iFuUh9-p2R+S!HPLQMwf62qG<6gCBABD_;v$8 zy(0J&I3409{-z6z1J`UicrFqmXk~RZ55?Tw3-Sd{L5>dH8YHeN#s>naeJJ0C#eac^ z{*yjHgTbgd9wI&tFw}=K@Hcuaj5>MV{|*iOwE?4Pa>ELi)a9Ym=ki^K9CCyJ+i?V7 zqGbcOIupGb?lk!LIuH&RC~E!l@`zMF+#&tufErW;G$W(-r=U_Xp*ys1=2FIEJzRf5q*AHR%~Fv}+51ya-(6;S{{c2A zA`?Gak)MC^pkjKs{JU?Hy}a~}iqQU|)-6ma$-*2{vxNWdcKlz^0K^fU-M5>QR=$2m zk!e*bbl8mp?*3<8Og1Um3{yxsYxJ8Dfd22eSe3#|rJ+^W#O=K0*&mES@(nlu^QW$o z9}TcT7nPKHlTcHUYDlP&Q8xkuDXl}+-j40CmyW9fMIAyuK!)g-toV=m7O})rnb;D~ zM#NtOn?|Lb(MC%r$n!5vP&oMR1Qoe9jey9U-%2(#Y%hK^X7k;6*QI_@GX_#DgVXm^hsdyVFxDZ;C%<>edYtD zIC$(em1a#fI-a!G(ft8jk9Co%=TuQUfAC%KKcr6p-$eWFc@FU1UrU#OCY5d+|M9+m z>nI)Vv16ED3U==)Sv5erVZ10Tc`&BZE@YmDSp68)5azowwL*4>BGc12yPSBKWhCj z-~e!=&!gOi^?=~yuaD6RDXf5r7&FQ*(b~s_9dt%}-q+^=F*7xyHbeeJLo3lDW>Ut* zDZj+_UM7A_sf7lAnM(LuA9whyd{rU1iUTbMG)k`Uz@YuwDy#7F+VxFdI;SLA+LZr? zmi)3K5V~!LGqeX1O@z89djX2VGL ze*B)BoKzMJVjCr3%^>a|)z98fWT=a?0rd zzsejx*y+Q;Q_01&VfKES-mh119Ii2Nc>_(O!O<(%KK{qUf156V2~LAA{>@Q7Kq|%e z+YB15*>#!o-D&Y_T}FVr>K&Wh*umYa5CD7aS?n5h-U*KaH%YtrKS^@-UsUmV81Rad z2=Q6Xd1yu()q|^l!*i+LCB7w}2^&O%#iu$AI(~m8>9C_#yerl?D#8Nf6BXA-jFCAV z_4ioHp7vgEoOJJZowORMeipLMt7PEo*H0TGwIdyEwIXV1uBd~`N%IIc^1xE@waS+f z;VUeUU6}T$ft7<{0f7gw=>Nh>Wdfiqc1BE;4`Ea5 zCWWbNdUjS4FTB)n=LxCm`7W0oX8Etmwfd*^CBnywXs>uI_P-pJX0TDeamTyODpb8y z?=o9Q;tsOR)3+k&8c?_?1ZIefN1M8S0NQIDJMgkYUuKK`G1QK)ekzr3ZJ{QUUna2Z^tnrtR*mBp%qJ9OjrsF8*`OSFid-*o$O)!YQ*&_~ zSaP4K2@xbvg`iIKIrgx#jN^~5Ky08-g9s*ik@*B~79{{!Yd{vADNK5?r9*mmE0fL|^}ti2e^S_TcZ)NlP5;%^|<@u>J7gp87vb;4fVB|3d|- z;J>)G{|3$e<+uMY!vD+i{xSduZkZa-0H0rw@H^B#Mcx30_gnr3_8%PYsJxs4UA_a|h4(7(LY?S`&o>qxFk1 zAV0Uo4yW}VzzkK|On$=Y zI|n|v@XjfVg_`12dR%t|^8rlLAwc8ZcHi+;mh#Ify?MnIKxZ%SXdQ1j(D!N0OI$;) zwl~7X)5%{B@|DYr*}S4J!Ug5BV{HL+6{~aqj=;dC8}l)rge~>6*Ok|m!?INeDCxpG zHye~`qc6PU*5K3~(6aUrC-1KEcbJ!)%kd?AAK0A^ekES>$iaL4AAl>}H9jHK@9ewj z?MJCz9H4d9e177OjnyYa&H z-ww1i^Y%H@ZTy!Fn=+ce0)$456S#uCjE%O(YEHkWGO^Q0slX(`1FxF+{R({R z(1%sWdUU=wmuBN;W$IKU!_AzSt0GXO|ZiZ+(EhUSO7QC0P%M0`!l6s>w zO$QTc#v=AU(4WR35+(xlJv4++xFDZSB7Z6@MCBh`YE1{+XZ~w^Pu`R)nvWO5qK9Nk33?{{NX+aa%6`J7`mS#!@lS}aPOJ!Y&-KHTiClBL zWS5DO%*x3b-0X?+lL8vKG|_Q8Wn<)YxNlwOjRTOow>ag%`4&CBjpGxD+LN({dcA>x z=cIoo9`Bp#)OByp<3KUwsH`@xk)JAt$2K5$+J~UShP&F0$NOjBTiJ8vdd0X}UL6Vd zu1tou`m7dK*^jmFru0BxhHF>WN)n%v3ME~df0IWSct?w8c6#~}>Tj9pru1p(F1lBS zH#{x#>~*Zt-oz*b-N#!Bu;-2m-;|gavv%#jJIb0U%q}o49M8%rI4QYC>>^g0pgEgX zayQb1*^79^AbK(+!|oUliN-dta$vSaDbxY{tL-(wdvNY6cY9&kYs(F zrg~Xkb?G*Ge!k*Sf5V*}EADPRwKhI^X-nGrUkB*6GJv)XynL}Q7*xH7ULA?p3ogUB zC|T_}Tj7PF!((7U>+uf;YM>iI9R_-FF>CP|m`lanL~Rg2ce;6GmSwv`zijWywzRgg zlTUzs9_10qy|(DNWT;-z?jwDE!ie^Htnp^Y-KxI7u;zVV#%528MdZY4)X0K(<3U41 zLN-VtuwZ%Mvl~4Uf-SyqY4^AcJLLT$p>$`>p!&JtX94xr7|^@O_@GYnkoP3bb0%t_ z*OBD|%;If3#g#f(!|dQ9=JB*WxAAzSZN<{?xM>%+F_=-MW+z4A;S+@l`AS5xqJO@H z@=yMnnZ5$>yb(i)k-C~}#cif-Um|Ma`ef}2f6$n&33{;C*yL-7QSWw`R=1wJS!t~b zw^Oaohi3&r+%5Xwy+UYqX6V3Fu$rkZ^3C?JGzEE1e#q=prx!Fwevq6Q%;g~t<^DEq zNV=0pNqivNG&7dXc*o|i!QnDNF11;au#Xvr7omQ6FQQD?I!bs&SQ;yav=?iScKqH) z*I^7S?rMC79&$CT)Mdfc}&#g#X%vfgJF1v`#TD>_5C>8 z|bDr2M2=sggWtO1c$a5! z-L)yprg#2+AJ((cw_T&j4InsFKutb^X`Vg(=)tnQ>)pp<(>t7ySr_#)vKggz6ii0K zCCxPNiKk0{1b@0erLV#@`V3sgD^zJ6)6w%Y@2pt;UhSKai)J&+4Llu?9Ck-lB@? z`<)vwUwx~kcrMep{9q%h2D(Y#Pvabvo%hn@bDTzYpJCj-IF!xdxwgxI2?MSe-m%DU zmr9p>%$#2!fW9)_;$XB;xXF+1LA%Bsv22<8omh-GIpAQN0?Tf^pID;xs@fbpHrs;4 zgM27@OW#$KM0j|qlJxBY%|CQp+~GNiBw1Gk8w$cXES@-Jr>_&*au);hGNKD7VkPbJ z^jHBkyosvKcwtZqOcT-fS;%?>E}iu@cBoC8CU7FJ`k91mQJv#%}6r8U7mxOOzerp)RN_RuB#)9 zHB?!OG8yW*L#P3Sso}Zg4f;71`J?+{G~-h8g;+6u{YtbX}MG;HY~}z(K~= z6xu8lkhQz%X}N`<zbW5jE9U+&Sgk_qT;0yuEiUM7TG)tLua2c;_n`Hq}z17L5T z9J4LJ1AUF(0Ct6OlG|FB%>X;}zUT85YtW4;sZ;~81Ow6?bhc79 zN5_olO_cBD$1CrPMQ;q>5CTb54?N@-j3GS$#VU!#ug0mPbCQ9**kjlh)CTND)-Hm# z4gzq|rGre*T(3;>CxpA?;daj4YI7X=I#GO2K>lv7$@jB&tzErSlRfj1)~FQhUr<`? zF35f;KCTuigt z=_(gNk+Gu^f--7lgVVE50#{k_8{bupNSE&-guetz%*7=rUG>z zhK>IYWqG<9$F=;2QOG+zgqv_hy-N8&U}@PEOB1z(o9A}^BrHfe|BQt@IC5sc&}oo# z8ty;Ie-;S;F6xz8=>VognX7$9#P*#?$wa&ROIU~rqm5pf%O8eRFVdUNZ#@#;C-(8B zyZ`WaAz*G?-*0J23<%|)yh1oTeL^tlg7XycYOywZ*rWS_ zlXmXMj%#~_WU4~;*dl-;*00mc380q=5W>mNU{u?CZ#;1aR_F5J>b!@Fuq3a>UCGXf zE#y7a-bGN_(jPGEw}Uq$o^V1fkARu`)P!9fXN0F4K4b&q{#Ypek;LCam;Y&Xw9{bs zE1;Jbt@AP-Y#7jAp}L%`{vNe9|l#m$N@EB<99;o(&SZB;xOc&qRI2a$QbBrYh~3l zy=^d4nUSGW;b$5N%#(}-v6+hoOPW!Y>iKcNEg8}D%0n>giecHhnJ8$HQ!LRS z`*a~9>rD;%YQ=HzYk7uPz#ePS$0yG|-6O=7SanF+EwX=cp2K4difcFRAZ-~madYG312*4H4{*6waWLoOTn3H_ z@Y`BT@!11vLP0(juv+U@3q!S$_pN|VK3}zBYWBSbZZGEJCcuiAW~+~O%sG$5hBDsr z(Y|Z~GT_pm6|t4}6wY#yp!5irrwX?{USPU8vY1iw)v)~6`9tR*z){aKQsPZ?tGovL!v z!j{8qrL>ng3gQPi$_8E;+)VPH9R%XPb4Rm-NiVfop1hO<>BWf&X@}*SCbVRPuYM3E zdFD}?6%G@?V!;Q|=3l^FFV-Jn`gggsWUM-C=*%r378Jb-I*Fe zQ_TuBFnN(3a+MlhySx^A{~pk6U^=~k-x+x?{RMUZY@fRjNYh+WgD`@Q%8R6i3IA)@ zUz|WtO8g_Sq(|8ynpMCG=d2UnQ{#l!(otoDhO=4zvmD4FM;l};xP3r?vDhQ-Uiu5e z`#{jDo&e7Uj152x7mJ}rvt_Y>JCKM(8*HLKIF$`#0z-6Voi3+t9bo-GT0QfDZ;l#o z29rdo(jDth%IQ}Vq#82q#l@o?$j!8tuB~%6-ygx?Pdu1sWia5cJ-9zRZ$~*t ztn}JMmrw&kSXsSITm%0>j*16pnYxO3vqULBOko>{6`Xhq*vZq9&FK%qa_yA0^Qgw( zWSG4`?iJO>%pA|Yk#>o?XxT^ZHAv1?i;{kiW3P~YX+WPC4v5pC`3H5^KG>6XO(rj- zN>O~Jwr3nB*Cz1&X*qR{kEi417r90OfAFPrpgcaqA8KoH1S~|ht0phtZD;%IFe`vr zOK)~}-YcsB%uj0ez$pQ1A86Uh;EVcZrV=^fz0U#n^1i6UG9*J+5T1($)VOS-9d^Gr zO3;X_*H?Qdu%3MLqg7GKBD{!xhC3GuW3R^=ZP?I)na4(*@+GOv1o_g+Al5^s>^bAO zEbsu7A+}5KWQ_soE&~v!gzrA0D)1ZX!AkoM{io#F$^`H^v+RE>#?RGqJO|oZPDwP) zXdB9sT0vgV3oRT6Jov(n%p7Uw{KugEQrkr9p}pR+TPnDp(VHqk)~>{26`vWGZ9@0? z)y^6*^h^(;Zl*;qMCq zs*0@;89?0!BmfAG*hM1I za01~ya44W$SIz1jQs>0lW=9u@3b9zN#+?oF(aU@nIpZ2OJx);*EccV0C{MivP)sGQ zHhKQm+`i2bbD*zn((+jk;*ITH>rT2L`3pg+pIQR>gZ4?Bz|})`&a)5%YLeA2CSpHe z^)1;4#=3d4abKV@8snM#8ZMSt7H8X~w75QLer|vgSlV};JmWf}T7QH+h=~K8tI+A% zZ)0@*9g!KdSS6_}VRb!o90Jy|A7}IKqdF8I#D7c|Q|x@#WLCT~E}En7@wkW-HNu6C zjR6WR6Wcpf({VP|dE^h;)PUF~;yxgB)YkbwTTd1Kj~Yk7Qpk>T1Ld(c+`3ESdw%EGjNP zG!vl6nw;l-f<~sZcdOuUxzg$e)H?i9KH)k%P0tW{6@FC8?&$1oh!kRC8w1W(D6Q`B zIf?DBj~;PN+%NUW;ptsQu)N2}{)lR9e41pF5eAfgEx5wL+jf46Wt{ELq>fR^~8K@I+7UYkvE@+kjmTy{IR6 znJknJtUWIv4W!4^z_bmV7yl_aJ$ouneb{!fv`;X{`FEBlt8rcPk?IL23Ib1i@)6?$ zFT^hIdgXKCldKklSA)FE~WTjl|1vlZLxy-tMgTS9oAfAH`MN~ zb1``-So_spTSs&#zTJA0_iF!%ouoJWlu8^KH@0v!hrAx#?zaanCR5U_f0`LHOzG+f z!zKHJ=FV)ho!^L4->{)iTRNGu{eE^tPY-^T$k+XRyMKtuq&=-Q%@j7Oo2EC2b^vRG{J)agCp6e#ctM^YhEERz-VzsF5t@h>4Y78 zAbVvR_XN|!d!vN?fGdQ50%o%|HqP7+c}kF{ zmAU4TOCoN1=}n0aYwMtUdo9eGo!1+c`khXb7DABUgRK-z+O1`gzpwA> z>{Sh<&G7~Oe6*r67qD5h@~ZF~#tD>d^PZJMjA;1-FPZk++E|Ic%HVR|S^b1r0g0Wp zBI4R+3iE7v9i^vZZ7p~e5%MhnYDF5kZ~s9xg1g9@OPW}2vbVAG2r%OhKVYn)F|9k( zIoKrec+O=a5qT*H)c9a8@WTAw$dPS7WP;Ur=PmT7(x%$oJ9|6(JL81_I%0QsSSMG7 zQuA!uG63$sYG8NXH5@*jdv8B?Y3<|mYz}aYagmTY)nU#RxG)5qDDGMq0a-#I&n0>r zFX5q>Bxmi=zG~m|UFV$ZI@kGroX?+;WUuwC=eeKzzSr7&zhB+K z2|VZG82{ISE=RgrCg|x+XYIaVOM8chF1tANhW|!#Q!z2omA}ycUzJSD`q->7=GpIc1=tD%9UGh&s;jI zcjim;TR+{?|CaTy?|&moUsp_Yn{m2}`3}OY=#m}gVp9?)I<9CUD>`7vF0sW!in?)h zLgcK}saa+C)E{D76-ovh!%rHK^rGyGe8(|1?admTSv0vb zR-A@<$`Cp`ZR}JP&r_uq4;k0&?NcNOWam4=-6RL^;?ju@>4N%FmPFD&f4n^-0qznz zgW!%}b6ToybML{NC~$VPslZ_APTVqk-<*-aWm%*4Jpb#~$+^)&!e+Nvw`HF|#XsSa zg;wqe<;{|PA0NJqTyCrIaBV;0eya4h9V`57xNS=8hxq2{Ju`}Da75(}PIe-PHT$GM z(PnebhuqvIT2ij3C=b`C{#y&yE$T<@fGEkLeUby$WltxkK+)1V(r!7ZWy|Eue8jZC zJss22oY{4xnS-0}O&PZr&od}SiG&;4Hqs9QPnUM-ht}Z(5;`_UF6Uf)S58rR9UbY0NyQhQ*W7bOAssfhs|d#}BZJ|2C&-xz{O#w)f3WT` z5{_>V*3Z^6%Dbs%RqAEr>ZHg>7@_!nFh@3v?b_#?68tO_x7h#?D<8Tym zd1YRm`!FUB8TD?gl~&-{YCl^9E9pEAkAxKtbKW<%z4GRoDYc%E)M@;xEd6CRE`^n? zjKe)_hV@3NCwfX@8DHs`hRhmqXQ>S);&YKNnCp8q{b>SofcGYI9VK9Gm#fj^hq~|n z3kmDUJd%5cn7&?dzvIv#y(xbO$xRjNbeCuGsG*|LiJOb@y>E0bT+M>XXZRSs1JPT{ zet9dyG#X=na5JXRvU7hTsD|GxoGh{(JA&7CKiXB$*~PbK)NVBQ=&a=Y7qZ)i-u0-- z(J40OjOZ=D`S$(O->}v1(b7&L>z!ukTT%=foX6_+D#^EKi!#lgqkzgK77j)V*Q#&~^(g)!f zia{yp99eo&P$5m-8O7+`4lAK?zM^`WY2)M@>HCR{Ms|67HK3BYEOu4&TTsvmw$Zoy@n%y;Gb?pg0d-Z%=CPN~WLwTCuh3 zjMi=0%O@J_(LJ#=@}`&0PT3B4dw$}VzMc}%pBmOB1@(^E&WB4qiFc4zJsk_%*Pa|Q zH6k6*Bg}DFgPAI9Ut^!EyS|CiK^_SlFCVGzTk&bpx)erMI7ge?LYO9uu zDHL)os|zDX4YQbSpkd_p%ye)A&F?68vQ+rR!@i~K0*|NUMA$7`;*oG#OQ%WVw6%Kt zsF__vRMd(j#s%2)YEZ}U-##A zfAQeMh^F&{ruONSr8mQ4mVJj5Y-}s>3?$5!-V+jncf8dT`TaQ;?Yv9AIS8a;!e^o%(LHl{Neqs}D+s`9+;(c6eRn8#RvLGN*Nnr*Eyq{)`E|bc zm;SQ+QmNBEB~DA!-%RgD+}h$Hytgx|q*hrfnok}0iW-2HQ=}xm&}b~+k{d=pODqqG;&VqylBvMn?3J^HSxi`e&DcbEbb{!kTMHidl(ln z_TfNsk0pHZOxX3&Yg3h`k__1KhxSPA|rK0WD``_$qMwu^>Vq64h zN&*JG@b>w>u5%td#S^nCE#n>v6)&k4EjxTp)JU6~JJ;+yb<@hBERim%U-iEX9>;c%6AMQVs8)ZTqsgIDYC7g{>4T2)3>VbYoIvo|Lg|czc0NXHvK8@=R`%TCGe_* z!iM@=u#7^?nj~0>V32on-kQOx6~GGtkGs6|j9Nd&8V;5IRAhoj?RLPU7%&>iCvxtR z#V@IWlNfVv;yc&eI~nOEpoxrVI3L>Bj_luX-hH&Yz0$kb*gLl_BYnMN`6Y7aIsWgU z(vR@7l!rqOFOT4ijm_qg?}YE@e@(S_6s$BRzYs(pi88D~^y%L5uFxDFb8E#+&~)C9 z*zT<4q#WP8Iqj%fpU7;Elx&rf)y(XYFTNGiyOYwpCr!CiP&}lJ=s1%jy*g!PPzdH{vuk(tkPOvG`#7%?w;bpdc3a;r{$9bgQYu zGCuP4Bt=ZA;Q!iNfO|m-j9?V8=;q-u*}BnK1--PG54rm)W0N%m*Er5y51(#d+wmgH z=XSw_X)4U?r}0g#WBb`nFEpkN8+0b#Z`aYiqpoIk7jQ||_LoH)8OH?Uzh3ueH~m5J z6)MdTazwYCyCv)NK{NiDv}h2%GSE1~dKZ!a<$W589z0id-GOYtdESArc@8<>MH;pT4-KBRDhKYmojJT@x1^mk%97{isVS9KF}Zy7OvRN)d(3j_QweNT7p zw<>O^H_!WqNNn58D41J{@BjG-z(bv*U!V+9I>sz?Yc_Tq3EQO$9%FpbyhbP^SYPE{ zBC6+|Af}ga-}_qlpkBUtO^4ka@3;mk)R}qfDgOr=*<12NyU1gRS6warWh0nEyNaY^ zw^UFrGB6stC!-b4?t>b~6f{ElXTbHoygoSQ<1@pyWa?^TfQmNOKBAz_r2HBj?+}_} zly`}X33aDggzeB}+(FW8==*yji~dZrJQ@6lVyvjk~!PB3WrU-&JlWEmbBs3Kb`xO@xNr#&-FsK26?%BIUBd1O} zY4lXM?x5Z-mPWkH^nTG3()I58VKv;~NkD=C?&R_BmeB_94E+R zJms7sGE_?^li>8drhzqc{I3G;>97qxu3*CqMj%x6_ogU!!meA47pvb1`;GBRyooJeG+}zs{=1Gh2l4BxnP4f9{WkO) zOiBgpS0YB0ly_6ML>r9mYD6w$>!0}an{sIPP21Wb?sWZ$*N?ik!CJ`n4eU*MSX1{4 z>0z#~t?mc0UnLs)Ud=jE|0UU2Sn2w8PiD$g#bvch*DtE|7$?5m2+)IZ*Y#_C)twP{ zb+pS%}Q1fDK$V#TM#NVy-zJsH`Ho&Oc1eA)<$)edv zSJRnksr&9S?C!+)PY|i%Uu%wZ__So?WhDco3Oe^DG5y9-Rg^)mH_{X3eAxD-&8^m^ zTmTc;Kb{l^C}+0a@VdB@m@rjGiM)8;%i-WRYniUwfw*zCJ$B4e(dcY)Tl4iVivx19 zOhB#9E&W|D+t+p<_aeS~P;?`s?L6L&IcM9Wp&5)p`Ecgjy~V+jJ>AhW;m4!6^v%$r zIxhujdNS^08?{UkHK2q+B|Ot^vfwRsXWEm|eU2}i@-yxf`0zL6{*jTs5w8wvt9^UC zwRQ>L;UI;1C-V7q2d~pw#hoS#R@r~nTc*z$$rF6)iC_)|7n4yIvvnR5*UQh7z1(NI z+pEDBO*<=%g%m(ae{t`!p||H}F*ZHrA80m8ymJp?TX`BcI+L0>OdED*6Ldu^%vlG; zc^LWbLb`f(x{g)GJY`6gUw$a;iQj0h8oW(o$Ro(F4PM; znRLdBO)c`1me~?iON`&$Vnz|Z$ZEAr)s?@$5@c^=@Y3M`JobIPWRSkq`$dNHLCr^B z1Bm_exR)>IfDr*IB?uEStzx^0~|w;sLcXaFu?e(bTHrn4j7VGaLJ_8>i2ITiVR8I4vrVXEqk zRB$+{t7}e5htKRZVB85L)7#%RuQDtDm*Q z$(UC*M!zT+Fh462J?(D0a}XEN)iki9JD|T@NP`xt9puVRuNPlnUY-Om=qisOck)ZG zlfAqyxyvZhgrhF=N6wbk_Gg3lBAN~xQU2;91sSw^g#dk(?&B(obm(a(nwMV8s^`*n zsfBw%h5tCy>%~`v=|^vaLODX(%teoiP;3k=^Ct1*tvyCdtzL)04d$ z$}XuvE2YS1Cgda?zW-vv^yW$D!>CF<|MH+G@2)?&?>eIACnL^#^))$L_V+AmSe5>2 z3LZ?YfNod!eY(LY0hTo}{xEV7(&3RWk|E*v5D3Q-toF`7egj2dlz05!9VBBcNIUNk zIa7tR=?J#ixYyYCDyW%F-U=u7W%%4qHS-T{0lKH2b5fsBp;4ATzMy;F>t|4s#Kk@6 zXZ4LUS!L!cSykP ztq6$pGp~-cTNCd^F4GEti(}VeYHr=P3ncz(*BA}brbohbQ$d~(*mrgpdgFDl?1-6J zM|8d3%&^@UNO7O~eB+AC1|2uoSkSUC=ZlRw5TQBZLTSU6*hqu?VOA={D%G!E>`C_Lw0P36p!$YWsRIp?9+D|);V3rm;NE6$9;{%4D;@8&(O66 z(e?&eeq!cl<2JyVfaVJ;9ULommYh#f@d7JPS4$|omRlH*gU7zF_yhC<0DmdF+`dM2 z{F!!w39Z$LEwDE*Upr=Jf|Q4v&kfVicGGhfOS+l_)3UIFlgz9OL--9U=KI9xu<~SV zn7AG~1b+D^JSw{i+ww$B>+8iAfzpdn?>m^zc8I-r_@`nKl-%3M*+Fc78tf#VpS;DlTm-`9jo#-8RRM`K8{P{1Z1#l)<*>i6Ni-%8CNynUy zcBB_Xq&%*F4O+9Waj;?Dhquqj1 zH&Uj1lkgWVyTXmINm90vTODYqEO!bMv?&=Il$AO*=o;P$on=3>1c0EL(^6qqVLtWd zo(JBReg#)T9GWFQEn)}l+dgm=*tyQbC7b<-ZR05-S|sEEthk3>5Ss!^vi?%LPH>R> z((<@b-=F6|`7~aMtBlLR^NoDl2Y+lcximcwx=gEK@OHGZTJvaV>)22A4a0f|q|kT3MW<;#@{Y?xqV+}z7PMEojsj{S?3i3fUt(p_ zmp-S#vU(oncI%09zxO4ot3w>yJWozdc{2mTbLV5o(>Lg_%NCWe+s`qCqmP+-jT(u? zKa}uy@}<&u%1&S~a(rMl93QMROUw>p+v~Rx%z7f(h`mgWyrZgM0QR7{x;MejKyg76 z$48Hjbm^%i-7W|SxLuiaxB%c2|JoL>;a%GDcpm_LN#8=@G7FmF>X|CF(&|9u!W0TR z-=*0hZs`~ctqJ9K93|K3Eg%u%#`MK)vp8wcOu0!BbSuQqCV=WStyE9W{ouPzyv5Ho z8AQ+=EaGhLC1}K~2DY!zd&TN|hWf z`8$P1B$z*OJOWv#*O8`J&^fg|-fFh)L}zMnsA2)*y(CUfDqT4(9)1%R|BsZ?~6MLpU37|Guab$GIYNqKMkp_Eq~u@`+npB zB^n5W9WV;96xaXY$wY`es z9T9LF1cd+vC9M!7KnLP+mtN+z+XX(ITJm@hrcMWECh+SOB2^{u4;a}+e^oeH{ph0>DjwKOpG{!TXg|o150^t zD~+6coty}?NRCtMlD>66Fqi>+7}0Xa z>nXXv6V39zQBzi0g~W+J9DHN`p8Eg_?^FPbAbJt-1x~n}=D{6c9NEsS3m^or8;PZz zW7we8Rv{~yhZHqhMl0b63i+nHBQ=-;!$9Tni!OkjdUyO^sqqik0Ljw&$Ce@=0Td-? zr@v5xc3~T$z3w6a5l28kIU^d0cYSKn>vQk!=evI9+bAg^*%$sPI3M*Y{H2ylqrWF|c+@Hx1(0Dxr+>=bc4CKvBR z97FutrKhBBmC*oN(jedGQou3fsGcfrSv`BXz}orfUI#EN$~>1b^lf~{rR4I9>u9Qlm(_01w*}pc*T1}=m-K^=YfT9R3Lo@3w3429pHBAZ`+ND(v@%~X<59#& zw_XCGxLeP==;OQ-(@@CH(ipU(;@S?Vi2*nO2>~Kdl=Nkx56FM8VIsS{)`IMk|JNB{ z*@r>iwV}_d-y`!Uk`8+Zcb?o<)&#o*d=#|WI4l&vk+HUWy$;!OlZodAr;p`A=Qp4rIEoSR1Y^f-4SEJk$Al01udvR^-dMI^Q{t;=*i$qTJCm!8NN3)OBt zXL|*Rptd8oKpSLG900TH&i^D^XV+K2P8X(`Ri?p_6g{YH0b2Y6CcNpeqls zDqG-He!Lcbr+MgUcwtBNfReiWEgy8cDL~lVyW2a1*PaZ`siG8v6?RwnxkJYaFIy!3 zsI59~BKTB2rPxxdGp=Hno$OYSM=OBRIFy1N-}_{l;`pSXmX5xJgmnw-53oD{VM0#F z{Wk@*Nu5si=SsO|;iEi{)RqszSywC8GIPC}$Uu0D8f6+?lv_#;NcycPVoxQ@9Z>0BA}3WpYgS&6~eYu?!Ahb z{5u|^;gAmJmTYB_r6j~ECC+3>f@?Nl!G+NV`O%)OCzm*nMa&xnH$s58u}xGFsS%yk zJoB3Vx@dtXKEM3LjKvESy&U6adMqD5?a{*Wp6L*X;~K!5GhedyNn*k$U(a_OOKO|t zJe-<6hYLTp45#;=&po6{F%I;=AVX(38lW`#a{jY1XUiHhWTrvbSDyE=N@*0|7J#N8 zE=r&Xz}(o^01$qT3JNKi=NohJ6W$8&Q2wQDh^`4dI#0bsGQBs@Hj}!AFk{Va^jOY| zX&J9by8ZrMS$O_V>A-Go%ZfhB&LC$i`}JDx^T6jT5&cmEn%w7^JS&$yD{&3tBYW&f z^^=nMp;KR{i6h@<4s^XP6Xr9T0%MIQu`D~TN=km&P)T@*>r(M zX;Y@R%$#E}+{1CpPc#h*8f)Ukg2S zVGh+G$mv{9Ej*ypS*smVMyEkLD7O3uXqN)e?$^dy{2QC=Ed_|PvIaE@(OL}s!nXZT zgL-(|r^}24*rXFJ(wt2rw=u)>R=(am@1$)OhFfmuVMI;5I~F?0ft;wA|5 zOAc;BSdSD|>F3}UJLi8b;5gYS989eoP|wc2S)AGCjfbtmTPuXHn*bg4(dw+4E7DSl z1{S$xX8`NURquZmu21eEU%=pR7X#`Vi~Obs^lS`*>uA78NB*mI4B8DX$pg9-qhOwg zR>1zT;7YlKxkVQo{-%5_bQmAp-2 zX$xLTOy_IE3mL9qg09}}8eoszUDD68EW*7n3=zHN6(!6SIJA=H#2^`c%M~K2X>Txxk9Z zd#!iA7xx;LQ8ZEy^L)O_uzn@t5#luD6}GL*vaod7<{))SC^2O_&qq%U z&$l(F*;j+}8QHy5I7=n#!BTn%howl^+^+p~#8bo)-x|}yZjBpRQ>(~+eU8;=FcdYk zr-&a|?t!k>)6Qt3sL6w>BMrHKjZ%#7tkO`#;P6+ROtS0PiG7Lm;dDLw>q3Gs`ptH$61V=G}!Eo{4q4I zovdb6;-Fw5L$7mnrHLGx2!%jandgY!W^?aFjPfL=+12w6hfBS7bQ%-_%;Aqxe(jSK zVXzB2m$gmmg!;Je;!jnaBB!6EYm^YxtV==BHDXGXq zqbU=X22t%4V1ziR$&O8qubw1vR#0Kma?(_BiCu+gz_LIQIU`4}(_x1| z)i^xIG=k3+*1mkpXPah0ib$z%Kk~9($I=EoijCa~&z^ay-60f*Oyd6FXjze#21d=D zHv5bDAwA>5BF)~RTQy!OtV|xMxlO2S&uZcqRSl3|+6NclHAzCWv;K?e8(BnJ@dpvWMx-X#<3Bp}w4EjV$--MbM!`t^f5Ivot? zWxyB+*yv@8PBW?uN4Hc>GrBTn2cM_pyGYAnrRfsW2>a!+@bd$4oWzi^*K>2YX82}S zra3t1ClrChTd#i=ISlb%PKg8SsIaP0hP|?-3q0IAZs~FG{+=`FG)9H$u)6%ck>24< z^MjPz#l#YoRSgAogC)e{ZXj&C?t_-V4?)5pcrZt*>38%Yew8^1k-5Kn#UTj=BOI)EfiBD{D;oiyfa zK4l#H6{vHdMrGpX0@TpacpY18Q10CfunX#{y~?hkX#fjjNl!f~5lm1be;-)c>-EhCbWaDMbgk|I6JZhjY-`e! zVt~LPZjcCFMdc{u54He`{;fP#KSd(H`%8U*4*VZ{(5^7;5RHU@S!Mkh8Pj*N{KAgx zgd~F2ABXdOfDX|M#lqFqvnf*q<1vaE@|8^|%oZ|ff+1JtX|5v_i$Patt94b6H~gi? z(;+<$bXf$GMrv>^ecINZN||@@;W=;UOi-8A?Y2X!suGq*EDs@Sf@(btqz~Y_#!9V! zzIWYb<>mmX!WclgGucl1-m0~LcTk>@JH;9xBChGD0P8K!f!!ry@JIPT|9#9NEEYPq zvvd?v3qX=Hv+VJI#Rrx|YdtBzUH4~3Z+B=SoX z#}b`>6e6&1w|D+YxlhbvpU?-b-Vn5U0wliqNxwlrn~G3n>H*{;`hd?WuiXYB-qFkZU)TM`IabN2FIWyeUt-c$=v)_+ZtnQww%DcrkRjkdfeb8!oBkqY zKo)@^NRg~Y4akE_KP*d-jI7Gi-vOb&_GJ;;;w7Z2Wj@}jzbP{+n2=4gni?QhGMOqP z$8;!UGL4$0xAe>4=0&Fb_+FsCoVH*aOwP$qDE!+G8L39ko^U0l(nvvlJG}m~ev5syoIr4}O|a{QY+7O$FupYNzDaDeg^b3Yy6c`YM3%wR7rnH08cl@>y_;PJ7<6p_E0cQ#ioONay3J~?d}+ZTyf~i!1q`s zbE~KVxk-AEoAe0sz08sQ5JzZQjhwax-q5!4U5@<~tjUD4g>C1N1d3WBSxmu(W>roy z*w7I4!0DL$Y`(NQ%yr8LkTk}->jW>Fz6Toj{#C4i;wMt8UH*@BSw;3G#DAYMz#{%D zW%!W0Vf&ZW^M#|Sa*hf_y#97M9RYm-t|M@I>=4fxKXiHrajBk&v|Q*xI%kkzcdQy4 zx956YkO-=?-779TyZl&}OywTXMtAMvj2fya3Bef)DD^dNQ$TE#JTrEKM{(gfWBpit zXny9Rt{lYi`p?iMO zelvNus?EKkRE*g!_p03RSX+I=%*TzEYtrf(5Yz_E&K@V|+>M;ZRmdeY|Bl+g^7F%~ zekB)TomDyXR2Fy0&ATRz4;$Q^H>2b!cc}K6(ZGZn_|oMiV8fqhh&R0qeCsY;e?ac8 z%V^f_&jHceBksbq_EWjb0g0cv)R4dc2S^mb&&V8{+|n;EPbYnBbYN8;2@oGC9jrv| zlqDnXv4p#%9gFG5N_*LyER~W)$(Z4@+50ATw$DU8pP(KvJVUAfT5~xPVZ^_^=5-+y zHJ@5Dfo0=Vi}QGI&hG*mRcg2?$aJ)>dv^s|cIsRX5(?jdA*qibM7y#GI@W7@sP{C= zN9&%=7MlcAuI7%+>!zutvmY&R@2>I7S@+`T_LB;zE0_-V4R|N^0kTOP6ATvolj#mj z8p?SvrzTeXJj7|JMkS-NoJFjOAF+TP2U zj`9MI>9zS5&f~EY$^=6@dVNCLr?KhaygcnR9rc#YEv#S>wW=HB@Bh(rI4H32aV{Aa zFd@VRZ`UdvHPrMD{m0iVU7Rpli-R>G13fCQ1J>j;2?nL1P+(AwH+rq1$fOuP61X>Rk;hOAFk2O@pF= zmKIHakyFidOLSK#Pm`X^XSVU`bYoZ6^cK2E)CSW1slp`ZqR^dM_Ps;MDI9NiZ&Brh z)7-_nABMW4J8A+4Qv(`C2Zw8n%-f}q574TSYBaBC|$LUVTFd|!n?z-VE)M0xDf zE@@dQ4&icY1<9>%qc_sb#|#2&3C*JA5SOOF7`R8nK^*T*9Qc1s+lLj2L^}I=MPz11 z+Nqq0k;b?a?WE7L`&xMRgxCYa!kqb8mRn{gEr-rh5gCW$xXU4rZ#4`1ZpUU%VHJ2^ zpZJ5HnCnZVJWxw^7MqWKOqiP(0*cj73|RWG#Jzjb{QZzYVI#F#GgdIvDyTGX*7p~c z>|~mFkUG}oWRx!k*MG@*7SdM~XhX=UF^_IKn0;wF+h@Qaz>e3>5fLjEV5Y&y zEr!-#B~Q*h5T=3h@!HsnRsU+3O4|HRXrHL##XG5GuSBXn zh>Oc%AmfIvxRa)_?|QiDqEWvl`0lVXY`>%EwB1RWHCdUvlNPwy2Iu*1y+vE+ggfxf z$0{5f-rf-Nc!jpD0(y&M%w}uR=CWkph>PiBJ~tLF^tY{b%WOU<8s2Pd5Hc?GB@;q)er%_h!A&Je>GK`yaCchfJSK$P+_~>ogVhh6 zyad5IvcI)Iq?JIYu*KX;)|+&5(J)PtX3d@!KWBD*jf=oGh+Jxm=yROJ!{h1dwAim$ zl@S5?uekM#7(e#7=e6gaZb_+aK|B2kZ6SOdr_qrIE|H z4On{pqNHW{nl3*wo#H!wPPdJrB8{6_bTlp<6r@iwohupQzRlp-Umu9BrRI%BZa>eN zemL)xj%qA-K!-kSgcHrfI&8V>tl<_K+0dLO13Up}(dQ3fP{{O)qEW@ZG0(Ms)`)5f zqNxYo@*n#98=qy~SYiujBoouz!c3lxFj=$2VtrN+rl-6D4tJYvE%O-=_X)J!NLroF zUue!?uPZpLv8ZYF%_|V@aMUchN+sRm`Fo=E*>5yNoRh+4K{Cp<;b*OfizEDXS}J#% zo&682GtDRqL!%-*8fIN&gOZb-oodCSR0>OY5#JAqxpkZ_01P5^^>C{twQP_&Me|OF zHY%E6@u-4cQl^%8@bxfeahuS{ZtmiCfnM>j|XqkAIemMC5d)5=4@lXt_fkTvr`>k z!?5wrb~AN%M5j?1&T!P==7~q^nQc^JtRRv`X0OCh+V-&+K7=Ccv6fRyF;)%BC7bgH z%m>w%Mh(4qOb?&6cT0)cGPJyF3tAUC3?15=%}uXRBHSDhl=*XOQZ~9OYthn8C>$Qo z&2y%uj9ZC|u#v2UhNltk4iEzxCF3CjO_-h$C4tr#Z$zij!@&r&n&vq6_s~6c`a`kO zHO&>bVv`weIqsQe)cX|QUZ#0k3$ZzHHe<$ekMxqjoR3G*mzK^CkXiO|Uq2E>vBJ{_ zW@zTjLl05I*!6ewONpzvygEQ9boF_Ezf1n!FCbKt%!b<1N8Z}IBRq}0+Jz5H!G*e2hdIA*1P{o%bhcrpV#daKmr>oG z%M{P-{Q-Ex;$v>~Fe2x)L3a5(`u+4Yh(| zCTcnke||C5hruwdP$i+)ash|u(#S|f#7Kj7jYepMBT$x~|EXn=eId*nYL`n!oR?}X zfIZObkD*Xa9-r^Mql8I6%iRS*g(v{zxBUUeU8B9a5)fhfC-b4)r%ZS>zkgdb{s_yo z^V6ocjzyFQc*qVC{rwv+n!rtB4)$*0Zp$j(`KKS+`mB{p`eX-XT(u-@n6o&KKS_0S zaoCs`>&a>m%E=K|xvh+{&^1dutGhK2C{;oA6M2Z7K#z5j?GC5AIN3JLvkcd{uNZI} zCd=1cRw;uorNS+p-536xCmu;5n^8MIiJV4JX`-S-D5hIurbYUAUd+Sok;}oi@UwaO z6|vKPkv`RfPpm#h5B&LQ?A!WkM7M_dqIXKSWNwpOj!o`^L`A1iPM^=mNWBQAk+WaB zJGmU~7~M^Zs}c~x=>%e3?>1y*ok+3{Jo27nL`b*IXyQ zdz2*o$t`>+RPIB*;(k=={nG(R#ZSjcC!ca#kOkYS=UVKTekgCP95{Ciogk8V<{Gu& zG&m2tViS*=+A4b8yiV@b^^fVBQB*4b@U(B&{x8mDDx%~b7=BES^TQOfLVP!I4a3%? ztXV@TCCyRpLBuEjj|3_H!1&`>{;#s@{|2u0l7m4A353o@F@RJE`J#jAy_Z4XW+Q%%p4I_3ts!P zMzZxTlA&E_Tg9~q>L(u3@s%efGfHRWelCHXbq{{jKR-xl`eC}J&D@7DB&4nEum)f! z`({wnMDkeaXKQge5`sMb9H#j0vcdHp_43h()}RuToX@q3T)MgyA@}|c4UE9Eboc-* z**S7*B6(8pW18g$p+nzZq4H`fSUKf}nt~e)$Czn$?2+m^>7^ zI~Td+{=Pg~&7JS{=1*$BWQm4%YQ3!8Q^OX7@M3F=)G0#XC`>X<0^c-LO_xsAjwAZ~ zdvi#eqQuKnm9DqTTSodrUhhnb)t0Q(fH(K_7Cz0XJh)phPG@>oGC~L1Mq6Bp3Z8D7 zYL-|#b9iYc!sj8v+5me38#^0|Cz%Z6diPZ_qF6N{^aO@Yw6|#2{7fcuh*f9QTp45b zXzQy>XcU!U)yEl>W|=Q7QEgZ;9}>9g(t ziAKC+QU`}2QIYqi;nD?`WEZCAb;wXXTcT|n4R)!Adrc0w#kQS!n0D=Ou@F5+;U?P` zJf-=s+0=i@%Xs7*Tl5Uu!_ap@>n~PsP7>~1j(W)W;Jq#)SCvk7bAgp4M+lEG+sf-z zW7z`(foR1xTN%>7QU<%D8dar!9qp9B#x+fdC@aKX*9WK;m5#62GJe+iz+cg}m zNSN7qsST*(i_}{buF+&PVZ&hhOv{*HzE68dHyo``k&4q8U5)u~yQ1YxA6KEB6o&}` zE9tV1ahZ(-zVcx@*|xUOKP?Rv%O{|g~N4xntUA+%!C-h&@|WZ@)4uscmD|M#oc4&*9^llZWRn7S zqPJJ?jjkwQROt@|i_5HqwTaWz*lDw7x>A5GX$WuDt3EU*Yzg#h;I1Lj%;~>V)GBAz z^hH1So{CM6=DjeVd=z$BWe29GX%%$*9YCO6F+HdJm}djc!P`jB=JX)Mbh#NIOrKMG zC+#g&n%Gq#Zyc`acP>{2Jc-Nm*e8JNmZ>`jbmT{L90+uEsoePX8aU{s{gGKybTBR!co-28+bXObuDJ_85We@@HYA0CEj`6vm`{7t&ub`cDL8c1){l6|b&?U~snpOf`ja2vcivB~!KMhgSg`+M%bQ4y3yZ+?4ib&+K}5LR;6zXH})}ovxR#%Jnpq$1d2I*m04gMnqeH8ZAUh>YqE6q7puX+4c zo-d>9({2uEJ-BcD_5|8{#Wsqz9bPP6Yy2SU-@N>N9WRDs!u9hIopwL1S^H_9h6Diu zrGN^h>aBBPt&>jO>i4+5W=J6(Rr-Kz()JlOZl+AMOT7?^AumkK#lX*f_G~C>Zcz1*eBkz}{Xgqwb4Smr3w* zF8rJkfWwKHayRkS|)-+;s(nf~&(RP+R-{lmM@GWC@|^7zM8I%T=m zKH#&W_@m_UD{?iR=>y9FXV={aHc-rKn%l;#BuAjBN1TPfsKC+8}|9F zhK!YX54+WUgT1T!24kP@`6c&>*NO_v1W}l2?N;#pKV;iYovYhTd8HFs+069<zG^n#zRR{vQdz~J#(7PH zms}pIg&Niw6c`u(7|A9u`Urc;ph??*DX&rmzbWYhCFEJFbK%L&Zg#&sh&FzgYhS48 z>`vU&7aB339tV$>9O=!sCE)b1w*A39$C!~jH&6KQ;foK>ua}CHeHgKu?V@*oXqX>V z+H+H2@_yqIHdLhb*PPrmOTtWou*SS+1kDcm!|_+^nfsMU59ftm^Jc&TkIr8+oD=)J zr2OSDzbmA3k9+Dt+skL94mD73N>jVN0bC7lpD@_yhEVSPO&3((*4b2Yv2z2PXJ0Z^efZVT9GX7vvm77Ip*zd$iQ@L4rSJz*Kg=iUv-u=E$+v*zS`wVJn zBFYhIw=rra36~efLX=2{Yx4Ky7&Ngo$QV>Rc;1y<9?vKmT`f}W;|$Dq&igTxBjTt} z6F89@(oji!zF(Lf?!!cpR+^)9;XN+heQA(ntG(3AXeXsSQCkt1n-uE98?n>Tpf^rK zme(UPv7l7VsqSKnnv+hfUbav$)}_Uc(>B%0mLxoU!b$bvk6Sap6;11`m`9Utb&B%% zOmn|wMj2YzQpNo(wX4!opN`mLt5PxTAKmHKb&nHZnQ4xX%|2zz7pW{z?OL^V*5-Hc zP7z-33iauq%Sr7b{_4LkzVnsP5C^8g7w%Rqai5>8C>;v_)1svoHvJWi3}P;h`N~w;L7FY<+GM3vKct{9MYsUy(ae>m{6-#`O=RD{n3@-LcYGH>TxN8>&>e zzUI8>xp&iIHuo~eW?3T^V;S0J2~T}3onZ1gao*J3%v;ghQaok@;||7tFpwhLPl*?b z?YBp1g2Ou-A#!~RrHe-;Y)c_2#WRUYMX{S^+`+gxJ;M_3v) zyti+*VKL8#S$|AZQD+}Zbb@QR(!4nx6{KAX-s(gZerUco8qqcx1%EnaW`KbU*B_ig zwO2%V_lHFCIOZP~CRn;w_w5U9l}NYTSCYWPq&WHWHJdST`Vr58%Nb5)it(=tlZG<{ zk(~iPMcLt&ew)np?Z&=vZ*6YM*$Nk?6yDHQ7ciWC{Ubf1Lh*KfVfNbQ80&u16HXhI zGxhseOSUdrwh5B{fW9V|s9E0miW}5{H3gG~NXJ;8F-Csx^z8NdLNij11X>EX5_rTT zBSD-rPvnR%8pl;+cd=R)RG0XnJg~`9C6l=&zE>#|H`g=A7hA={Ph0;ao;z5isg$kr zAb9_<-~5(9Wa35haJK`zCxuZGQ$2F+RCF(UeOK~nD-jTr{CTcCdisxQ#GK9^z zq3|B90@%fafem`uaWnDU`Sk}m>ONYhPM_7C@v{r-s^HMu>&-w*7KJRRUNc7;F-atinDgp%{7q-3 zLM(aS9_7Jw;qK+AP#(H@!&@zGPah+)cu}oYv>_e!&{Lgi?>s);T#-iYJ;r?QJTKT@ zPgm33;uS;R!*~#nyw4-MnjX~0A{8)9m))gvMhQjdhEv7aZ<>snr>B()v2(xq zwzr(4lN+e}TCpHY#uw*J{fi_Rxu>;uA`+7nxRSBL>v+o_7+%KDvr?oUT?G4;jH)97 zVuKLuQ1wH*b(c}wFpu|BnT=H*_U`|3d!?---&o+bDbl;)et1fL4C!7PYHTVkdGTpo z_kh+??}SHxV4cDn&hFlP>*277ps#CfVk5g6e;h(aE3y%WZ>K^Yclnm77Fhd#d!Vl) zE%%;=GELE`q>B-qqEv0F&hAPN!T(Xrx&9;-fMGmqt!Zm*UA#5Tt(mSStcwMj&?;wMk@Bc` zflTdY(lorFc}SaX`oShs*_a-h&G1eNiehBVXgMm%R*5HY!w~Vp5kv^Nb@p4UY_TV?Lo3%PFdl6<;989cQ~RYzts$HS?I3wyz0CvxNSI){|FR?;E-{()?5xI zgWlqn1II94ndtGF+^~h+jhkSg;PhdQxLAX#if&*^+zGL+Rl)5zdh9TpmcG$-y{Y=f zAU-^(#29%7^&9%qw^gHwn=jQ`-vuGF_gy1TAL^IG#Ylb+-sO}|B~T)_Az?e9NSKTb zkT4c{D(M`N{QW0%oDUx-a8p14LGFRX0h2w#&^BhPTdcC~fr~XU{U(7k4oSiqRFkU| zrFB$)S^4@h3t4f>X5447d^&E0r?tLl(phROT=RSk)Iua6he?S~JGO`s&fLI9|G-e@ z(H&qRs%FBzEy)Eln_Xf-_I9_+5GM=WQO!}oY%NoZb@1AtQHz7|waqBG35O#1!rmT# z_Co&&Sn#R>ppS=q*jsBFsbf{aC(`2 zNc2ao(FqIW?dGvXs!$ZTP9wNm8C1*W+=%wMQc>_Q>ZVGm??OL^sMn~ruOcHL{TW;e z&L1K^UqjyDrKY0P5}S-l4p3-iXj@Uc2LfvpN_k&69}*j99Z-HYYD>*AoU%jB`rL+6 zvvkBsnjz#P$QQKk=;Du9@*M11Tb$K%tm>1|1YKtp81*2DGHPZWv{mP#s_RZ-dB?3O zta4sQJXT{CyO`9hTsz`oMmg#gSNUtgd9CbN7=V4OE{jap*vW2Kk8*s^L{X+J(FYF2ZCx+{Z1;oZ5XsPBvdx4=@AXY*;w! zsgqa~@>GMA)kU literal 0 HcmV?d00001 diff --git a/minigpt4/conversation/conversation.py b/minigpt4/conversation/conversation.py index 676d89f..518d29d 100644 --- a/minigpt4/conversation/conversation.py +++ b/minigpt4/conversation/conversation.py @@ -117,7 +117,6 @@ CONV_VISION = Conversation( ) - class Chat: def __init__(self, model, vis_processor, device='cuda:0'): self.device = device @@ -131,6 +130,8 @@ class Chat: if len(conv.messages) > 0 and conv.messages[-1][0] == conv.roles[0] \ and conv.messages[-1][1][-6:] == '': # last message is image. conv.messages[-1][1] = ' '.join([conv.messages[-1][1], text]) + elif len(conv.messages) == 0: + conv.append_message(conv.roles[0], " " + text) else: conv.append_message(conv.roles[0], text) @@ -181,7 +182,7 @@ class Chat: if len(image.shape) == 3: image = image.unsqueeze(0) image = image.to(self.device) - + print(image.shape) image_emb, _ = self.model.encode_img(image) img_list.append(image_emb) conv.append_message(conv.roles[0], "") @@ -189,9 +190,16 @@ class Chat: # self.conv.append_message(self.conv.roles[1], msg) return msg - def get_context_emb(self, conv, img_list): + def get_context_emb(self, conv, img_list=None): + if img_list is None: + img = torch.zeros((1, 3, 224, 224)).to(self.device) + image_emb, _ = self.model.encode_img(img) + img_list = [image_emb] + prompt = conv.get_prompt() + print(prompt) prompt_segs = prompt.split('') + assert len(prompt_segs) == len(img_list) + 1, "Unmatched numbers of image placeholders and images." seg_tokens = [ self.model.llama_tokenizer( @@ -200,6 +208,7 @@ class Chat: for i, seg in enumerate(prompt_segs) ] seg_embs = [self.model.llama_model.model.embed_tokens(seg_t) for seg_t in seg_tokens] + mixed_embs = [emb for pair in zip(seg_embs[:-1], img_list) for emb in pair] + [seg_embs[-1]] mixed_embs = torch.cat(mixed_embs, dim=1) return mixed_embs